diff options
107 files changed, 22461 insertions, 7728 deletions
@@ -10,3 +10,4 @@ MANIFEST *.pyc *.log *.*~ +*.csv @@ -1,3 +1,47 @@ +0.7.0 Jun 01 2015: +Client: + o Do not depend on pysqlite2 (#2945). + o Reset syncer connection when getting HTTP error during sync (#5855). + o Wait for last post request to finish before starting a new one (#5975). + o Use TLS v1 in soledad client. Fixes partially #6437 + o Retry on sqlcipher thread timeouts (#6625). + o Fix the order of insertion of documents when using workers for decrypting + incoming documents during a sync. Closes #6757. + o Add dependency on Twisted. Closes #6797. + o Fix the log message when a local secret is not found so it's less + confusing. Closes #6892. + o Always initialize the sync db to allow for both asynchronous encryption + and asynchronous decryption when syncing. + o Fallback to utf-8 if confidence on chardet guessing is too low. + o Refactor asynchronous encryption/decryption code to its own file. + o Fix logging and graceful failing when exceptions are raised during sync. + o Improve log messages when concurrently fetching documents from the server. + o Store all incoming documents in the sync db (#5895). + o Include the IV in the encrypted document MAC (#6400). + o Adapt soledad to the new events api on leap.common. Related to #6359. + o Add a pool of HTTP/HTTPS connections that is able to verify the server + certificate against a given CA certificate. + o Use twisted.enterprise.adbapi for access to the sync database. + o Use twisted.web.client for client sync. + +Common: + o Include couch design docs source files in source distribution and only + compile ddocs.py when building the package (#5896). + o Bail out if cdocs/ dir does not exist. Closes: #6671 + o Remove unneeded parameters from CouchServerState initialization. Closes + #6833. + o Adapt soledad to the new events api on leap.common. Related to #6359. + +Server: + o Run daemon as user soledad (#6436). + o Avoid use of SSLv3 (#6437). + o Fix server initscript location (#6557). + o Add dependency on Twisted. Closes #6797. + o Remove unneeded parameters from CouchServerState initialization. Closes + #6833. + o Fix server daemon uid and gid by passing them to twistd on the initscript. + o Use monthly token databases. Closes #6785. + 0.6.5 Apr 09 2015: Server: o Remove unneeded parameters from CouchServerState initialization. Closes @@ -27,10 +27,16 @@ repository: :target: https://crate.io/packages/leap.soledad.server -Library dependencies --------------------- +Compatibility +------------- + +* Soledad Server >= 0.7.0 is incompatible with client < 0.7.0 because of + modifications on encrypted document MAC calculation. + +* Soledad Server >= 0.7.0 is incompatible with LEAP Platform < 0.6.1 because + that platform version implements ephemeral tokens databases and Soledad + Server needs to act accordingly. -* ``libsqlite3-dev`` Tests ----- diff --git a/client/changes/VERSION_COMPAT b/client/changes/VERSION_COMPAT index e69de29b..cc00ecf7 100644 --- a/client/changes/VERSION_COMPAT +++ b/client/changes/VERSION_COMPAT @@ -0,0 +1,10 @@ +################################################# +# This file keeps track of the recent changes +# introduced in internal leap dependencies. +# Add your changes here so we can properly update +# requirements.pip during the release process. +# (leave header when resetting) +################################################# +# +# BEGIN DEPENDENCY LIST ------------------------- +# leap.foo.bar>=x.y.z diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip index ae8d2dac..9fffdbe3 100644 --- a/client/pkg/requirements.pip +++ b/client/pkg/requirements.pip @@ -1,25 +1,17 @@ -pysqlcipher +pysqlcipher>2.6.3 simplejson u1db scrypt pycryptopp cchardet -taskthread zope.proxy +twisted -# -# leap deps -# +# leap deps -- bump me! +leap.common>=0.4 +leap.soledad.common>=0.6.5 -leap.soledad.common>=0.3.8 - -# -# XXX things to fix yet: -# - -# this is not strictly needed by us, but we need it -# until u1db adds it to its release as a dep. +# XXX -- fix me! +# oauth is not strictly needed by us, but we need it until u1db adds it to its +# release as a dep. oauth - -# pysqlite should not be a dep, see #2945 -pysqlite diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 07255406..245a8971 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -16,1351 +16,12 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. """ Soledad - Synchronization Of Locally Encrypted Data Among Devices. - -Soledad is the part of LEAP that manages storage and synchronization of -application data. It is built on top of U1DB reference Python API and -implements (1) a SQLCipher backend for local storage in the client, (2) a -SyncTarget that encrypts data before syncing, and (3) a CouchDB backend for -remote storage in the server side. -""" -import binascii -import errno -import httplib -import logging -import os -import socket -import ssl -import urlparse -import hmac - -from hashlib import sha256 - -try: - import cchardet as chardet -except ImportError: - import chardet - -from u1db.remote import http_client -from u1db.remote.ssl_match_hostname import match_hostname - -import scrypt -import simplejson as json - -from leap.common.config import get_path_prefix -from leap.soledad.common import ( - SHARED_DB_NAME, - soledad_assert, - soledad_assert_type -) -from leap.soledad.common.errors import ( - InvalidTokenError, - NotLockedError, - AlreadyLockedError, - LockTimedOutError, -) -from leap.soledad.common.crypto import ( - MacMethods, - UnknownMacMethod, - WrongMac, - MAC_KEY, - MAC_METHOD_KEY, -) -from leap.soledad.client.events import ( - SOLEDAD_CREATING_KEYS, - SOLEDAD_DONE_CREATING_KEYS, - SOLEDAD_DOWNLOADING_KEYS, - SOLEDAD_DONE_DOWNLOADING_KEYS, - SOLEDAD_UPLOADING_KEYS, - SOLEDAD_DONE_UPLOADING_KEYS, - SOLEDAD_NEW_DATA_TO_SYNC, - SOLEDAD_DONE_DATA_SYNC, - signal, -) -from leap.soledad.common.document import SoledadDocument -from leap.soledad.client.crypto import SoledadCrypto -from leap.soledad.client.shared_db import SoledadSharedDatabase -from leap.soledad.client.sqlcipher import open as sqlcipher_open -from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.client.target import SoledadSyncTarget - - -logger = logging.getLogger(name=__name__) - - -# -# Constants -# - -SOLEDAD_CERT = None """ -Path to the certificate file used to certify the SSL connection between -Soledad client and server. -""" - - -# -# Soledad: local encrypted storage and remote encrypted sync. -# - -class NoStorageSecret(Exception): - """ - Raised when trying to use a storage secret but none is available. - """ - pass - - -class PassphraseTooShort(Exception): - """ - Raised when trying to change the passphrase but the provided passphrase is - too short. - """ - - -class BootstrapSequenceError(Exception): - """ - Raised when an attempt to generate a secret and store it in a recovery - documents on server failed. - """ - - -class Soledad(object): - """ - Soledad provides encrypted data storage and sync. - - A Soledad instance is used to store and retrieve data in a local encrypted - database and synchronize this database with Soledad server. - - This class is also responsible for bootstrapping users' account by - creating cryptographic secrets and/or storing/fetching them on Soledad - server. - - Soledad uses C{leap.common.events} to signal events. The possible events - to be signaled are: - - SOLEDAD_CREATING_KEYS: emitted during bootstrap sequence when key - generation starts. - SOLEDAD_DONE_CREATING_KEYS: emitted during bootstrap sequence when key - generation finishes. - SOLEDAD_UPLOADING_KEYS: emitted during bootstrap sequence when soledad - starts sending keys to server. - SOLEDAD_DONE_UPLOADING_KEYS: emitted during bootstrap sequence when - soledad finishes sending keys to server. - SOLEDAD_DOWNLOADING_KEYS: emitted during bootstrap sequence when - soledad starts to retrieve keys from server. - SOLEDAD_DONE_DOWNLOADING_KEYS: emitted during bootstrap sequence when - soledad finishes downloading keys from server. - SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when - there's indeed new data to be synchronized between local database - replica and server's replica. - SOLEDAD_DONE_DATA_SYNC: emitted inside C{sync()} method when it has - finished synchronizing with remote replica. - """ - - LOCAL_DATABASE_FILE_NAME = 'soledad.u1db' - """ - The name of the local SQLCipher U1DB database file. - """ - - STORAGE_SECRETS_FILE_NAME = "soledad.json" - """ - The name of the file where the storage secrets will be stored. - """ - - GENERATED_SECRET_LENGTH = 1024 - """ - The length of the generated secret used to derive keys for symmetric - encryption for local and remote storage. - """ - - LOCAL_STORAGE_SECRET_LENGTH = 512 - """ - The length of the secret used to derive a passphrase for the SQLCipher - database. - """ - - REMOTE_STORAGE_SECRET_LENGTH = \ - GENERATED_SECRET_LENGTH - LOCAL_STORAGE_SECRET_LENGTH - """ - The length of the secret used to derive an encryption key and a MAC auth - key for remote storage. - """ - - SALT_LENGTH = 64 - """ - The length of the salt used to derive the key for the storage secret - encryption. - """ - - MINIMUM_PASSPHRASE_LENGTH = 6 - """ - The minimum length for a passphrase. The passphrase length is only checked - when the user changes her passphrase, not when she instantiates Soledad. - """ - - IV_SEPARATOR = ":" - """ - A separator used for storing the encryption initial value prepended to the - ciphertext. - """ - - UUID_KEY = 'uuid' - STORAGE_SECRETS_KEY = 'storage_secrets' - SECRET_KEY = 'secret' - CIPHER_KEY = 'cipher' - LENGTH_KEY = 'length' - KDF_KEY = 'kdf' - KDF_SALT_KEY = 'kdf_salt' - KDF_LENGTH_KEY = 'kdf_length' - KDF_SCRYPT = 'scrypt' - CIPHER_AES256 = 'aes256' - """ - Keys used to access storage secrets in recovery documents. - """ - - DEFAULT_PREFIX = os.path.join(get_path_prefix(), 'leap', 'soledad') - """ - Prefix for default values for path. - """ - - def __init__(self, uuid, passphrase, secrets_path, local_db_path, - server_url, cert_file, - auth_token=None, secret_id=None, defer_encryption=True): - """ - Initialize configuration, cryptographic keys and dbs. - - :param uuid: User's uuid. - :type uuid: str - - :param passphrase: The passphrase for locking and unlocking encryption - secrets for local and remote storage. - :type passphrase: unicode - - :param secrets_path: Path for storing encrypted key used for - symmetric encryption. - :type secrets_path: str - - :param local_db_path: Path for local encrypted storage db. - :type local_db_path: str - - :param server_url: URL for Soledad server. This is used either to sync - with the user's remote db and to interact with the - shared recovery database. - :type server_url: str - - :param cert_file: Path to the certificate of the ca used - to validate the SSL certificate used by the remote - soledad server. - :type cert_file: str - - :param auth_token: Authorization token for accessing remote databases. - :type auth_token: str - - :param secret_id: The id of the storage secret to be used. - :type secret_id: str - - :param defer_encryption: Whether to defer encryption/decryption of - documents, or do it inline while syncing. - :type defer_encryption: bool - - :raise BootstrapSequenceError: Raised when the secret generation and - storage on server sequence has failed - for some reason. - """ - # get config params - self._uuid = uuid - soledad_assert_type(passphrase, unicode) - self._passphrase = passphrase - # init crypto variables - self._secrets = {} - self._secret_id = secret_id - self._defer_encryption = defer_encryption - - self._init_config(secrets_path, local_db_path, server_url) - - self._set_token(auth_token) - self._shared_db_instance = None - # configure SSL certificate - global SOLEDAD_CERT - SOLEDAD_CERT = cert_file - # initiate bootstrap sequence - self._bootstrap() # might raise BootstrapSequenceError() - - def _init_config(self, secrets_path, local_db_path, server_url): - """ - Initialize configuration using default values for missing params. - """ - # initialize secrets_path - self._secrets_path = secrets_path - if self._secrets_path is None: - self._secrets_path = os.path.join( - self.DEFAULT_PREFIX, self.STORAGE_SECRETS_FILE_NAME) - # initialize local_db_path - self._local_db_path = local_db_path - if self._local_db_path is None: - self._local_db_path = os.path.join( - self.DEFAULT_PREFIX, self.LOCAL_DATABASE_FILE_NAME) - # initialize server_url - self._server_url = server_url - soledad_assert( - self._server_url is not None, - 'Missing URL for Soledad server.') - - # - # initialization/destruction methods - # - - def _get_or_gen_crypto_secrets(self): - """ - Retrieves or generates the crypto secrets. - - Might raise BootstrapSequenceError - """ - doc = self._get_secrets_from_shared_db() - - if doc: - logger.info( - 'Found cryptographic secrets in shared recovery ' - 'database.') - _, mac = self.import_recovery_document(doc.content) - if mac is False: - self.put_secrets_in_shared_db() - self._store_secrets() # save new secrets in local file - if self._secret_id is None: - self._set_secret_id(self._secrets.items()[0][0]) - else: - # STAGE 3 - there are no secrets in server also, so - # generate a secret and store it in remote db. - logger.info( - 'No cryptographic secrets found, creating new ' - ' secrets...') - self._set_secret_id(self._gen_secret()) - try: - self._put_secrets_in_shared_db() - except Exception as ex: - # storing generated secret in shared db failed for - # some reason, so we erase the generated secret and - # raise. - try: - os.unlink(self._secrets_path) - except OSError as e: - if e.errno != errno.ENOENT: # no such file or directory - logger.exception(e) - logger.exception(ex) - raise BootstrapSequenceError( - 'Could not store generated secret in the shared ' - 'database, bailing out...') - - def _bootstrap(self): - """ - Bootstrap local Soledad instance. - - Soledad Client bootstrap is the following sequence of stages: - - * stage 0 - local environment setup. - - directory initialization. - - crypto submodule initialization - * stage 1 - local secret loading: - - if secrets exist locally, load them. - * stage 2 - remote secret loading: - - else, if secrets exist in server, download them. - * stage 3 - secret generation: - - else, generate a new secret and store in server. - * stage 4 - database initialization. - - This method decides which bootstrap stages have already been performed - and performs the missing ones in order. - - :raise BootstrapSequenceError: Raised when the secret generation and - storage on server sequence has failed for some reason. - """ - # STAGE 0 - local environment setup - self._init_dirs() - self._crypto = SoledadCrypto(self) - - secrets_problem = None - - # STAGE 1 - verify if secrets exist locally - if not self._has_secret(): # try to load from local storage. - - # STAGE 2 - there are no secrets in local storage, so try to fetch - # encrypted secrets from server. - logger.info( - 'Trying to fetch cryptographic secrets from shared recovery ' - 'database...') - - # --- start of atomic operation in shared db --- - - # obtain lock on shared db - token = timeout = None - try: - token, timeout = self._shared_db.lock() - except AlreadyLockedError: - raise BootstrapSequenceError('Database is already locked.') - except LockTimedOutError: - raise BootstrapSequenceError('Lock operation timed out.') - - try: - self._get_or_gen_crypto_secrets() - except Exception as e: - secrets_problem = e - - # release the lock on shared db - try: - self._shared_db.unlock(token) - self._shared_db.close() - except NotLockedError: - # for some reason the lock expired. Despite that, secret - # loading or generation/storage must have been executed - # successfully, so we pass. - pass - except InvalidTokenError: - # here, our lock has not only expired but also some other - # client application has obtained a new lock and is currently - # doing its thing in the shared database. Using the same - # reasoning as above, we assume everything went smooth and - # pass. - pass - except Exception as e: - logger.error("Unhandled exception when unlocking shared " - "database.") - logger.exception(e) - - # --- end of atomic operation in shared db --- - - # STAGE 4 - local database initialization - if secrets_problem is None: - self._init_db() - else: - raise secrets_problem - - def _init_dirs(self): - """ - Create work directories. - - :raise OSError: in case file exists and is not a dir. - """ - paths = map( - lambda x: os.path.dirname(x), - [self._local_db_path, self._secrets_path]) - for path in paths: - try: - if not os.path.isdir(path): - logger.info('Creating directory: %s.' % path) - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - - def _init_db(self): - """ - Initialize the U1DB SQLCipher database for local storage. - - Currently, Soledad uses the default SQLCipher cipher, i.e. - 'aes-256-cbc'. We use scrypt to derive a 256-bit encryption key and - uses the 'raw PRAGMA key' format to handle the key to SQLCipher. - - The first C{self.REMOTE_STORAGE_SECRET_LENGTH} bytes of the storage - secret are used for remote storage encryption. We use the next - C{self.LOCAL_STORAGE_SECRET} bytes to derive a key for local storage. - From these bytes, the first C{self.SALT_LENGTH} are used as the salt - and the rest as the password for the scrypt hashing. - """ - # salt indexes - salt_start = self.REMOTE_STORAGE_SECRET_LENGTH - salt_end = salt_start + self.SALT_LENGTH - # password indexes - pwd_start = salt_end - pwd_end = salt_start + self.LOCAL_STORAGE_SECRET_LENGTH - # calculate the key for local encryption - secret = self._get_storage_secret() - key = scrypt.hash( - secret[pwd_start:pwd_end], # the password - secret[salt_start:salt_end], # the salt - buflen=32, # we need a key with 256 bits (32 bytes) - ) - - self._db = sqlcipher_open( - self._local_db_path, - binascii.b2a_hex(key), # sqlcipher only accepts the hex version - create=True, - document_factory=SoledadDocument, - crypto=self._crypto, - raw_key=True, - defer_encryption=self._defer_encryption) - - def close(self): - """ - Close underlying U1DB database. - """ - logger.debug("Closing soledad") - if hasattr(self, '_db') and isinstance( - self._db, - SQLCipherDatabase): - self._db.stop_sync() - self._db.close() - - # - # Management of secret for symmetric encryption. - # - - def _get_storage_secret(self): - """ - Return the storage secret. - - Storage secret is encrypted before being stored. This method decrypts - and returns the stored secret. - - :return: The storage secret. - :rtype: str - """ - # calculate the encryption key - key = scrypt.hash( - self._passphrase_as_string(), - # the salt is stored base64 encoded - binascii.a2b_base64( - self._secrets[self._secret_id][self.KDF_SALT_KEY]), - buflen=32, # we need a key with 256 bits (32 bytes). - ) - # recover the initial value and ciphertext - iv, ciphertext = self._secrets[self._secret_id][self.SECRET_KEY].split( - self.IV_SEPARATOR, 1) - ciphertext = binascii.a2b_base64(ciphertext) - return self._crypto.decrypt_sym(ciphertext, key, iv=iv) - - def _set_secret_id(self, secret_id): - """ - Define the id of the storage secret to be used. - - This method will also replace the secret in the crypto object. - - :param secret_id: The id of the storage secret to be used. - :type secret_id: str - """ - self._secret_id = secret_id - - def _load_secrets(self): - """ - Load storage secrets from local file. - """ - # does the file exist in disk? - if not os.path.isfile(self._secrets_path): - raise IOError('File does not exist: %s' % self._secrets_path) - # read storage secrets from file - content = None - with open(self._secrets_path, 'r') as f: - content = json.loads(f.read()) - _, mac = self.import_recovery_document(content) - if mac is False: - self._store_secrets() - self._put_secrets_in_shared_db() - # choose first secret if no secret_id was given - if self._secret_id is None: - self._set_secret_id(self._secrets.items()[0][0]) - - def _has_secret(self): - """ - Return whether there is a storage secret available for use or not. - - :return: Whether there's a storage secret for symmetric encryption. - :rtype: bool - """ - if self._secret_id is None or self._secret_id not in self._secrets: - try: - self._load_secrets() # try to load from disk - except IOError, e: - logger.warning('IOError: %s' % str(e)) - try: - self._get_storage_secret() - return True - except Exception: - return False - - def _gen_secret(self): - """ - Generate a secret for symmetric encryption and store in a local - encrypted file. - - This method emits the following signals: - - * SOLEDAD_CREATING_KEYS - * SOLEDAD_DONE_CREATING_KEYS - - A secret has the following structure: - - { - '<secret_id>': { - 'kdf': 'scrypt', - 'kdf_salt': '<b64 repr of salt>' - 'kdf_length': <key length> - 'cipher': 'aes256', - 'length': <secret length>, - 'secret': '<encrypted b64 repr of storage_secret>', - } - } - - :return: The id of the generated secret. - :rtype: str - """ - signal(SOLEDAD_CREATING_KEYS, self._uuid) - # generate random secret - secret = os.urandom(self.GENERATED_SECRET_LENGTH) - secret_id = sha256(secret).hexdigest() - # generate random salt - salt = os.urandom(self.SALT_LENGTH) - # get a 256-bit key - key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) - iv, ciphertext = self._crypto.encrypt_sym(secret, key) - self._secrets[secret_id] = { - # leap.soledad.crypto submodule uses AES256 for symmetric - # encryption. - self.KDF_KEY: self.KDF_SCRYPT, - self.KDF_SALT_KEY: binascii.b2a_base64(salt), - self.KDF_LENGTH_KEY: len(key), - self.CIPHER_KEY: self.CIPHER_AES256, - self.LENGTH_KEY: len(secret), - self.SECRET_KEY: '%s%s%s' % ( - str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), - } - self._store_secrets() - signal(SOLEDAD_DONE_CREATING_KEYS, self._uuid) - return secret_id - - def _store_secrets(self): - """ - Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}. - """ - with open(self._secrets_path, 'w') as f: - f.write( - json.dumps( - self.export_recovery_document())) - - def change_passphrase(self, new_passphrase): - """ - Change the passphrase that encrypts the storage secret. - - :param new_passphrase: The new passphrase. - :type new_passphrase: unicode - - :raise NoStorageSecret: Raised if there's no storage secret available. - """ - # maybe we want to add more checks to guarantee passphrase is - # reasonable? - soledad_assert_type(new_passphrase, unicode) - if len(new_passphrase) < self.MINIMUM_PASSPHRASE_LENGTH: - raise PassphraseTooShort( - 'Passphrase must be at least %d characters long!' % - self.MINIMUM_PASSPHRASE_LENGTH) - # ensure there's a secret for which the passphrase will be changed. - if not self._has_secret(): - raise NoStorageSecret() - secret = self._get_storage_secret() - # generate random salt - new_salt = os.urandom(self.SALT_LENGTH) - # get a 256-bit key - key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32) - iv, ciphertext = self._crypto.encrypt_sym(secret, key) - # XXX update all secrets in the dict - self._secrets[self._secret_id] = { - # leap.soledad.crypto submodule uses AES256 for symmetric - # encryption. - self.KDF_KEY: self.KDF_SCRYPT, # TODO: remove hard coded kdf - self.KDF_SALT_KEY: binascii.b2a_base64(new_salt), - self.KDF_LENGTH_KEY: len(key), - self.CIPHER_KEY: self.CIPHER_AES256, - self.LENGTH_KEY: len(secret), - self.SECRET_KEY: '%s%s%s' % ( - str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), - } - self._passphrase = new_passphrase - self._store_secrets() - self._put_secrets_in_shared_db() - - # - # General crypto utility methods. - # - - @property - def _shared_db(self): - """ - Return an instance of the shared recovery database object. - - :return: The shared database. - :rtype: SoledadSharedDatabase - """ - if self._shared_db_instance is None: - self._shared_db_instance = SoledadSharedDatabase.open_database( - urlparse.urljoin(self.server_url, SHARED_DB_NAME), - self._uuid, - False, # db should exist at this point. - creds=self._creds) - return self._shared_db_instance - - def _shared_db_doc_id(self): - """ - Calculate the doc_id of the document in the shared db that stores key - material. - - :return: the hash - :rtype: str - """ - return sha256( - '%s%s' % - (self._passphrase_as_string(), self.uuid)).hexdigest() - - def _get_secrets_from_shared_db(self): - """ - Retrieve the document with encrypted key material from the shared - database. - - :return: a document with encrypted key material in its contents - :rtype: SoledadDocument - """ - signal(SOLEDAD_DOWNLOADING_KEYS, self._uuid) - db = self._shared_db - if not db: - logger.warning('No shared db found') - return - doc = db.get_doc(self._shared_db_doc_id()) - signal(SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid) - return doc - - def _put_secrets_in_shared_db(self): - """ - Assert local keys are the same as shared db's ones. - - Try to fetch keys from shared recovery database. If they already exist - in the remote db, assert that that data is the same as local data. - Otherwise, upload keys to shared recovery database. - """ - soledad_assert( - self._has_secret(), - 'Tried to send keys to server but they don\'t exist in local ' - 'storage.') - # try to get secrets doc from server, otherwise create it - doc = self._get_secrets_from_shared_db() - if doc is None: - doc = SoledadDocument( - doc_id=self._shared_db_doc_id()) - # fill doc with encrypted secrets - doc.content = self.export_recovery_document() - # upload secrets to server - signal(SOLEDAD_UPLOADING_KEYS, self._uuid) - db = self._shared_db - if not db: - logger.warning('No shared db found') - return - db.put_doc(doc) - signal(SOLEDAD_DONE_UPLOADING_KEYS, self._uuid) - - # - # Document storage, retrieval and sync. - # - - def put_doc(self, doc): - """ - Update a document in the local encrypted database. - - ============================== WARNING ============================== - This method converts the document's contents to unicode in-place. This - means that after calling C{put_doc(doc)}, the contents of the - document, i.e. C{doc.content}, might be different from before the - call. - ============================== WARNING ============================== - - :param doc: the document to update - :type doc: SoledadDocument - - :return: the new revision identifier for the document - :rtype: str - """ - doc.content = self._convert_to_unicode(doc.content) - return self._db.put_doc(doc) - - def delete_doc(self, doc): - """ - Delete a document from the local encrypted database. - - :param doc: the document to delete - :type doc: SoledadDocument - - :return: the new revision identifier for the document - :rtype: str - """ - return self._db.delete_doc(doc) - - def get_doc(self, doc_id, include_deleted=False): - """ - Retrieve a document from the local encrypted database. - - :param doc_id: the unique document identifier - :type doc_id: str - :param include_deleted: if True, deleted documents will be - returned with empty content; otherwise asking - for a deleted document will return None - :type include_deleted: bool - - :return: the document object or None - :rtype: SoledadDocument - """ - return self._db.get_doc(doc_id, include_deleted=include_deleted) - - def get_docs(self, doc_ids, check_for_conflicts=True, - include_deleted=False): - """ - Get the content for many documents. - - :param doc_ids: a list of document identifiers - :type doc_ids: list - :param check_for_conflicts: if set False, then the conflict check will - be skipped, and 'None' will be returned instead of True/False - :type check_for_conflicts: bool - - :return: iterable giving the Document object for each document id - in matching doc_ids order. - :rtype: generator - """ - return self._db.get_docs( - doc_ids, check_for_conflicts=check_for_conflicts, - include_deleted=include_deleted) - - def get_all_docs(self, include_deleted=False): - """Get the JSON content for all documents in the database. - - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise deleted - documents will not be included in the results. - :return: (generation, [Document]) - The current generation of the database, followed by a list of - all the documents in the database. - """ - return self._db.get_all_docs(include_deleted) - - def _convert_to_unicode(self, content): - """ - Converts content to unicode (or all the strings in content) - - NOTE: Even though this method supports any type, it will - currently ignore contents of lists, tuple or any other - iterable than dict. We don't need support for these at the - moment - - :param content: content to convert - :type content: object - - :rtype: object - """ - if isinstance(content, unicode): - return content - elif isinstance(content, str): - result = chardet.detect(content) - default = "utf-8" - encoding = result["encoding"] or default - try: - content = content.decode(encoding) - except UnicodeError as e: - logger.error("Unicode error: {0!r}. Using 'replace'".format(e)) - content = content.decode(encoding, 'replace') - return content - else: - if isinstance(content, dict): - for key in content.keys(): - content[key] = self._convert_to_unicode(content[key]) - return content - - def create_doc(self, content, doc_id=None): - """ - Create a new document in the local encrypted database. - - :param content: the contents of the new document - :type content: dict - :param doc_id: an optional identifier specifying the document id - :type doc_id: str - - :return: the new document - :rtype: SoledadDocument - """ - return self._db.create_doc( - self._convert_to_unicode(content), doc_id=doc_id) - - def create_doc_from_json(self, json, doc_id=None): - """ - Create a new document. - - You can optionally specify the document identifier, but the document - must not already exist. See 'put_doc' if you want to override an - existing document. - If the database specifies a maximum document size and the document - exceeds it, create will fail and raise a DocumentTooBig exception. - - :param json: The JSON document string - :type json: str - :param doc_id: An optional identifier specifying the document id. - :type doc_id: - :return: The new document - :rtype: SoledadDocument - """ - return self._db.create_doc_from_json(json, doc_id=doc_id) - - def create_index(self, index_name, *index_expressions): - """ - Create an named index, which can then be queried for future lookups. - Creating an index which already exists is not an error, and is cheap. - Creating an index which does not match the index_expressions of the - existing index is an error. - Creating an index will block until the expressions have been evaluated - and the index generated. - - :param index_name: A unique name which can be used as a key prefix - :type index_name: str - :param index_expressions: index expressions defining the index - information. - :type index_expressions: dict - - Examples: - - "fieldname", or "fieldname.subfieldname" to index alphabetically - sorted on the contents of a field. - - "number(fieldname, width)", "lower(fieldname)" - """ - if self._db: - return self._db.create_index( - index_name, *index_expressions) - - def delete_index(self, index_name): - """ - Remove a named index. - - :param index_name: The name of the index we are removing - :type index_name: str - """ - if self._db: - return self._db.delete_index(index_name) - - def list_indexes(self): - """ - List the definitions of all known indexes. - - :return: A list of [('index-name', ['field', 'field2'])] definitions. - :rtype: list - """ - if self._db: - return self._db.list_indexes() - - def get_from_index(self, index_name, *key_values): - """ - Return documents that match the keys supplied. - - You must supply exactly the same number of values as have been defined - in the index. It is possible to do a prefix match by using '*' to - indicate a wildcard match. You can only supply '*' to trailing entries, - (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) - It is also possible to append a '*' to the last supplied value (eg - 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') - - :param index_name: The index to query - :type index_name: str - :param key_values: values to match. eg, if you have - an index with 3 fields then you would have: - get_from_index(index_name, val1, val2, val3) - :type key_values: tuple - :return: List of [Document] - :rtype: list - """ - if self._db: - return self._db.get_from_index(index_name, *key_values) - - def get_count_from_index(self, index_name, *key_values): - """ - Return the count of the documents that match the keys and - values supplied. - - :param index_name: The index to query - :type index_name: str - :param key_values: values to match. eg, if you have - an index with 3 fields then you would have: - get_from_index(index_name, val1, val2, val3) - :type key_values: tuple - :return: count. - :rtype: int - """ - if self._db: - return self._db.get_count_from_index(index_name, *key_values) - - def get_range_from_index(self, index_name, start_value, end_value): - """ - Return documents that fall within the specified range. - - Both ends of the range are inclusive. For both start_value and - end_value, one must supply exactly the same number of values as have - been defined in the index, or pass None. In case of a single column - index, a string is accepted as an alternative for a tuple with a single - value. It is possible to do a prefix match by using '*' to indicate - a wildcard match. You can only supply '*' to trailing entries, (eg - 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also - possible to append a '*' to the last supplied value (eg 'val*', '*', - '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') - - :param index_name: The index to query - :type index_name: str - :param start_values: tuples of values that define the lower bound of - the range. eg, if you have an index with 3 fields then you would - have: (val1, val2, val3) - :type start_values: tuple - :param end_values: tuples of values that define the upper bound of the - range. eg, if you have an index with 3 fields then you would have: - (val1, val2, val3) - :type end_values: tuple - :return: List of [Document] - :rtype: list - """ - if self._db: - return self._db.get_range_from_index( - index_name, start_value, end_value) - - def get_index_keys(self, index_name): - """ - Return all keys under which documents are indexed in this index. - - :param index_name: The index to query - :type index_name: str - :return: [] A list of tuples of indexed keys. - :rtype: list - """ - if self._db: - return self._db.get_index_keys(index_name) - - def get_doc_conflicts(self, doc_id): - """ - Get the list of conflicts for the given document. - - :param doc_id: the document id - :type doc_id: str - - :return: a list of the document entries that are conflicted - :rtype: list - """ - if self._db: - return self._db.get_doc_conflicts(doc_id) - - def resolve_doc(self, doc, conflicted_doc_revs): - """ - Mark a document as no longer conflicted. - - :param doc: a document with the new content to be inserted. - :type doc: SoledadDocument - :param conflicted_doc_revs: a list of revisions that the new content - supersedes. - :type conflicted_doc_revs: list - """ - if self._db: - return self._db.resolve_doc(doc, conflicted_doc_revs) - - def sync(self, defer_decryption=True): - """ - Synchronize the local encrypted replica with a remote replica. - - This method blocks until a syncing lock is acquired, so there are no - attempts of concurrent syncs from the same client replica. - - :param url: the url of the target replica to sync with - :type url: str - - :param defer_decryption: Whether to defer the decryption process using - the intermediate database. If False, - decryption will be done inline. - :type defer_decryption: bool - - :return: The local generation before the synchronisation was - performed. - :rtype: str - """ - if self._db: - try: - local_gen = self._db.sync( - urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), - creds=self._creds, autocreate=False, - defer_decryption=defer_decryption) - signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) - return local_gen - except Exception as e: - logger.error("Soledad exception when syncing: %s" % str(e)) - - def stop_sync(self): - """ - Stop the current syncing process. - """ - if self._db: - self._db.stop_sync() - - def need_sync(self, url): - """ - Return if local db replica differs from remote url's replica. - - :param url: The remote replica to compare with local replica. - :type url: str - - :return: Whether remote replica and local replica differ. - :rtype: bool - """ - target = SoledadSyncTarget( - url, self._db._get_replica_uid(), creds=self._creds, - crypto=self._crypto) - info = target.get_sync_info(self._db._get_replica_uid()) - # compare source generation with target's last known source generation - if self._db._get_generation() != info[4]: - signal(SOLEDAD_NEW_DATA_TO_SYNC, self._uuid) - return True - return False - - @property - def syncing(self): - """ - Property, True if the syncer is syncing. - """ - return self._db.syncing - - def _set_token(self, token): - """ - Set the authentication token for remote database access. - - Build the credentials dictionary with the following format: - - self._{ - 'token': { - 'uuid': '<uuid>' - 'token': '<token>' - } - - :param token: The authentication token. - :type token: str - """ - self._creds = { - 'token': { - 'uuid': self._uuid, - 'token': token, - } - } - - def _get_token(self): - """ - Return current token from credentials dictionary. - """ - return self._creds['token']['token'] - - token = property(_get_token, _set_token, doc='The authentication Token.') - - # - # Recovery document export and import methods - # - - def export_recovery_document(self): - """ - Export the storage secrets. - - A recovery document has the following structure: - - { - 'storage_secrets': { - '<storage_secret id>': { - 'kdf': 'scrypt', - 'kdf_salt': '<b64 repr of salt>' - 'kdf_length': <key length> - 'cipher': 'aes256', - 'length': <secret length>, - 'secret': '<encrypted storage_secret>', - }, - }, - 'kdf': 'scrypt', - 'kdf_salt': '<b64 repr of salt>', - 'kdf_length: <key length>, - '_mac_method': 'hmac', - '_mac': '<mac>' - } - - Note that multiple storage secrets might be stored in one recovery - document. This method will also calculate a MAC of a string - representation of the secrets dictionary. - - :return: The recovery document. - :rtype: dict - """ - # create salt and key for calculating MAC - salt = os.urandom(self.SALT_LENGTH) - key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) - data = { - self.STORAGE_SECRETS_KEY: self._secrets, - self.KDF_KEY: self.KDF_SCRYPT, - self.KDF_SALT_KEY: binascii.b2a_base64(salt), - self.KDF_LENGTH_KEY: len(key), - MAC_METHOD_KEY: MacMethods.HMAC, - MAC_KEY: hmac.new( - key, - json.dumps(self._secrets), - sha256).hexdigest(), - } - return data - - def import_recovery_document(self, data): - """ - Import storage secrets for symmetric encryption and uuid (if present) - from a recovery document. - - Note that this method does not store the imported data on disk. For - that, use C{self._store_secrets()}. - - :param data: The recovery document. - :type data: dict - - :return: A tuple containing the number of imported secrets and whether - there was MAC informationa available for authenticating. - :rtype: (int, bool) - """ - soledad_assert(self.STORAGE_SECRETS_KEY in data) - # check mac of the recovery document - mac = None - if MAC_KEY in data: - soledad_assert(data[MAC_KEY] is not None) - soledad_assert(MAC_METHOD_KEY in data) - soledad_assert(self.KDF_KEY in data) - soledad_assert(self.KDF_SALT_KEY in data) - soledad_assert(self.KDF_LENGTH_KEY in data) - if data[MAC_METHOD_KEY] == MacMethods.HMAC: - key = scrypt.hash( - self._passphrase_as_string(), - binascii.a2b_base64(data[self.KDF_SALT_KEY]), - buflen=32) - mac = hmac.new( - key, - json.dumps(data[self.STORAGE_SECRETS_KEY]), - sha256).hexdigest() - else: - raise UnknownMacMethod('Unknown MAC method: %s.' % - data[MAC_METHOD_KEY]) - if mac != data[MAC_KEY]: - raise WrongMac('Could not authenticate recovery document\'s ' - 'contents.') - # include secrets in the secret pool. - secrets = 0 - for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items(): - if secret_id not in self._secrets: - secrets += 1 - self._secrets[secret_id] = secret_data - return secrets, mac - - # - # Setters/getters - # - - def _get_uuid(self): - return self._uuid - - uuid = property(_get_uuid, doc='The user uuid.') - - def _get_secret_id(self): - return self._secret_id - - secret_id = property( - _get_secret_id, - doc='The active secret id.') - - def _get_secrets_path(self): - return self._secrets_path - - secrets_path = property( - _get_secrets_path, - doc='The path for the file containing the encrypted symmetric secret.') - - def _get_local_db_path(self): - return self._local_db_path - - local_db_path = property( - _get_local_db_path, - doc='The path for the local database replica.') - - def _get_server_url(self): - return self._server_url - - server_url = property( - _get_server_url, - doc='The URL of the Soledad server.') - - storage_secret = property( - _get_storage_secret, - doc='The secret used for symmetric encryption.') - - def _get_passphrase(self): - return self._passphrase - - passphrase = property( - _get_passphrase, - doc='The passphrase for locking and unlocking encryption secrets for ' - 'local and remote storage.') - - def _passphrase_as_string(self): - return self._passphrase.encode('utf-8') - - -# ---------------------------------------------------------------------------- -# Monkey patching u1db to be able to provide a custom SSL cert -# ---------------------------------------------------------------------------- - -# We need a more reasonable timeout (in seconds) -SOLEDAD_TIMEOUT = 120 - - -class VerifiedHTTPSConnection(httplib.HTTPSConnection): - """ - HTTPSConnection verifying server side certificates. - """ - # derived from httplib.py - - def connect(self): - """ - Connect to a host on a given (SSL) port. - """ - try: - source = self.source_address - sock = socket.create_connection((self.host, self.port), - SOLEDAD_TIMEOUT, source) - except AttributeError: - # source_address was introduced in 2.7 - sock = socket.create_connection((self.host, self.port), - SOLEDAD_TIMEOUT) - if self._tunnel_host: - self.sock = sock - self._tunnel() - - highest_supported = ssl.PROTOCOL_SSLv23 - - try: - # needs python 2.7.9+ - # negotiate the best available version, - # but explicitely disabled bad ones. - ctx = ssl.SSLContext(highest_supported) - ctx.options |= ssl.OP_NO_SSLv2 - ctx.options |= ssl.OP_NO_SSLv3 - - ctx.load_verify_locations(cafile=SOLEDAD_CERT) - ctx.verify_mode = ssl.CERT_REQUIRED - self.sock = ctx.wrap_socket(sock) - - except AttributeError: - self.sock = ssl.wrap_socket( - sock, ca_certs=SOLEDAD_CERT, cert_reqs=ssl.CERT_REQUIRED, - ssl_version=highest_supported) - - match_hostname(self.sock.getpeercert(), self.host) - - -old__VerifiedHTTPSConnection = http_client._VerifiedHTTPSConnection -http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection - - -__all__ = ['soledad_assert', 'Soledad'] +from leap.soledad.client.api import Soledad +from leap.soledad.common import soledad_assert from ._version import get_versions __version__ = get_versions()['version'] del get_versions + +__all__ = ['soledad_assert', 'Soledad', '__version__'] diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py new file mode 100644 index 00000000..5b882bbe --- /dev/null +++ b/client/src/leap/soledad/client/adbapi.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +# adbapi.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +An asyncrhonous interface to soledad using sqlcipher backend. +It uses twisted.enterprise.adbapi. +""" +import re +import os +import sys +import logging + +from functools import partial +from threading import BoundedSemaphore + +from twisted.enterprise import adbapi +from twisted.python import log +from zope.proxy import ProxyBase, setProxiedObject +from pysqlcipher.dbapi2 import OperationalError + +from leap.soledad.client import sqlcipher as soledad_sqlcipher +from leap.soledad.client.pragmas import set_init_pragmas + + +logger = logging.getLogger(name=__name__) + + +DEBUG_SQL = os.environ.get("LEAP_DEBUG_SQL") +if DEBUG_SQL: + log.startLogging(sys.stdout) + +""" +How long the SQLCipher connection should wait for the lock to go away until +raising an exception. +""" +SQLCIPHER_CONNECTION_TIMEOUT = 10 + +""" +How many times a SQLCipher query should be retried in case of timeout. +""" +SQLCIPHER_MAX_RETRIES = 10 + + +def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): + """ + Return a connection pool. + + :param opts: + Options for the SQLCipher connection. + :type opts: SQLCipherOptions + :param openfun: + Callback invoked after every connect() on the underlying DB-API + object. + :type openfun: callable + :param driver: + The connection driver. + :type driver: str + + :return: A U1DB connection pool. + :rtype: U1DBConnectionPool + """ + if openfun is None and driver == "pysqlcipher": + openfun = partial(set_init_pragmas, opts=opts) + return U1DBConnectionPool( + "%s.dbapi2" % driver, database=opts.path, + check_same_thread=False, cp_openfun=openfun, + timeout=SQLCIPHER_CONNECTION_TIMEOUT) + + +class U1DBConnection(adbapi.Connection): + """ + A wrapper for a U1DB connection instance. + """ + + u1db_wrapper = soledad_sqlcipher.SoledadSQLCipherWrapper + """ + The U1DB wrapper to use. + """ + + def __init__(self, pool, init_u1db=False): + """ + :param pool: The pool of connections to that owns this connection. + :type pool: adbapi.ConnectionPool + :param init_u1db: Wether the u1db database should be initialized. + :type init_u1db: bool + """ + self.init_u1db = init_u1db + adbapi.Connection.__init__(self, pool) + + def reconnect(self): + """ + Reconnect to the U1DB database. + """ + if self._connection is not None: + self._pool.disconnect(self._connection) + self._connection = self._pool.connect() + + if self.init_u1db: + self._u1db = self.u1db_wrapper(self._connection) + + def __getattr__(self, name): + """ + Route the requested attribute either to the U1DB wrapper or to the + connection. + + :param name: The name of the attribute. + :type name: str + """ + if name.startswith('u1db_'): + attr = re.sub('^u1db_', '', name) + return getattr(self._u1db, attr) + else: + return getattr(self._connection, name) + + +class U1DBTransaction(adbapi.Transaction): + """ + A wrapper for a U1DB 'cursor' object. + """ + + def __getattr__(self, name): + """ + Route the requested attribute either to the U1DB wrapper of the + connection or to the actual connection cursor. + + :param name: The name of the attribute. + :type name: str + """ + if name.startswith('u1db_'): + attr = re.sub('^u1db_', '', name) + return getattr(self._connection._u1db, attr) + else: + return getattr(self._cursor, name) + + +class U1DBConnectionPool(adbapi.ConnectionPool): + """ + Represent a pool of connections to an U1DB database. + """ + + connectionFactory = U1DBConnection + transactionFactory = U1DBTransaction + + def __init__(self, *args, **kwargs): + """ + Initialize the connection pool. + """ + adbapi.ConnectionPool.__init__(self, *args, **kwargs) + # all u1db connections, hashed by thread-id + self._u1dbconnections = {} + + # The replica uid, primed by the connections on init. + self.replica_uid = ProxyBase(None) + + conn = self.connectionFactory(self, init_u1db=True) + replica_uid = conn._u1db._real_replica_uid + setProxiedObject(self.replica_uid, replica_uid) + + def runU1DBQuery(self, meth, *args, **kw): + """ + Execute a U1DB query in a thread, using a pooled connection. + + Concurrent threads trying to update the same database may timeout + because of other threads holding the database lock. Because of this, + we will retry SQLCIPHER_MAX_RETRIES times and fail after that. + + :param meth: The U1DB wrapper method name. + :type meth: str + + :return: a Deferred which will fire the return value of + 'self._runU1DBQuery(Transaction(...), *args, **kw)', or a Failure. + :rtype: twisted.internet.defer.Deferred + """ + meth = "u1db_%s" % meth + semaphore = BoundedSemaphore(SQLCIPHER_MAX_RETRIES - 1) + + def _run_interaction(): + return self.runInteraction( + self._runU1DBQuery, meth, *args, **kw) + + def _errback(failure): + failure.trap(OperationalError) + if failure.getErrorMessage() == "database is locked": + should_retry = semaphore.acquire(False) + if should_retry: + logger.warning( + "Database operation timed out while waiting for " + "lock, trying again...") + return _run_interaction() + return failure + + d = _run_interaction() + d.addErrback(_errback) + return d + + def _runU1DBQuery(self, trans, meth, *args, **kw): + """ + Execute a U1DB query. + + :param trans: An U1DB transaction. + :type trans: adbapi.Transaction + :param meth: the U1DB wrapper method name. + :type meth: str + """ + meth = getattr(trans, meth) + return meth(*args, **kw) + + def _runInteraction(self, interaction, *args, **kw): + """ + Interact with the database and return the result. + + :param interaction: + A callable object whose first argument is an + L{adbapi.Transaction}. + :type interaction: callable + :return: a Deferred which will fire the return value of + 'interaction(Transaction(...), *args, **kw)', or a Failure. + :rtype: twisted.internet.defer.Deferred + """ + tid = self.threadID() + u1db = self._u1dbconnections.get(tid) + conn = self.connectionFactory(self, init_u1db=not bool(u1db)) + + if self.replica_uid is None: + replica_uid = conn._u1db._real_replica_uid + setProxiedObject(self.replica_uid, replica_uid) + + if u1db is None: + self._u1dbconnections[tid] = conn._u1db + else: + conn._u1db = u1db + + trans = self.transactionFactory(self, conn) + try: + result = interaction(trans, *args, **kw) + trans.close() + conn.commit() + return result + except: + excType, excValue, excTraceback = sys.exc_info() + try: + conn.rollback() + except: + log.err(None, "Rollback failed") + raise excType, excValue, excTraceback + + def finalClose(self): + """ + A final close, only called by the shutdown trigger. + """ + self.shutdownID = None + self.threadpool.stop() + self.running = False + for conn in self.connections.values(): + self._close(conn) + for u1db in self._u1dbconnections.values(): + self._close(u1db) + self.connections.clear() diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py new file mode 100644 index 00000000..76d6acc3 --- /dev/null +++ b/client/src/leap/soledad/client/api.py @@ -0,0 +1,882 @@ +# -*- coding: utf-8 -*- +# api.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Soledad - Synchronization Of Locally Encrypted Data Among Devices. + +This module holds the public api for Soledad. + +Soledad is the part of LEAP that manages storage and synchronization of +application data. It is built on top of U1DB reference Python API and +implements (1) a SQLCipher backend for local storage in the client, (2) a +SyncTarget that encrypts data before syncing, and (3) a CouchDB backend for +remote storage in the server side. +""" +import binascii +import errno +import httplib +import logging +import os +import socket +import ssl +import urlparse + +try: + import cchardet as chardet +except ImportError: + import chardet + +from StringIO import StringIO +from u1db.remote import http_client +from u1db.remote.ssl_match_hostname import match_hostname +from zope.interface import implements + +from leap.common.config import get_path_prefix + +from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common import soledad_assert +from leap.soledad.common import soledad_assert_type + +from leap.soledad.client import adbapi +from leap.soledad.client import events as soledad_events +from leap.soledad.client import interfaces as soledad_interfaces +from leap.soledad.client.crypto import SoledadCrypto +from leap.soledad.client.secrets import SoledadSecrets +from leap.soledad.client.shared_db import SoledadSharedDatabase +from leap.soledad.client.sqlcipher import SQLCipherOptions, SQLCipherU1DBSync + +logger = logging.getLogger(name=__name__) + +# +# Constants +# + +""" +Path to the certificate file used to certify the SSL connection between +Soledad client and server. +""" +SOLEDAD_CERT = None + + +class Soledad(object): + """ + Soledad provides encrypted data storage and sync. + + A Soledad instance is used to store and retrieve data in a local encrypted + database and synchronize this database with Soledad server. + + This class is also responsible for bootstrapping users' account by + creating cryptographic secrets and/or storing/fetching them on Soledad + server. + + Soledad uses ``leap.common.events`` to signal events. The possible events + to be signaled are: + + SOLEDAD_CREATING_KEYS: emitted during bootstrap sequence when key + generation starts. + SOLEDAD_DONE_CREATING_KEYS: emitted during bootstrap sequence when key + generation finishes. + SOLEDAD_UPLOADING_KEYS: emitted during bootstrap sequence when soledad + starts sending keys to server. + SOLEDAD_DONE_UPLOADING_KEYS: emitted during bootstrap sequence when + soledad finishes sending keys to server. + SOLEDAD_DOWNLOADING_KEYS: emitted during bootstrap sequence when + soledad starts to retrieve keys from server. + SOLEDAD_DONE_DOWNLOADING_KEYS: emitted during bootstrap sequence when + soledad finishes downloading keys from server. + SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when + there's indeed new data to be synchronized between local database + replica and server's replica. + SOLEDAD_DONE_DATA_SYNC: emitted inside C{sync()} method when it has + finished synchronizing with remote replica. + """ + implements(soledad_interfaces.ILocalStorage, + soledad_interfaces.ISyncableStorage, + soledad_interfaces.ISecretsStorage) + + local_db_file_name = 'soledad.u1db' + secrets_file_name = "soledad.json" + default_prefix = os.path.join(get_path_prefix(), 'leap', 'soledad') + + def __init__(self, uuid, passphrase, secrets_path, local_db_path, + server_url, cert_file, shared_db=None, + auth_token=None, defer_encryption=False, syncable=True): + """ + Initialize configuration, cryptographic keys and dbs. + + :param uuid: User's uuid. + :type uuid: str + + :param passphrase: + The passphrase for locking and unlocking encryption secrets for + local and remote storage. + :type passphrase: unicode + + :param secrets_path: + Path for storing encrypted key used for symmetric encryption. + :type secrets_path: str + + :param local_db_path: Path for local encrypted storage db. + :type local_db_path: str + + :param server_url: + URL for Soledad server. This is used either to sync with the user's + remote db and to interact with the shared recovery database. + :type server_url: str + + :param cert_file: + Path to the certificate of the ca used to validate the SSL + certificate used by the remote soledad server. + :type cert_file: str + + :param shared_db: + The shared database. + :type shared_db: HTTPDatabase + + :param auth_token: + Authorization token for accessing remote databases. + :type auth_token: str + + :param defer_encryption: + Whether to defer encryption/decryption of documents, or do it + inline while syncing. + :type defer_encryption: bool + + :param syncable: + If set to ``False``, this database will not attempt to synchronize + with remote replicas (default is ``True``) + :type syncable: bool + + :raise BootstrapSequenceError: + Raised when the secret initialization sequence (i.e. retrieval + from server or generation and storage on server) has failed for + some reason. + """ + # store config params + self._uuid = uuid + self._passphrase = passphrase + self._local_db_path = local_db_path + self._server_url = server_url + self._defer_encryption = defer_encryption + self._secrets_path = None + + self.shared_db = shared_db + + # configure SSL certificate + global SOLEDAD_CERT + SOLEDAD_CERT = cert_file + + # init crypto variables + self._set_token(auth_token) + self._crypto = SoledadCrypto(self) + + self._init_config_with_defaults() + self._init_working_dirs() + + self._secrets_path = secrets_path + + # Initialize shared recovery database + self.init_shared_db(server_url, uuid, self._creds, syncable=syncable) + + # The following can raise BootstrapSequenceError, that will be + # propagated upwards. + self._init_secrets() + self._init_u1db_sqlcipher_backend() + + if syncable: + self._init_u1db_syncer() + + # + # initialization/destruction methods + # + def _init_config_with_defaults(self): + """ + Initialize configuration using default values for missing params. + """ + soledad_assert_type(self._passphrase, unicode) + initialize = lambda attr, val: getattr( + self, attr, None) is None and setattr(self, attr, val) + + initialize("_secrets_path", os.path.join( + self.default_prefix, self.secrets_file_name)) + initialize("_local_db_path", os.path.join( + self.default_prefix, self.local_db_file_name)) + # initialize server_url + soledad_assert(self._server_url is not None, + 'Missing URL for Soledad server.') + + def _init_working_dirs(self): + """ + Create work directories. + + :raise OSError: in case file exists and is not a dir. + """ + paths = map(lambda x: os.path.dirname(x), [ + self._local_db_path, self._secrets_path]) + for path in paths: + create_path_if_not_exists(path) + + def _init_secrets(self): + """ + Initialize Soledad secrets. + """ + self._secrets = SoledadSecrets( + self.uuid, self._passphrase, self._secrets_path, + self.shared_db, self._crypto) + self._secrets.bootstrap() + + def _init_u1db_sqlcipher_backend(self): + """ + Initialize the U1DB SQLCipher database for local storage. + + Instantiates a modified twisted adbapi that will maintain a threadpool + with a u1db-sqclipher connection for each thread, and will return + deferreds for each u1db query. + + Currently, Soledad uses the default SQLCipher cipher, i.e. + 'aes-256-cbc'. We use scrypt to derive a 256-bit encryption key, + and internally the SQLCipherDatabase initialization uses the 'raw + PRAGMA key' format to handle the key to SQLCipher. + """ + tohex = binascii.b2a_hex + # sqlcipher only accepts the hex version + key = tohex(self._secrets.get_local_storage_key()) + sync_db_key = tohex(self._secrets.get_sync_db_key()) + + opts = SQLCipherOptions( + self._local_db_path, key, + is_raw_key=True, create=True, + defer_encryption=self._defer_encryption, + sync_db_key=sync_db_key, + ) + self._sqlcipher_opts = opts + self._dbpool = adbapi.getConnectionPool(opts) + + def _init_u1db_syncer(self): + """ + Initialize the U1DB synchronizer. + """ + replica_uid = self._dbpool.replica_uid + self._dbsyncer = SQLCipherU1DBSync( + self._sqlcipher_opts, self._crypto, replica_uid, + SOLEDAD_CERT, + defer_encryption=self._defer_encryption) + + # + # Closing methods + # + + def close(self): + """ + Close underlying U1DB database. + """ + logger.debug("Closing soledad") + self._dbpool.close() + if getattr(self, '_dbsyncer', None): + self._dbsyncer.close() + + # + # ILocalStorage + # + + def _defer(self, meth, *args, **kw): + """ + Defer a method to be run on a U1DB connection pool. + + :param meth: A method to defer to the U1DB connection pool. + :type meth: callable + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ + return self._dbpool.runU1DBQuery(meth, *args, **kw) + + def put_doc(self, doc): + """ + Update a document. + + If the document currently has conflicts, put will fail. + If the database specifies a maximum document size and the document + exceeds it, put will fail and raise a DocumentTooBig exception. + + ============================== WARNING ============================== + This method converts the document's contents to unicode in-place. This + means that after calling `put_doc(doc)`, the contents of the + document, i.e. `doc.content`, might be different from before the + call. + ============================== WARNING ============================== + + :param doc: A document with new content. + :type doc: leap.soledad.common.document.SoledadDocument + :return: A deferred whose callback will be invoked with the new + revision identifier for the document. The document object will + also be updated. + :rtype: twisted.internet.defer.Deferred + """ + doc.content = _convert_to_unicode(doc.content) + return self._defer("put_doc", doc) + + def delete_doc(self, doc): + """ + Mark a document as deleted. + + Will abort if the current revision doesn't match doc.rev. + This will also set doc.content to None. + + :param doc: A document to be deleted. + :type doc: leap.soledad.common.document.SoledadDocument + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer("delete_doc", doc) + + def get_doc(self, doc_id, include_deleted=False): + """ + Get the JSON string for the given document. + + :param doc_id: The unique document identifier + :type doc_id: str + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise asking for a deleted + document will return None. + :type include_deleted: bool + :return: A deferred whose callback will be invoked with a document + object. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer( + "get_doc", doc_id, include_deleted=include_deleted) + + def get_docs( + self, doc_ids, check_for_conflicts=True, include_deleted=False): + """ + Get the JSON content for many documents. + + :param doc_ids: A list of document identifiers. + :type doc_ids: list + :param check_for_conflicts: If set to False, then the conflict check + will be skipped, and 'None' will be returned instead of True/False. + :type check_for_conflicts: bool + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted documents will not + be included in the results. + :type include_deleted: bool + :return: A deferred whose callback will be invoked with an iterable + giving the document object for each document id in matching + doc_ids order. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer( + "get_docs", doc_ids, check_for_conflicts=check_for_conflicts, + include_deleted=include_deleted) + + def get_all_docs(self, include_deleted=False): + """ + Get the JSON content for all documents in the database. + + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted documents will not + be included in the results. + :type include_deleted: bool + + :return: A deferred which, when fired, will pass the a tuple + containing (generation, [Document]) to the callback, with the + current generation of the database, followed by a list of all the + documents in the database. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer("get_all_docs", include_deleted) + + def create_doc(self, content, doc_id=None): + """ + Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param content: A Python dictionary. + :type content: dict + :param doc_id: An optional identifier specifying the document id. + :type doc_id: str + :return: A deferred whose callback will be invoked with a document. + :rtype: twisted.internet.defer.Deferred + """ + # TODO we probably should pass an optional "encoding" parameter to + # create_doc (and probably to put_doc too). There are cases (mail + # payloads for example) in which we already have the encoding in the + # headers, so we don't need to guess it. + return self._defer( + "create_doc", _convert_to_unicode(content), doc_id=doc_id) + + def create_doc_from_json(self, json, doc_id=None): + """ + Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param json: The JSON document string + :type json: dict + :param doc_id: An optional identifier specifying the document id. + :type doc_id: str + :return: A deferred whose callback will be invoked with a document. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer("create_doc_from_json", json, doc_id=doc_id) + + def create_index(self, index_name, *index_expressions): + """ + Create a named index, which can then be queried for future lookups. + + Creating an index which already exists is not an error, and is cheap. + Creating an index which does not match the index_expressions of the + existing index is an error. + Creating an index will block until the expressions have been evaluated + and the index generated. + + :param index_name: A unique name which can be used as a key prefix + :type index_name: str + :param index_expressions: index expressions defining the index + information. + + Examples: + + "fieldname", or "fieldname.subfieldname" to index alphabetically + sorted on the contents of a field. + + "number(fieldname, width)", "lower(fieldname)" + :type index_expresions: list of str + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer("create_index", index_name, *index_expressions) + + def delete_index(self, index_name): + """ + Remove a named index. + + :param index_name: The name of the index we are removing + :type index_name: str + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer("delete_index", index_name) + + def list_indexes(self): + """ + List the definitions of all known indexes. + + :return: A deferred whose callback will be invoked with a list of + [('index-name', ['field', 'field2'])] definitions. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer("list_indexes") + + def get_from_index(self, index_name, *key_values): + """ + Return documents that match the keys supplied. + + You must supply exactly the same number of values as have been defined + in the index. It is possible to do a prefix match by using '*' to + indicate a wildcard match. You can only supply '*' to trailing entries, + (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) + It is also possible to append a '*' to the last supplied value (eg + 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: list + :return: A deferred whose callback will be invoked with a list of + [Document]. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer("get_from_index", index_name, *key_values) + + def get_count_from_index(self, index_name, *key_values): + """ + Return the count for a given combination of index_name + and key values. + + Extension method made from similar methods in u1db version 13.09 + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: A deferred whose callback will be invoked with the count. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer("get_count_from_index", index_name, *key_values) + + def get_range_from_index(self, index_name, start_value, end_value): + """ + Return documents that fall within the specified range. + + Both ends of the range are inclusive. For both start_value and + end_value, one must supply exactly the same number of values as have + been defined in the index, or pass None. In case of a single column + index, a string is accepted as an alternative for a tuple with a single + value. It is possible to do a prefix match by using '*' to indicate + a wildcard match. You can only supply '*' to trailing entries, (eg + 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also + possible to append a '*' to the last supplied value (eg 'val*', '*', + '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param start_values: tuples of values that define the lower bound of + the range. eg, if you have an index with 3 fields then you would + have: (val1, val2, val3) + :type start_values: tuple + :param end_values: tuples of values that define the upper bound of the + range. eg, if you have an index with 3 fields then you would have: + (val1, val2, val3) + :type end_values: tuple + :return: A deferred whose callback will be invoked with a list of + [Document]. + :rtype: twisted.internet.defer.Deferred + """ + + return self._defer( + "get_range_from_index", index_name, start_value, end_value) + + def get_index_keys(self, index_name): + """ + Return all keys under which documents are indexed in this index. + + :param index_name: The index to query + :type index_name: str + :return: A deferred whose callback will be invoked with a list of + tuples of indexed keys. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer("get_index_keys", index_name) + + def get_doc_conflicts(self, doc_id): + """ + Get the list of conflicts for the given document. + + The order of the conflicts is such that the first entry is the value + that would be returned by "get_doc". + + :param doc_id: The unique document identifier + :type doc_id: str + :return: A deferred whose callback will be invoked with a list of the + Document entries that are conflicted. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer("get_doc_conflicts", doc_id) + + def resolve_doc(self, doc, conflicted_doc_revs): + """ + Mark a document as no longer conflicted. + + We take the list of revisions that the client knows about that it is + superseding. This may be a different list from the actual current + conflicts, in which case only those are removed as conflicted. This + may fail if the conflict list is significantly different from the + supplied information. (sync could have happened in the background from + the time you GET_DOC_CONFLICTS until the point where you RESOLVE) + + :param doc: A Document with the new content to be inserted. + :type doc: SoledadDocument + :param conflicted_doc_revs: A list of revisions that the new content + supersedes. + :type conflicted_doc_revs: list(str) + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer("resolve_doc", doc, conflicted_doc_revs) + + @property + def local_db_path(self): + return self._local_db_path + + @property + def uuid(self): + return self._uuid + + # + # ISyncableStorage + # + + def sync(self, defer_decryption=True): + """ + Synchronize documents with the server replica. + + :param defer_decryption: + Whether to defer decryption of documents, or do it inline while + syncing. + :type defer_decryption: bool + + :return: A deferred whose callback will be invoked with the local + generation before the synchronization was performed. + :rtype: twisted.internet.defer.Deferred + """ + + # ----------------------------------------------------------------- + # TODO this needs work. + # Should review/write tests to check that this: + + # (1) Defer to the syncer pool -- DONE (on dbsyncer) + # (2) Return the deferred + # (3) Add the callback for signaling the event (executed on reactor + # thread) + # (4) Check that the deferred is called with the local gen. + + # ----------------------------------------------------------------- + + sync_url = urlparse.urljoin(self._server_url, 'user-%s' % self.uuid) + d = self._dbsyncer.sync( + sync_url, + creds=self._creds, + defer_decryption=defer_decryption) + + def _sync_callback(local_gen): + soledad_events.emit( + soledad_events.SOLEDAD_DONE_DATA_SYNC, self.uuid) + return local_gen + + def _sync_errback(failure): + s = StringIO() + failure.printDetailedTraceback(file=s) + msg = "Soledad exception when syncing!\n" + s.getvalue() + logger.error(msg) + return failure + + d.addCallbacks(_sync_callback, _sync_errback) + return d + + @property + def syncing(self): + """ + Return wether Soledad is currently synchronizing with the server. + + :return: Wether Soledad is currently synchronizing with the server. + :rtype: bool + """ + return self._dbsyncer.syncing + + def _set_token(self, token): + """ + Set the authentication token for remote database access. + + Internally, this builds the credentials dictionary with the following + format: + + { + 'token': { + 'uuid': '<uuid>' + 'token': '<token>' + } + } + + :param token: The authentication token. + :type token: str + """ + self._creds = { + 'token': { + 'uuid': self.uuid, + 'token': token, + } + } + + def _get_token(self): + """ + Return current token from credentials dictionary. + """ + return self._creds['token']['token'] + + token = property(_get_token, _set_token, doc='The authentication Token.') + + # + # ISecretsStorage + # + + def init_shared_db(self, server_url, uuid, creds, syncable=True): + """ + Initialize the shared database. + + :param server_url: URL of the remote database. + :type server_url: str + :param uuid: The user's unique id. + :type uuid: str + :param creds: A tuple containing the authentication method and + credentials. + :type creds: tuple + :param syncable: + If syncable is False, the database will not attempt to sync against + a remote replica. + :type syncable: bool + """ + # only case this is False is for testing purposes + if self.shared_db is None: + shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME) + self.shared_db = SoledadSharedDatabase.open_database( + shared_db_url, + uuid, + creds=creds, + syncable=syncable) + + @property + def storage_secret(self): + """ + Return the secret used for local storage encryption. + + :return: The secret used for local storage encryption. + :rtype: str + """ + return self._secrets.storage_secret + + @property + def remote_storage_secret(self): + """ + Return the secret used for encryption of remotely stored data. + + :return: The secret used for remote storage encryption. + :rtype: str + """ + return self._secrets.remote_storage_secret + + @property + def secrets(self): + """ + Return the secrets object. + + :return: The secrets object. + :rtype: SoledadSecrets + """ + return self._secrets + + def change_passphrase(self, new_passphrase): + """ + Change the passphrase that encrypts the storage secret. + + :param new_passphrase: The new passphrase. + :type new_passphrase: unicode + + :raise NoStorageSecret: Raised if there's no storage secret available. + """ + self._secrets.change_passphrase(new_passphrase) + + # + # Raw SQLCIPHER Queries + # + + def raw_sqlcipher_query(self, *args, **kw): + """ + Run a raw sqlcipher query in the local database. + """ + return self._dbpool.runQuery(*args, **kw) + + +def _convert_to_unicode(content): + """ + Convert content to unicode (or all the strings in content). + + NOTE: Even though this method supports any type, it will + currently ignore contents of lists, tuple or any other + iterable than dict. We don't need support for these at the + moment + + :param content: content to convert + :type content: object + + :rtype: object + """ + # Chardet doesn't guess very well with some smallish payloads. + # This parameter might need some empirical tweaking. + CUTOFF_CONFIDENCE = 0.90 + + if isinstance(content, unicode): + return content + elif isinstance(content, str): + encoding = "utf-8" + result = chardet.detect(content) + if result["confidence"] > CUTOFF_CONFIDENCE: + encoding = result["encoding"] + try: + content = content.decode(encoding) + except UnicodeError as e: + logger.error("Unicode error: {0!r}. Using 'replace'".format(e)) + content = content.decode(encoding, 'replace') + return content + else: + if isinstance(content, dict): + for key in content.keys(): + content[key] = _convert_to_unicode(content[key]) + return content + + +def create_path_if_not_exists(path): + try: + if not os.path.isdir(path): + logger.info('Creating directory: %s.' % path) + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +# ---------------------------------------------------------------------------- +# Monkey patching u1db to be able to provide a custom SSL cert +# ---------------------------------------------------------------------------- + +# We need a more reasonable timeout (in seconds) +SOLEDAD_TIMEOUT = 120 + + +class VerifiedHTTPSConnection(httplib.HTTPSConnection): + """ + HTTPSConnection verifying server side certificates. + """ + # derived from httplib.py + + def connect(self): + """ + Connect to a host on a given (SSL) port. + """ + try: + source = self.source_address + sock = socket.create_connection((self.host, self.port), + SOLEDAD_TIMEOUT, source) + except AttributeError: + # source_address was introduced in 2.7 + sock = socket.create_connection((self.host, self.port), + SOLEDAD_TIMEOUT) + if self._tunnel_host: + self.sock = sock + self._tunnel() + + self.sock = ssl.wrap_socket(sock, + ca_certs=SOLEDAD_CERT, + cert_reqs=ssl.CERT_REQUIRED) + match_hostname(self.sock.getpeercert(), self.host) + + +old__VerifiedHTTPSConnection = http_client._VerifiedHTTPSConnection +http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection diff --git a/client/src/leap/soledad/client/auth.py b/client/src/leap/soledad/client/auth.py index 72ab0008..6dfabeb4 100644 --- a/client/src/leap/soledad/client/auth.py +++ b/client/src/leap/soledad/client/auth.py @@ -14,15 +14,13 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. - - """ Methods for token-based authentication. These methods have to be included in all classes that extend HTTPClient so they can do token-based auth requests to the Soledad server. """ - +import base64 from u1db import errors @@ -49,7 +47,7 @@ class TokenBasedAuth(object): Return an authorization header to be included in the HTTP request, in the form: - [('Authorization', 'Token <base64 encoded creds')] + [('Authorization', 'Token <(base64 encoded) uuid:token>')] :param method: The HTTP method. :type method: str @@ -64,7 +62,8 @@ class TokenBasedAuth(object): if 'token' in self._creds: uuid, token = self._creds['token'] auth = '%s:%s' % (uuid, token) - return [('Authorization', 'Token %s' % auth.encode('base64')[:-1])] + b64_token = base64.b64encode(auth) + return [('Authorization', 'Token %s' % b64_token)] else: raise errors.UnknownAuthMethod( 'Wrong credentials: %s' % self._creds) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 7133f804..bdbaa8e0 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -23,31 +23,14 @@ import hmac import hashlib import json import logging -import multiprocessing -import threading from pycryptopp.cipher.aes import AES from pycryptopp.cipher.xsalsa20 import XSalsa20 -from zope.proxy import sameProxiedObjects from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type -from leap.soledad.common.document import SoledadDocument - - -from leap.soledad.common.crypto import ( - EncryptionSchemes, - UnknownEncryptionScheme, - MacMethods, - UnknownMacMethod, - WrongMac, - ENC_JSON_KEY, - ENC_SCHEME_KEY, - ENC_METHOD_KEY, - ENC_IV_KEY, - MAC_KEY, - MAC_METHOD_KEY, -) +from leap.soledad.common import crypto + logger = logging.getLogger(__name__) @@ -55,37 +38,23 @@ logger = logging.getLogger(__name__) MAC_KEY_LENGTH = 64 -class EncryptionMethods(object): - """ - Representation of encryption methods that can be used. +def _assert_known_encryption_method(method): """ + Assert that we can encrypt/decrypt the given C{method} - AES_256_CTR = 'aes-256-ctr' - XSALSA20 = 'xsalsa20' - -# -# Exceptions -# - - -class DocumentNotEncrypted(Exception): - """ - Raised for failures in document encryption. - """ - pass - - -class UnknownEncryptionMethod(Exception): - """ - Raised when trying to encrypt/decrypt with unknown method. - """ - pass - + :param method: The encryption method to assert. + :type method: str -class NoSymmetricSecret(Exception): - """ - Raised when trying to get a hashed passphrase. + :raise UnknownEncryptionMethodError: Raised when C{method} is unknown. """ + valid_methods = [ + crypto.EncryptionMethods.AES_256_CTR, + crypto.EncryptionMethods.XSALSA20, + ] + try: + soledad_assert(method in valid_methods) + except AssertionError: + raise crypto.UnknownEncryptionMethodError def encrypt_sym(data, key, method): @@ -104,25 +73,26 @@ def encrypt_sym(data, key, method): :return: A tuple with the initial value and the encrypted data. :rtype: (long, str) + + :raise AssertionError: Raised if C{method} is unknown. """ soledad_assert_type(key, str) - soledad_assert( len(key) == 32, # 32 x 8 = 256 bits. 'Wrong key size: %s bits (must be 256 bits long).' % (len(key) * 8)) + _assert_known_encryption_method(method) + iv = None # AES-256 in CTR mode - if method == EncryptionMethods.AES_256_CTR: + if method == crypto.EncryptionMethods.AES_256_CTR: iv = os.urandom(16) ciphertext = AES(key=key, iv=iv).process(data) # XSalsa20 - elif method == EncryptionMethods.XSALSA20: + elif method == crypto.EncryptionMethods.XSALSA20: iv = os.urandom(24) ciphertext = XSalsa20(key=key, iv=iv).process(data) - else: - # raise if method is unknown - raise UnknownEncryptionMethod('Unkwnown method: %s' % method) + return binascii.b2a_base64(iv), ciphertext @@ -143,6 +113,8 @@ def decrypt_sym(data, key, method, **kwargs): :return: The decrypted data. :rtype: str + + :raise UnknownEncryptionMethodError: Raised when C{method} is unknown. """ soledad_assert_type(key, str) # assert params @@ -152,17 +124,15 @@ def decrypt_sym(data, key, method, **kwargs): soledad_assert( 'iv' in kwargs, '%s needs an initial value.' % method) + _assert_known_encryption_method(method) # AES-256 in CTR mode - if method == EncryptionMethods.AES_256_CTR: + if method == crypto.EncryptionMethods.AES_256_CTR: return AES( key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data) - elif method == EncryptionMethods.XSALSA20: + elif method == crypto.EncryptionMethods.XSALSA20: return XSalsa20( key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data) - # raise if method is unknown - raise UnknownEncryptionMethod('Unkwnown method: %s' % method) - def doc_mac_key(doc_id, secret): """ @@ -176,17 +146,13 @@ def doc_mac_key(doc_id, secret): :param doc_id: The id of the document. :type doc_id: str - :param secret: soledad secret storage - :type secret: Soledad.storage_secret + :param secret: The Soledad storage secret + :type secret: str :return: The key. :rtype: str - - :raise NoSymmetricSecret: if no symmetric secret was supplied. """ - if secret is None: - raise NoSymmetricSecret() - + soledad_assert(secret is not None) return hmac.new( secret[:MAC_KEY_LENGTH], doc_id, @@ -208,11 +174,11 @@ class SoledadCrypto(object): self._soledad = soledad def encrypt_sym(self, data, key, - method=EncryptionMethods.AES_256_CTR): + method=crypto.EncryptionMethods.AES_256_CTR): return encrypt_sym(data, key, method) def decrypt_sym(self, data, key, - method=EncryptionMethods.AES_256_CTR, **kwargs): + method=crypto.EncryptionMethods.AES_256_CTR, **kwargs): return decrypt_sym(data, key, method, **kwargs) def doc_mac_key(self, doc_id, secret): @@ -224,7 +190,7 @@ class SoledadCrypto(object): The password is derived using HMAC having sha256 as underlying hash function. The key used for HMAC are the first - C{soledad.REMOTE_STORAGE_SECRET_KENGTH} bytes of Soledad's storage + C{soledad.REMOTE_STORAGE_SECRET_LENGTH} bytes of Soledad's storage secret stripped from the first MAC_KEY_LENGTH characters. The HMAC message is C{doc_id}. @@ -234,15 +200,10 @@ class SoledadCrypto(object): :return: The passphrase. :rtype: str - - :raise NoSymmetricSecret: if no symmetric secret was supplied. """ - if self.secret is None: - raise NoSymmetricSecret() + soledad_assert(self.secret is not None) return hmac.new( - self.secret[ - MAC_KEY_LENGTH: - self._soledad.REMOTE_STORAGE_SECRET_LENGTH], + self.secret[MAC_KEY_LENGTH:], doc_id, hashlib.sha256).digest() @@ -251,17 +212,18 @@ class SoledadCrypto(object): # def _get_secret(self): - return self._soledad.storage_secret + return self._soledad.secrets.remote_storage_secret secret = property( _get_secret, doc='The secret used for symmetric encryption') + # # Crypto utilities for a SoledadDocument. # - -def mac_doc(doc_id, doc_rev, ciphertext, mac_method, secret): +def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv, + mac_method, secret): """ Calculate a MAC for C{doc} using C{ciphertext}. @@ -277,21 +239,38 @@ def mac_doc(doc_id, doc_rev, ciphertext, mac_method, secret): :type doc_rev: str :param ciphertext: The content of the document. :type ciphertext: str + :param enc_scheme: The encryption scheme. + :type enc_scheme: str + :param enc_method: The encryption method. + :type enc_method: str + :param enc_iv: The encryption initialization vector. + :type enc_iv: str :param mac_method: The MAC method to use. :type mac_method: str - :param secret: soledad secret - :type secret: Soledad.secret_storage + :param secret: The Soledad storage secret + :type secret: str :return: The calculated MAC. :rtype: str - """ - if mac_method == MacMethods.HMAC: - return hmac.new( - doc_mac_key(doc_id, secret), - str(doc_id) + str(doc_rev) + ciphertext, - hashlib.sha256).digest() - # raise if we do not know how to handle this MAC method - raise UnknownMacMethod('Unknown MAC method: %s.' % mac_method) + + :raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown. + """ + try: + soledad_assert(mac_method == crypto.MacMethods.HMAC) + except AssertionError: + raise crypto.UnknownMacMethodError + template = "{doc_id}{doc_rev}{ciphertext}{enc_scheme}{enc_method}{enc_iv}" + content = template.format( + doc_id=doc_id, + doc_rev=doc_rev, + ciphertext=ciphertext, + enc_scheme=enc_scheme, + enc_method=enc_method, + enc_iv=enc_iv) + return hmac.new( + doc_mac_key(doc_id, secret), + content, + hashlib.sha256).digest() def encrypt_doc(crypto, doc): @@ -319,12 +298,12 @@ def encrypt_docstr(docstr, doc_id, doc_rev, key, secret): string representing the following: { - ENC_JSON_KEY: '<encrypted doc JSON string>', - ENC_SCHEME_KEY: 'symkey', - ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR, - ENC_IV_KEY: '<the initial value used to encrypt>', + crypto.ENC_JSON_KEY: '<encrypted doc JSON string>', + crypto.ENC_SCHEME_KEY: 'symkey', + crypto.ENC_METHOD_KEY: crypto.EncryptionMethods.AES_256_CTR, + crypto.ENC_IV_KEY: '<the initial value used to encrypt>', MAC_KEY: '<mac>' - MAC_METHOD_KEY: 'hmac' + crypto.MAC_METHOD_KEY: 'hmac' } :param docstr: A representation of the document to be encrypted. @@ -339,30 +318,40 @@ def encrypt_docstr(docstr, doc_id, doc_rev, key, secret): :param key: The key used to encrypt ``data`` (must be 256 bits long). :type key: str - :param secret: The Soledad secret (used for MAC auth). + :param secret: The Soledad storage secret (used for MAC auth). :type secret: str :return: The JSON serialization of the dict representing the encrypted content. :rtype: str """ - # encrypt content using AES-256 CTR mode - iv, ciphertext = encrypt_sym( + enc_scheme = crypto.EncryptionSchemes.SYMKEY + enc_method = crypto.EncryptionMethods.AES_256_CTR + mac_method = crypto.MacMethods.HMAC + enc_iv, ciphertext = encrypt_sym( str(docstr), # encryption/decryption routines expect str - key, method=EncryptionMethods.AES_256_CTR) + key, method=enc_method) + mac = binascii.b2a_hex( # store the mac as hex. + mac_doc( + doc_id, + doc_rev, + ciphertext, + enc_scheme, + enc_method, + enc_iv, + mac_method, + secret)) # Return a representation for the encrypted content. In the following, we # convert binary data to hexadecimal representation so the JSON # serialization does not complain about what it tries to serialize. hex_ciphertext = binascii.b2a_hex(ciphertext) return json.dumps({ - ENC_JSON_KEY: hex_ciphertext, - ENC_SCHEME_KEY: EncryptionSchemes.SYMKEY, - ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR, - ENC_IV_KEY: iv, - MAC_KEY: binascii.b2a_hex(mac_doc( # store the mac as hex. - doc_id, doc_rev, ciphertext, - MacMethods.HMAC, secret)), - MAC_METHOD_KEY: MacMethods.HMAC, + crypto.ENC_JSON_KEY: hex_ciphertext, + crypto.ENC_SCHEME_KEY: enc_scheme, + crypto.ENC_METHOD_KEY: enc_method, + crypto.ENC_IV_KEY: enc_iv, + crypto.MAC_KEY: mac, + crypto.MAC_METHOD_KEY: mac_method, }) @@ -384,27 +373,77 @@ def decrypt_doc(crypto, doc): return decrypt_doc_dict(doc.content, doc.doc_id, doc.rev, key, secret) +def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, + enc_iv, mac_method, secret, doc_mac): + """ + Verify that C{doc_mac} is a correct MAC for the given document. + + :param doc_id: The id of the document. + :type doc_id: str + :param doc_rev: The revision of the document. + :type doc_rev: str + :param ciphertext: The content of the document. + :type ciphertext: str + :param enc_scheme: The encryption scheme. + :type enc_scheme: str + :param enc_method: The encryption method. + :type enc_method: str + :param enc_iv: The encryption initialization vector. + :type enc_iv: str + :param mac_method: The MAC method to use. + :type mac_method: str + :param secret: The Soledad storage secret + :type secret: str + :param doc_mac: The MAC to be verified against. + :type doc_mac: str + + :raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown. + :raise crypto.WrongMacError: Raised when MAC could not be verified. + """ + calculated_mac = mac_doc( + doc_id, + doc_rev, + ciphertext, + enc_scheme, + enc_method, + enc_iv, + mac_method, + secret) + # we compare mac's hashes to avoid possible timing attacks that might + # exploit python's builtin comparison operator behaviour, which fails + # immediatelly when non-matching bytes are found. + doc_mac_hash = hashlib.sha256( + binascii.a2b_hex( # the mac is stored as hex + doc_mac)).digest() + calculated_mac_hash = hashlib.sha256(calculated_mac).digest() + + if doc_mac_hash != calculated_mac_hash: + logger.warning("Wrong MAC while decrypting doc...") + raise crypto.WrongMacError("Could not authenticate document's " + "contents.") + + def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret): """ - Decrypt C{doc}'s content. + Decrypt a symmetrically encrypted C{doc}'s content. Return the JSON string representation of the document's decrypted content. The passed doc_dict argument should have the following structure: { - ENC_JSON_KEY: '<enc_blob>', - ENC_SCHEME_KEY: '<enc_scheme>', - ENC_METHOD_KEY: '<enc_method>', - ENC_IV_KEY: '<initial value used to encrypt>', # (optional) + crypto.ENC_JSON_KEY: '<enc_blob>', + crypto.ENC_SCHEME_KEY: '<enc_scheme>', + crypto.ENC_METHOD_KEY: '<enc_method>', + crypto.ENC_IV_KEY: '<initial value used to encrypt>', # (optional) MAC_KEY: '<mac>' - MAC_METHOD_KEY: 'hmac' + crypto.MAC_METHOD_KEY: 'hmac' } C{enc_blob} is the encryption of the JSON serialization of the document's content. For now Soledad just deals with documents whose C{enc_scheme} is - EncryptionSchemes.SYMKEY and C{enc_method} is - EncryptionMethods.AES_256_CTR. + crypto.EncryptionSchemes.SYMKEY and C{enc_method} is + crypto.EncryptionMethods.AES_256_CTR. :param doc_dict: The content of the document to be decrypted. :type doc_dict: dict @@ -423,48 +462,35 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret): :return: The JSON serialization of the decrypted content. :rtype: str + + :raise UnknownEncryptionMethodError: Raised when trying to decrypt from an + unknown encryption method. """ - soledad_assert(ENC_JSON_KEY in doc_dict) - soledad_assert(ENC_SCHEME_KEY in doc_dict) - soledad_assert(ENC_METHOD_KEY in doc_dict) - soledad_assert(MAC_KEY in doc_dict) - soledad_assert(MAC_METHOD_KEY in doc_dict) - - # verify MAC - ciphertext = binascii.a2b_hex( # content is stored as hex. - doc_dict[ENC_JSON_KEY]) - mac = mac_doc( - doc_id, doc_rev, - ciphertext, - doc_dict[MAC_METHOD_KEY], secret) - # we compare mac's hashes to avoid possible timing attacks that might - # exploit python's builtin comparison operator behaviour, which fails - # immediatelly when non-matching bytes are found. - doc_mac_hash = hashlib.sha256( - binascii.a2b_hex( # the mac is stored as hex - doc_dict[MAC_KEY])).digest() - calculated_mac_hash = hashlib.sha256(mac).digest() + # assert document dictionary structure + expected_keys = set([ + crypto.ENC_JSON_KEY, + crypto.ENC_SCHEME_KEY, + crypto.ENC_METHOD_KEY, + crypto.ENC_IV_KEY, + crypto.MAC_KEY, + crypto.MAC_METHOD_KEY, + ]) + soledad_assert(expected_keys.issubset(set(doc_dict.keys()))) - if doc_mac_hash != calculated_mac_hash: - logger.warning("Wrong MAC while decrypting doc...") - raise WrongMac('Could not authenticate document\'s contents.') - # decrypt doc's content - enc_scheme = doc_dict[ENC_SCHEME_KEY] - plainjson = None - if enc_scheme == EncryptionSchemes.SYMKEY: - enc_method = doc_dict[ENC_METHOD_KEY] - if enc_method == EncryptionMethods.AES_256_CTR: - soledad_assert(ENC_IV_KEY in doc_dict) - plainjson = decrypt_sym( - ciphertext, key, - method=enc_method, - iv=doc_dict[ENC_IV_KEY]) - else: - raise UnknownEncryptionMethod(enc_method) - else: - raise UnknownEncryptionScheme(enc_scheme) - - return plainjson + ciphertext = binascii.a2b_hex(doc_dict[crypto.ENC_JSON_KEY]) + enc_scheme = doc_dict[crypto.ENC_SCHEME_KEY] + enc_method = doc_dict[crypto.ENC_METHOD_KEY] + enc_iv = doc_dict[crypto.ENC_IV_KEY] + doc_mac = doc_dict[crypto.MAC_KEY] + mac_method = doc_dict[crypto.MAC_METHOD_KEY] + + soledad_assert(enc_scheme == crypto.EncryptionSchemes.SYMKEY) + + _verify_doc_mac( + doc_id, doc_rev, ciphertext, enc_scheme, enc_method, + enc_iv, mac_method, secret, doc_mac) + + return decrypt_sym(ciphertext, key, method=enc_method, iv=enc_iv) def is_symmetrically_encrypted(doc): @@ -476,534 +502,8 @@ def is_symmetrically_encrypted(doc): :rtype: bool """ - if doc.content and ENC_SCHEME_KEY in doc.content: - if doc.content[ENC_SCHEME_KEY] == EncryptionSchemes.SYMKEY: + if doc.content and crypto.ENC_SCHEME_KEY in doc.content: + if doc.content[crypto.ENC_SCHEME_KEY] \ + == crypto.EncryptionSchemes.SYMKEY: return True return False - - -# -# Encrypt/decrypt pools of workers -# - -class SyncEncryptDecryptPool(object): - """ - Base class for encrypter/decrypter pools. - """ - WORKERS = 5 - - def __init__(self, crypto, sync_db, write_lock): - """ - Initialize the pool of encryption-workers. - - :param crypto: A SoledadCryto instance to perform the encryption. - :type crypto: leap.soledad.crypto.SoledadCrypto - - :param sync_db: a database connection handle - :type sync_db: handle - - :param write_lock: a write lock for controlling concurrent access - to the sync_db - :type write_lock: threading.Lock - """ - self._pool = multiprocessing.Pool(self.WORKERS) - self._crypto = crypto - self._sync_db = sync_db - self._sync_db_write_lock = write_lock - - def close(self): - """ - Cleanly close the pool of workers. - """ - logger.debug("Closing %s" % (self.__class__.__name__,)) - self._pool.close() - try: - self._pool.join() - except Exception: - pass - - def terminate(self): - """ - Terminate the pool of workers. - """ - logger.debug("Terminating %s" % (self.__class__.__name__,)) - self._pool.terminate() - - -def encrypt_doc_task(doc_id, doc_rev, content, key, secret): - """ - Encrypt the content of the given document. - - :param doc_id: The document id. - :type doc_id: str - :param doc_rev: The document revision. - :type doc_rev: str - :param content: The serialized content of the document. - :type content: str - :param key: The encryption key. - :type key: str - :param secret: The Soledad secret (used for MAC auth). - :type secret: str - - :return: A tuple containing the doc id, revision and encrypted content. - :rtype: tuple(str, str, str) - """ - encrypted_content = encrypt_docstr( - content, doc_id, doc_rev, key, secret) - return doc_id, doc_rev, encrypted_content - - -class SyncEncrypterPool(SyncEncryptDecryptPool): - """ - Pool of workers that spawn subprocesses to execute the symmetric encryption - of documents to be synced. - """ - # TODO implement throttling to reduce cpu usage?? - WORKERS = 5 - TABLE_NAME = "docs_tosync" - FIELD_NAMES = "doc_id, rev, content" - - def encrypt_doc(self, doc, workers=True): - """ - Symmetrically encrypt a document. - - :param doc: The document with contents to be encrypted. - :type doc: SoledadDocument - - :param workers: Whether to defer the decryption to the multiprocess - pool of workers. Useful for debugging purposes. - :type workers: bool - """ - soledad_assert(self._crypto is not None, "need a crypto object") - docstr = doc.get_json() - key = self._crypto.doc_passphrase(doc.doc_id) - secret = self._crypto.secret - args = doc.doc_id, doc.rev, docstr, key, secret - - try: - if workers: - res = self._pool.apply_async( - encrypt_doc_task, args, - callback=self.encrypt_doc_cb) - else: - # encrypt inline - res = encrypt_doc_task(*args) - self.encrypt_doc_cb(res) - - except Exception as exc: - logger.exception(exc) - - def encrypt_doc_cb(self, result): - """ - Insert results of encryption routine into the local sync database. - - :param result: A tuple containing the doc id, revision and encrypted - content. - :type result: tuple(str, str, str) - """ - doc_id, doc_rev, content = result - self.insert_encrypted_local_doc(doc_id, doc_rev, content) - - def insert_encrypted_local_doc(self, doc_id, doc_rev, content): - """ - Insert the contents of the encrypted doc into the local sync - database. - - :param doc_id: The document id. - :type doc_id: str - :param doc_rev: The document revision. - :type doc_rev: str - :param content: The serialized content of the document. - :type content: str - :param content: The encrypted document. - :type content: str - """ - sql_del = "DELETE FROM '%s' WHERE doc_id=?" % (self.TABLE_NAME,) - sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?)" % (self.TABLE_NAME,) - - con = self._sync_db - with self._sync_db_write_lock: - with con: - con.execute(sql_del, (doc_id, )) - con.execute(sql_ins, (doc_id, doc_rev, content)) - - -def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret): - """ - Decrypt the content of the given document. - - :param doc_id: The document id. - :type doc_id: str - :param doc_rev: The document revision. - :type doc_rev: str - :param content: The encrypted content of the document. - :type content: str - :param gen: The generation corresponding to the modification of that - document. - :type gen: int - :param trans_id: The transaction id corresponding to the modification of - that document. - :type trans_id: str - :param key: The encryption key. - :type key: str - :param secret: The Soledad secret (used for MAC auth). - :type secret: str - - :return: A tuple containing the doc id, revision and encrypted content. - :rtype: tuple(str, str, str) - """ - decrypted_content = decrypt_doc_dict( - content, doc_id, doc_rev, key, secret) - return doc_id, doc_rev, decrypted_content, gen, trans_id - - -def get_insertable_docs_by_gen(expected, got): - """ - Return a list of documents ready to be inserted. This list is computed - by aligning the expected list with the already gotten docs, and returning - the maximum number of docs that can be processed in the expected order - before finding a gap. - - :param expected: A list of generations to be inserted. - :type expected: list - - :param got: A dictionary whose values are the docs to be inserted. - :type got: dict - """ - ordered = [got.get(i) for i in expected] - if None in ordered: - return ordered[:ordered.index(None)] - else: - return ordered - - -class SyncDecrypterPool(SyncEncryptDecryptPool): - """ - Pool of workers that spawn subprocesses to execute the symmetric decryption - of documents that were received. - - The decryption of the received documents is done in two steps: - - 1. All the encrypted docs are collected, together with their generation - and transaction-id - 2. The docs are enqueued for decryption. When completed, they are - inserted following the generation order. - """ - # TODO implement throttling to reduce cpu usage?? - TABLE_NAME = "docs_received" - FIELD_NAMES = "doc_id, rev, content, gen, trans_id" - - write_encrypted_lock = threading.Lock() - - def __init__(self, *args, **kwargs): - """ - Initialize the decrypter pool, and setup a dict for putting the - results of the decrypted docs until they are picked by the insert - routine that gets them in order. - """ - self._insert_doc_cb = kwargs.pop("insert_doc_cb") - SyncEncryptDecryptPool.__init__(self, *args, **kwargs) - self.decrypted_docs = {} - self.source_replica_uid = None - - def set_source_replica_uid(self, source_replica_uid): - """ - Set the source replica uid for this decrypter pool instance. - - :param source_replica_uid: The uid of the source replica. - :type source_replica_uid: str - """ - self.source_replica_uid = source_replica_uid - - def insert_encrypted_received_doc(self, doc_id, doc_rev, content, - gen, trans_id): - """ - Insert a received message with encrypted content, to be decrypted later - on. - - :param doc_id: The Document ID. - :type doc_id: str - :param doc_rev: The Document Revision - :param doc_rev: str - :param content: the Content of the document - :type content: str - :param gen: the Document Generation - :type gen: int - :param trans_id: Transaction ID - :type trans_id: str - """ - docstr = json.dumps(content) - sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?)" % ( - self.TABLE_NAME,) - - con = self._sync_db - with self._sync_db_write_lock: - with con: - con.execute(sql_ins, (doc_id, doc_rev, docstr, gen, trans_id)) - - def insert_marker_for_received_doc(self, doc_id, doc_rev, gen): - """ - Insert a marker with the document id, revision and generation on the - sync db. This document does not have an encrypted payload, so the - content has already been inserted into the decrypted_docs dictionary - from where it can be picked following generation order. - We need to leave here the marker to be able to calculate the expected - insertion order for a synchronization batch. - - :param doc_id: The Document ID. - :type doc_id: str - :param doc_rev: The Document Revision - :param doc_rev: str - :param gen: the Document Generation - :type gen: int - """ - sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?)" % ( - self.TABLE_NAME,) - con = self._sync_db - with self._sync_db_write_lock: - with con: - con.execute(sql_ins, (doc_id, doc_rev, '', gen, '')) - - def insert_received_doc(self, doc_id, doc_rev, content, gen, trans_id): - """ - Insert a document that is not symmetrically encrypted. - We store it in the staging area (the decrypted_docs dictionary) to be - picked up in order as the preceding documents are decrypted. - - :param doc_id: The Document ID. - :type doc_id: str - :param doc_rev: The Document Revision - :param doc_rev: str - :param content: the Content of the document - :type content: str - :param gen: the Document Generation - :type gen: int - :param trans_id: Transaction ID - :type trans_id: str - """ - # XXX this need a deeper review / testing. - # I believe that what I'm doing here is prone to problems - # if the sync is interrupted (ie, client crash) in the worst possible - # moment. We would need a recover strategy in that case - # (or, insert the document in the table all the same, but with a flag - # saying if the document is sym-encrypted or not), - content = json.dumps(content) - result = doc_id, doc_rev, content, gen, trans_id - self.decrypted_docs[gen] = result - self.insert_marker_for_received_doc(doc_id, doc_rev, gen) - - def delete_encrypted_received_doc(self, doc_id, doc_rev): - """ - Delete a encrypted received doc after it was inserted into the local - db. - - :param doc_id: Document ID. - :type doc_id: str - :param doc_rev: Document revision. - :type doc_rev: str - """ - sql_del = "DELETE FROM '%s' WHERE doc_id=? AND rev=?" % ( - self.TABLE_NAME,) - con = self._sync_db - with self._sync_db_write_lock: - with con: - con.execute(sql_del, (doc_id, doc_rev)) - - def decrypt_doc(self, doc_id, rev, source_replica_uid, workers=True): - """ - Symmetrically decrypt a document. - - :param doc_id: The ID for the document with contents to be encrypted. - :type doc: str - :param rev: The revision of the document. - :type rev: str - :param source_replica_uid: - :type source_replica_uid: str - - :param workers: Whether to defer the decryption to the multiprocess - pool of workers. Useful for debugging purposes. - :type workers: bool - """ - self.source_replica_uid = source_replica_uid - - # insert_doc_cb is a proxy object that gets updated with the right - # insert function only when the sync_target invokes the sync_exchange - # method. so, if we don't still have a non-empty callback, we refuse - # to proceed. - if sameProxiedObjects(self._insert_doc_cb.get(source_replica_uid), - None): - logger.debug("Sync decrypter pool: no insert_doc_cb() yet.") - return - - # XXX move to get_doc function... - c = self._sync_db.cursor() - sql = "SELECT * FROM '%s' WHERE doc_id=? AND rev=?" % ( - self.TABLE_NAME,) - try: - c.execute(sql, (doc_id, rev)) - res = c.fetchone() - except Exception as exc: - logger.warning("Error getting docs from syncdb: %r" % (exc,)) - return - if res is None: - logger.debug("Doc %s:%s does not exist in sync db" % (doc_id, rev)) - return - - soledad_assert(self._crypto is not None, "need a crypto object") - try: - doc_id, rev, docstr, gen, trans_id = res - except ValueError: - logger.warning("Wrong entry in sync db") - return - - if len(docstr) == 0: - # not encrypted payload - return - - try: - content = json.loads(docstr) - except TypeError: - logger.warning("Wrong type while decoding json: %s" % repr(docstr)) - return - - key = self._crypto.doc_passphrase(doc_id) - secret = self._crypto.secret - args = doc_id, rev, content, gen, trans_id, key, secret - - try: - if workers: - # Ouch. This is sent to the workers asynchronously, so - # we have no way of logging errors. We'd have to inspect - # lingering results by querying successful / get() over them... - # Or move the heck out of it to twisted. - res = self._pool.apply_async( - decrypt_doc_task, args, - callback=self.decrypt_doc_cb) - else: - # decrypt inline - res = decrypt_doc_task(*args) - self.decrypt_doc_cb(res) - - except Exception as exc: - logger.exception(exc) - - def decrypt_doc_cb(self, result): - """ - Temporarily store the decryption result in a dictionary where it will - be picked by process_decrypted. - - :param result: A tuple containing the doc id, revision and encrypted - content. - :type result: tuple(str, str, str) - """ - doc_id, rev, content, gen, trans_id = result - logger.debug("Sync decrypter pool: decrypted doc %s: %s %s" % (doc_id, rev, gen)) - self.decrypted_docs[gen] = result - - def get_docs_by_generation(self): - """ - Get all documents in the received table from the sync db, - ordered by generation. - - :return: list of doc_id, rev, generation - """ - c = self._sync_db.cursor() - sql = "SELECT doc_id, rev, gen FROM %s ORDER BY gen" % ( - self.TABLE_NAME,) - c.execute(sql) - return c.fetchall() - - def count_received_encrypted_docs(self): - """ - Count how many documents we have in the table for received and - encrypted docs. - - :return: The count of documents. - :rtype: int - """ - if self._sync_db is None: - logger.warning("cannot return count with null sync_db") - return - c = self._sync_db.cursor() - sql = "SELECT COUNT(*) FROM %s" % (self.TABLE_NAME,) - c.execute(sql) - res = c.fetchone() - if res is not None: - return res[0] - else: - return 0 - - def decrypt_received_docs(self): - """ - Get all the encrypted documents from the sync database and dispatch a - decrypt worker to decrypt each one of them. - """ - docs_by_generation = self.get_docs_by_generation() - logger.debug("Sync decrypter pool: There are %d documents to " \ - "decrypt." % len(docs_by_generation)) - for doc_id, rev, gen in filter(None, docs_by_generation): - self.decrypt_doc(doc_id, rev, self.source_replica_uid) - - def process_decrypted(self): - """ - Process the already decrypted documents, and insert as many documents - as can be taken from the expected order without finding a gap. - - :return: Whether we have processed all the pending docs. - :rtype: bool - """ - # Acquire the lock to avoid processing while we're still - # getting data from the syncing stream, to avoid InvalidGeneration - # problems. - with self.write_encrypted_lock: - already_decrypted = self.decrypted_docs - docs = self.get_docs_by_generation() - docs = filter(lambda entry: len(entry) > 0, docs) - expected = [gen for doc_id, rev, gen in docs] - docs_to_insert = get_insertable_docs_by_gen( - expected, already_decrypted) - for doc_fields in docs_to_insert: - self.insert_decrypted_local_doc(*doc_fields) - remaining = self.count_received_encrypted_docs() - return remaining == 0 - - def insert_decrypted_local_doc(self, doc_id, doc_rev, content, - gen, trans_id): - """ - Insert the decrypted document into the local sqlcipher database. - Makes use of the passed callback `return_doc_cb` passed to the caller - by u1db sync. - - :param doc_id: The document id. - :type doc_id: str - :param doc_rev: The document revision. - :type doc_rev: str - :param content: The serialized content of the document. - :type content: str - :param gen: The generation corresponding to the modification of that - document. - :type gen: int - :param trans_id: The transaction id corresponding to the modification - of that document. - :type trans_id: str - """ - # could pass source_replica in params for callback chain - insert_fun = self._insert_doc_cb[self.source_replica_uid] - logger.debug("Sync decrypter pool: inserting doc in local db: " \ - "%s:%s %s" % (doc_id, doc_rev, gen)) - try: - # convert deleted documents to avoid error on document creation - if content == 'null': - content = None - doc = SoledadDocument(doc_id, doc_rev, content) - insert_fun(doc, int(gen), trans_id) - except Exception as exc: - logger.error("Sync decrypter pool: error while inserting " - "decrypted doc into local db.") - logger.exception(exc) - - else: - # If no errors found, remove it from the local temporary dict - # and from the received database. - self.decrypted_docs.pop(gen) - self.delete_encrypted_received_doc(doc_id, doc_rev) diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py new file mode 100644 index 00000000..d9a72b25 --- /dev/null +++ b/client/src/leap/soledad/client/encdecpool.py @@ -0,0 +1,746 @@ +# -*- coding: utf-8 -*- +# encdecpool.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +A pool of encryption/decryption concurrent and parallel workers for using +during synchronization. +""" + + +import multiprocessing +import Queue +import json +import logging + +from twisted.internet import reactor +from twisted.internet import defer +from twisted.internet.threads import deferToThread + +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common import soledad_assert + +from leap.soledad.client.crypto import encrypt_docstr +from leap.soledad.client.crypto import decrypt_doc_dict + + +logger = logging.getLogger(__name__) + + +# +# Encrypt/decrypt pools of workers +# + +class SyncEncryptDecryptPool(object): + """ + Base class for encrypter/decrypter pools. + """ + + # TODO implement throttling to reduce cpu usage?? + WORKERS = multiprocessing.cpu_count() + + def __init__(self, crypto, sync_db): + """ + Initialize the pool of encryption-workers. + + :param crypto: A SoledadCryto instance to perform the encryption. + :type crypto: leap.soledad.crypto.SoledadCrypto + + :param sync_db: A database connection handle + :type sync_db: pysqlcipher.dbapi2.Connection + """ + self._crypto = crypto + self._sync_db = sync_db + self._pool = multiprocessing.Pool(self.WORKERS) + + def close(self): + """ + Cleanly close the pool of workers. + """ + logger.debug("Closing %s" % (self.__class__.__name__,)) + self._pool.close() + try: + self._pool.join() + except Exception: + pass + + def terminate(self): + """ + Terminate the pool of workers. + """ + logger.debug("Terminating %s" % (self.__class__.__name__,)) + self._pool.terminate() + + def _runOperation(self, query, *args): + """ + Run an operation on the sync db. + + :param query: The query to be executed. + :type query: str + :param args: A list of query arguments. + :type args: list + + :return: A deferred that will fire when the operation in the database + has finished. + :rtype: twisted.internet.defer.Deferred + """ + return self._sync_db.runOperation(query, *args) + + def _runQuery(self, query, *args): + """ + Run a query on the sync db. + + :param query: The query to be executed. + :type query: str + :param args: A list of query arguments. + :type args: list + + :return: A deferred that will fire with the results of the database + query. + :rtype: twisted.internet.defer.Deferred + """ + return self._sync_db.runQuery(query, *args) + + +def encrypt_doc_task(doc_id, doc_rev, content, key, secret): + """ + Encrypt the content of the given document. + + :param doc_id: The document id. + :type doc_id: str + :param doc_rev: The document revision. + :type doc_rev: str + :param content: The serialized content of the document. + :type content: str + :param key: The encryption key. + :type key: str + :param secret: The Soledad storage secret (used for MAC auth). + :type secret: str + + :return: A tuple containing the doc id, revision and encrypted content. + :rtype: tuple(str, str, str) + """ + encrypted_content = encrypt_docstr( + content, doc_id, doc_rev, key, secret) + return doc_id, doc_rev, encrypted_content + + +class SyncEncrypterPool(SyncEncryptDecryptPool): + """ + Pool of workers that spawn subprocesses to execute the symmetric encryption + of documents to be synced. + """ + TABLE_NAME = "docs_tosync" + FIELD_NAMES = "doc_id PRIMARY KEY, rev, content" + + ENCRYPT_LOOP_PERIOD = 0.5 + + def __init__(self, *args, **kwargs): + """ + Initialize the sync encrypter pool. + """ + SyncEncryptDecryptPool.__init__(self, *args, **kwargs) + + self._stopped = False + self._sync_queue = multiprocessing.Queue() + + # start the encryption loop + self._deferred_loop = deferToThread(self._encrypt_docs_loop) + self._deferred_loop.addCallback( + lambda _: logger.debug("Finished encrypter thread.")) + + def enqueue_doc_for_encryption(self, doc): + """ + Enqueue a document for encryption. + + :param doc: The document to be encrypted. + :type doc: SoledadDocument + """ + try: + self.sync_queue.put_nowait(doc) + except multiprocessing.Queue.Full: + # do not asynchronously encrypt this file if the queue is full + pass + + def _encrypt_docs_loop(self): + """ + Process the syncing queue and send the documents there to be encrypted + in the sync db. They will be read by the SoledadSyncTarget during the + sync_exchange. + """ + logger.debug("Starting encrypter thread.") + while not self._stopped: + try: + doc = self._sync_queue.get(True, self.ENCRYPT_LOOP_PERIOD) + self._encrypt_doc(doc) + except Queue.Empty: + pass + + def _encrypt_doc(self, doc): + """ + Symmetrically encrypt a document. + + :param doc: The document with contents to be encrypted. + :type doc: SoledadDocument + + :param workers: Whether to defer the decryption to the multiprocess + pool of workers. Useful for debugging purposes. + :type workers: bool + """ + soledad_assert(self._crypto is not None, "need a crypto object") + docstr = doc.get_json() + key = self._crypto.doc_passphrase(doc.doc_id) + secret = self._crypto.secret + args = doc.doc_id, doc.rev, docstr, key, secret + # encrypt asynchronously + self._pool.apply_async( + encrypt_doc_task, args, + callback=self._encrypt_doc_cb) + + def _encrypt_doc_cb(self, result): + """ + Insert results of encryption routine into the local sync database. + + :param result: A tuple containing the doc id, revision and encrypted + content. + :type result: tuple(str, str, str) + """ + doc_id, doc_rev, content = result + return self._insert_encrypted_local_doc(doc_id, doc_rev, content) + + def _insert_encrypted_local_doc(self, doc_id, doc_rev, content): + """ + Insert the contents of the encrypted doc into the local sync + database. + + :param doc_id: The document id. + :type doc_id: str + :param doc_rev: The document revision. + :type doc_rev: str + :param content: The serialized content of the document. + :type content: str + """ + query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?)" \ + % (self.TABLE_NAME,) + return self._runOperation(query, (doc_id, doc_rev, content)) + + @defer.inlineCallbacks + def get_encrypted_doc(self, doc_id, doc_rev): + """ + Get an encrypted document from the sync db. + + :param doc_id: The id of the document. + :type doc_id: str + :param doc_rev: The revision of the document. + :type doc_rev: str + + :return: A deferred that will fire with the encrypted content of the + document or None if the document was not found in the sync + db. + :rtype: twisted.internet.defer.Deferred + """ + logger.debug("Trying to get encrypted doc from sync db: %s" % doc_id) + query = "SELECT content FROM %s WHERE doc_id=? and rev=?" \ + % self.TABLE_NAME + result = yield self._runQuery(query, (doc_id, doc_rev)) + if result: + val = result.pop() + defer.returnValue(val[0]) + defer.returnValue(None) + + def delete_encrypted_doc(self, doc_id, doc_rev): + """ + Delete an encrypted document from the sync db. + + :param doc_id: The id of the document. + :type doc_id: str + :param doc_rev: The revision of the document. + :type doc_rev: str + + :return: A deferred that will fire when the operation in the database + has finished. + :rtype: twisted.internet.defer.Deferred + """ + query = "DELETE FROM %s WHERE doc_id=? and rev=?" \ + % self.TABLE_NAME + self._runOperation(query, (doc_id, doc_rev)) + + def close(self): + """ + Close the encrypter pool. + """ + self._stopped = True + self._sync_queue.close() + q = self._sync_queue + del q + self._sync_queue = None + + +def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret, + idx): + """ + Decrypt the content of the given document. + + :param doc_id: The document id. + :type doc_id: str + :param doc_rev: The document revision. + :type doc_rev: str + :param content: The encrypted content of the document. + :type content: str + :param gen: The generation corresponding to the modification of that + document. + :type gen: int + :param trans_id: The transaction id corresponding to the modification of + that document. + :type trans_id: str + :param key: The encryption key. + :type key: str + :param secret: The Soledad storage secret (used for MAC auth). + :type secret: str + :param idx: The index of this document in the current sync process. + :type idx: int + + :return: A tuple containing the doc id, revision and encrypted content. + :rtype: tuple(str, str, str) + """ + decrypted_content = decrypt_doc_dict(content, doc_id, doc_rev, key, secret) + return doc_id, doc_rev, decrypted_content, gen, trans_id, idx + + +class SyncDecrypterPool(SyncEncryptDecryptPool): + """ + Pool of workers that spawn subprocesses to execute the symmetric decryption + of documents that were received. + + The decryption of the received documents is done in two steps: + + 1. Encrypted documents are stored in the sync db by the actual soledad + sync loop. + 2. The soledad sync loop tells us how many documents we should expect + to process. + 3. We start a decrypt-and-process loop: + + a. Encrypted documents are fetched. + b. Encrypted documents are decrypted. + c. The longest possible list of decrypted documents are inserted + in the soledad db (this depends on which documents have already + arrived and which documents have already been decrypte, because + the order of insertion in the local soledad db matters). + d. Processed documents are deleted from the database. + + 4. When we have processed as many documents as we should, the loop + finishes. + """ + # TODO implement throttling to reduce cpu usage?? + TABLE_NAME = "docs_received" + FIELD_NAMES = "doc_id PRIMARY KEY, rev, content, gen, " \ + "trans_id, encrypted, idx" + + """ + Period of recurrence of the periodic decrypting task, in seconds. + """ + DECRYPT_LOOP_PERIOD = 0.5 + + def __init__(self, *args, **kwargs): + """ + Initialize the decrypter pool, and setup a dict for putting the + results of the decrypted docs until they are picked by the insert + routine that gets them in order. + + :param insert_doc_cb: A callback for inserting received documents from + target. If not overriden, this will call u1db + insert_doc_from_target in synchronizer, which + implements the TAKE OTHER semantics. + :type insert_doc_cb: function + :param source_replica_uid: The source replica uid, used to find the + correct callback for inserting documents. + :type source_replica_uid: str + """ + self._insert_doc_cb = kwargs.pop("insert_doc_cb") + self.source_replica_uid = kwargs.pop("source_replica_uid") + SyncEncryptDecryptPool.__init__(self, *args, **kwargs) + + self._last_inserted_idx = 0 + self._docs_to_process = None + self._processed_docs = 0 + + self._async_results = [] + self._failure = None + self._finished = False + + # XXX we want to empty the database before starting, but this is an + # asynchronous call, so we have to somehow make sure that it is + # executed before any other call to the database, without + # blocking. + self._empty() + + def _launch_decrypt_and_process(self): + d = self._decrypt_and_process_docs() + d.addErrback(lambda f: self._set_failure(f)) + + def _schedule_decrypt_and_process(self): + reactor.callLater( + self.DECRYPT_LOOP_PERIOD, + self._launch_decrypt_and_process) + + @property + def failure(self): + return self._failure + + def _set_failure(self, failure): + self._failure = failure + self._finished = True + + def failed(self): + return bool(self._failure) + + def start(self, docs_to_process): + """ + Set the number of documents we expect to process. + + This should be called by the during the sync exchange process as soon + as we know how many documents are arriving from the server. + + :param docs_to_process: The number of documents to process. + :type docs_to_process: int + """ + self._docs_to_process = docs_to_process + self._schedule_decrypt_and_process() + + def insert_encrypted_received_doc( + self, doc_id, doc_rev, content, gen, trans_id, idx): + """ + Insert a received message with encrypted content, to be decrypted later + on. + + :param doc_id: The Document ID. + :type doc_id: str + :param doc_rev: The Document Revision + :param doc_rev: str + :param content: the Content of the document + :type content: str + :param gen: the Document Generation + :type gen: int + :param trans_id: Transaction ID + :type trans_id: str + :param idx: The index of this document in the current sync process. + :type idx: int + + :return: A deferred that will fire when the operation in the database + has finished. + :rtype: twisted.internet.defer.Deferred + """ + docstr = json.dumps(content) + query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?, ?, ?, ?, ?)" \ + % self.TABLE_NAME + return self._runOperation( + query, (doc_id, doc_rev, docstr, gen, trans_id, 1, idx)) + + def insert_received_doc( + self, doc_id, doc_rev, content, gen, trans_id, idx): + """ + Insert a document that is not symmetrically encrypted. + We store it in the staging area (the decrypted_docs dictionary) to be + picked up in order as the preceding documents are decrypted. + + :param doc_id: The Document ID. + :type doc_id: str + :param doc_rev: The Document Revision + :param doc_rev: str + :param content: the Content of the document + :type content: str + :param gen: the Document Generation + :type gen: int + :param trans_id: Transaction ID + :type trans_id: str + :param idx: The index of this document in the current sync process. + :type idx: int + + :return: A deferred that will fire when the operation in the database + has finished. + :rtype: twisted.internet.defer.Deferred + """ + if not isinstance(content, str): + content = json.dumps(content) + query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?, ?, ?, ?, ?)" \ + % self.TABLE_NAME + return self._runOperation( + query, (doc_id, doc_rev, content, gen, trans_id, 0, idx)) + + def _delete_received_doc(self, doc_id): + """ + Delete a received doc after it was inserted into the local db. + + :param doc_id: Document ID. + :type doc_id: str + + :return: A deferred that will fire when the operation in the database + has finished. + :rtype: twisted.internet.defer.Deferred + """ + query = "DELETE FROM '%s' WHERE doc_id=?" \ + % self.TABLE_NAME + return self._runOperation(query, (doc_id,)) + + def _async_decrypt_doc(self, doc_id, rev, content, gen, trans_id, idx): + """ + Dispatch an asynchronous document decrypting routine and save the + result object. + + :param doc_id: The ID for the document with contents to be encrypted. + :type doc: str + :param rev: The revision of the document. + :type rev: str + :param content: The serialized content of the document. + :type content: str + :param gen: The generation corresponding to the modification of that + document. + :type gen: int + :param trans_id: The transaction id corresponding to the modification + of that document. + :type trans_id: str + :param idx: The index of this document in the current sync process. + :type idx: int + + :return: A deferred that will fire after the document hasa been + decrypted and inserted in the sync db. + :rtype: twisted.internet.defer.Deferred + """ + soledad_assert(self._crypto is not None, "need a crypto object") + + content = json.loads(content) + key = self._crypto.doc_passphrase(doc_id) + secret = self._crypto.secret + args = doc_id, rev, content, gen, trans_id, key, secret, idx + # decrypt asynchronously + self._async_results.append( + self._pool.apply_async( + decrypt_doc_task, args)) + + def _decrypt_doc_cb(self, result): + """ + Store the decryption result in the sync db from where it will later be + picked by _process_decrypted_docs. + + :param result: A tuple containing the document's id, revision, + content, generation, transaction id and sync index. + :type result: tuple(str, str, str, int, str, int) + + :return: A deferred that will fire after the document has been + inserted in the sync db. + :rtype: twisted.internet.defer.Deferred + """ + doc_id, rev, content, gen, trans_id, idx = result + logger.debug("Sync decrypter pool: decrypted doc %s: %s %s %s" + % (doc_id, rev, gen, trans_id)) + return self.insert_received_doc( + doc_id, rev, content, gen, trans_id, idx) + + def _get_docs(self, encrypted=None, order_by='idx', order='ASC'): + """ + Get documents from the received docs table in the sync db. + + :param encrypted: If not None, only return documents with encrypted + field equal to given parameter. + :type encrypted: bool or None + :param order_by: The name of the field to order results. + :type order_by: str + :param order: Whether the order should be ASC or DESC. + :type order: str + + :return: A deferred that will fire with the results of the database + query. + :rtype: twisted.internet.defer.Deferred + """ + query = "SELECT doc_id, rev, content, gen, trans_id, encrypted, " \ + "idx FROM %s" % self.TABLE_NAME + if encrypted is not None: + query += " WHERE encrypted = %d" % int(encrypted) + query += " ORDER BY %s %s" % (order_by, order) + return self._runQuery(query) + + @defer.inlineCallbacks + def _get_insertable_docs(self): + """ + Return a list of non-encrypted documents ready to be inserted. + + :return: A deferred that will fire with the list of insertable + documents. + :rtype: twisted.internet.defer.Deferred + """ + # here, we fetch the list of decrypted documents and compare with the + # index of the last succesfully processed document. + decrypted_docs = yield self._get_docs(encrypted=False) + insertable = [] + last_idx = self._last_inserted_idx + for doc_id, rev, content, gen, trans_id, encrypted, idx in \ + decrypted_docs: + # XXX for some reason, a document might not have been deleted from + # the database. This is a bug. In this point, already + # processed documents should have been removed from the sync + # database and we should not have to skip them here. We need + # to find out why this is happening, fix, and remove the + # skipping below. + if (idx < last_idx + 1): + continue + if (idx != last_idx + 1): + break + insertable.append((doc_id, rev, content, gen, trans_id, idx)) + last_idx += 1 + defer.returnValue(insertable) + + @defer.inlineCallbacks + def _async_decrypt_received_docs(self): + """ + Get all the encrypted documents from the sync database and dispatch a + decrypt worker to decrypt each one of them. + + :return: A deferred that will fire after all documents have been + decrypted and inserted back in the sync db. + :rtype: twisted.internet.defer.Deferred + """ + docs = yield self._get_docs(encrypted=True) + for doc_id, rev, content, gen, trans_id, _, idx in docs: + self._async_decrypt_doc( + doc_id, rev, content, gen, trans_id, idx) + + @defer.inlineCallbacks + def _process_decrypted_docs(self): + """ + Fetch as many decrypted documents as can be taken from the expected + order and insert them in the local replica. + + :return: A deferred that will fire with the list of inserted + documents. + :rtype: twisted.internet.defer.Deferred + """ + insertable = yield self._get_insertable_docs() + for doc_fields in insertable: + self._insert_decrypted_local_doc(*doc_fields) + defer.returnValue(insertable) + + def _delete_processed_docs(self, inserted): + """ + Delete from the sync db documents that have been processed. + + :param inserted: List of documents inserted in the previous process + step. + :type inserted: list + + :return: A list of deferreds that will fire when each operation in the + database has finished. + :rtype: twisted.internet.defer.DeferredList + """ + deferreds = [] + for doc_id, doc_rev, _, _, _, _ in inserted: + deferreds.append( + self._delete_received_doc(doc_id)) + if not deferreds: + return defer.succeed(None) + return defer.gatherResults(deferreds) + + def _insert_decrypted_local_doc(self, doc_id, doc_rev, content, + gen, trans_id, idx): + """ + Insert the decrypted document into the local replica. + + Make use of the passed callback `insert_doc_cb` passed to the caller + by u1db sync. + + :param doc_id: The document id. + :type doc_id: str + :param doc_rev: The document revision. + :type doc_rev: str + :param content: The serialized content of the document. + :type content: str + :param gen: The generation corresponding to the modification of that + document. + :type gen: int + :param trans_id: The transaction id corresponding to the modification + of that document. + :type trans_id: str + """ + # could pass source_replica in params for callback chain + logger.debug("Sync decrypter pool: inserting doc in local db: " + "%s:%s %s" % (doc_id, doc_rev, gen)) + + # convert deleted documents to avoid error on document creation + if content == 'null': + content = None + doc = SoledadDocument(doc_id, doc_rev, content) + gen = int(gen) + self._insert_doc_cb(doc, gen, trans_id) + + # store info about processed docs + self._last_inserted_idx = idx + self._processed_docs += 1 + + def _empty(self): + """ + Empty the received docs table of the sync database. + + :return: A deferred that will fire when the operation in the database + has finished. + :rtype: twisted.internet.defer.Deferred + """ + query = "DELETE FROM %s WHERE 1" % (self.TABLE_NAME,) + return self._runOperation(query) + + def _collect_async_decryption_results(self): + """ + Collect the results of the asynchronous doc decryptions and re-raise + any exception raised by a multiprocessing async decryption call. + + :raise Exception: Raised if an async call has raised an exception. + """ + async_results = self._async_results[:] + for res in async_results: + if res.ready(): + self._decrypt_doc_cb(res.get()) # might raise an exception! + self._async_results.remove(res) + + @defer.inlineCallbacks + def _decrypt_and_process_docs(self): + """ + Decrypt the documents received from remote replica and insert them + into the local one. + + This method implicitelly returns a defferred (see the decorator + above). It should only be called by _launch_decrypt_and_process(). + because this way any exceptions raised here will be stored by the + errback attached to the deferred returned. + + :return: A deferred which will fire after all decrypt, process and + delete operations have been executed. + :rtype: twisted.internet.defer.Deferred + """ + if not self.failed(): + if self._processed_docs < self._docs_to_process: + yield self._async_decrypt_received_docs() + yield self._collect_async_decryption_results() + docs = yield self._process_decrypted_docs() + yield self._delete_processed_docs(docs) + # recurse + self._schedule_decrypt_and_process() + else: + self._finished = True + + def has_finished(self): + """ + Return whether the decrypter has finished its work. + """ + return self._finished diff --git a/client/src/leap/soledad/client/events.py b/client/src/leap/soledad/client/events.py index c4c09ac5..b1379521 100644 --- a/client/src/leap/soledad/client/events.py +++ b/client/src/leap/soledad/client/events.py @@ -20,39 +20,35 @@ Signaling functions. """ +from leap.common.events import emit +from leap.common.events import catalog -SOLEDAD_CREATING_KEYS = 'Creating keys...' -SOLEDAD_DONE_CREATING_KEYS = 'Done creating keys.' -SOLEDAD_DOWNLOADING_KEYS = 'Downloading keys...' -SOLEDAD_DONE_DOWNLOADING_KEYS = 'Done downloading keys.' -SOLEDAD_UPLOADING_KEYS = 'Uploading keys...' -SOLEDAD_DONE_UPLOADING_KEYS = 'Done uploading keys.' -SOLEDAD_NEW_DATA_TO_SYNC = 'New data available.' -SOLEDAD_DONE_DATA_SYNC = 'Done data sync.' -SOLEDAD_SYNC_SEND_STATUS = 'Sync: sent one document.' -SOLEDAD_SYNC_RECEIVE_STATUS = 'Sync: received one document.' -# we want to use leap.common.events to emits signals, if it is available. -try: - from leap.common import events - from leap.common.events import signal - SOLEDAD_CREATING_KEYS = events.proto.SOLEDAD_CREATING_KEYS - SOLEDAD_DONE_CREATING_KEYS = events.proto.SOLEDAD_DONE_CREATING_KEYS - SOLEDAD_DOWNLOADING_KEYS = events.proto.SOLEDAD_DOWNLOADING_KEYS - SOLEDAD_DONE_DOWNLOADING_KEYS = \ - events.proto.SOLEDAD_DONE_DOWNLOADING_KEYS - SOLEDAD_UPLOADING_KEYS = events.proto.SOLEDAD_UPLOADING_KEYS - SOLEDAD_DONE_UPLOADING_KEYS = \ - events.proto.SOLEDAD_DONE_UPLOADING_KEYS - SOLEDAD_NEW_DATA_TO_SYNC = events.proto.SOLEDAD_NEW_DATA_TO_SYNC - SOLEDAD_DONE_DATA_SYNC = events.proto.SOLEDAD_DONE_DATA_SYNC - SOLEDAD_SYNC_SEND_STATUS = events.proto.SOLEDAD_SYNC_SEND_STATUS - SOLEDAD_SYNC_RECEIVE_STATUS = events.proto.SOLEDAD_SYNC_RECEIVE_STATUS +SOLEDAD_CREATING_KEYS = catalog.SOLEDAD_CREATING_KEYS +SOLEDAD_DONE_CREATING_KEYS = catalog.SOLEDAD_DONE_CREATING_KEYS +SOLEDAD_DOWNLOADING_KEYS = catalog.SOLEDAD_DOWNLOADING_KEYS +SOLEDAD_DONE_DOWNLOADING_KEYS = \ + catalog.SOLEDAD_DONE_DOWNLOADING_KEYS +SOLEDAD_UPLOADING_KEYS = catalog.SOLEDAD_UPLOADING_KEYS +SOLEDAD_DONE_UPLOADING_KEYS = \ + catalog.SOLEDAD_DONE_UPLOADING_KEYS +SOLEDAD_NEW_DATA_TO_SYNC = catalog.SOLEDAD_NEW_DATA_TO_SYNC +SOLEDAD_DONE_DATA_SYNC = catalog.SOLEDAD_DONE_DATA_SYNC +SOLEDAD_SYNC_SEND_STATUS = catalog.SOLEDAD_SYNC_SEND_STATUS +SOLEDAD_SYNC_RECEIVE_STATUS = catalog.SOLEDAD_SYNC_RECEIVE_STATUS -except ImportError: - # we define a fake signaling function and fake signal constants that will - # allow for logging signaling attempts in case leap.common.events is not - # available. - def signal(signal, content=""): - logger.info("Would signal: %s - %s." % (str(signal), content)) +__all__ = [ + "catalog", + "emit", + "SOLEDAD_CREATING_KEYS", + "SOLEDAD_DONE_CREATING_KEYS", + "SOLEDAD_DOWNLOADING_KEYS", + "SOLEDAD_DONE_DOWNLOADING_KEYS", + "SOLEDAD_UPLOADING_KEYS", + "SOLEDAD_DONE_UPLOADING_KEYS", + "SOLEDAD_NEW_DATA_TO_SYNC", + "SOLEDAD_DONE_DATA_SYNC", + "SOLEDAD_SYNC_SEND_STATUS", + "SOLEDAD_SYNC_RECEIVE_STATUS", +] diff --git a/client/src/leap/soledad/client/examples/README b/client/src/leap/soledad/client/examples/README new file mode 100644 index 00000000..3aed8377 --- /dev/null +++ b/client/src/leap/soledad/client/examples/README @@ -0,0 +1,4 @@ +Right now, you can find here both an example of use +and the benchmarking scripts. +TODO move benchmark scripts to root scripts/ folder, +and leave here only a minimal example. diff --git a/client/src/leap/soledad/client/examples/benchmarks/.gitignore b/client/src/leap/soledad/client/examples/benchmarks/.gitignore new file mode 100644 index 00000000..2211df63 --- /dev/null +++ b/client/src/leap/soledad/client/examples/benchmarks/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/client/src/leap/soledad/client/examples/benchmarks/get_sample.sh b/client/src/leap/soledad/client/examples/benchmarks/get_sample.sh new file mode 100755 index 00000000..1995eee1 --- /dev/null +++ b/client/src/leap/soledad/client/examples/benchmarks/get_sample.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir tmp +wget http://www.gutenberg.org/cache/epub/101/pg101.txt -O hacker_crackdown.txt diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py new file mode 100644 index 00000000..7fa1e38f --- /dev/null +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +# measure_index_times.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Measure u1db retrieval times for different u1db index situations. +""" +from __future__ import print_function +from functools import partial +import datetime +import hashlib +import os +import sys + +import u1db +from twisted.internet import defer, reactor + +from leap.soledad.client import adbapi +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +folder = os.environ.get("TMPDIR", "tmp") +numdocs = int(os.environ.get("DOCS", "1000")) +silent = os.environ.get("SILENT", False) +tmpdb = os.path.join(folder, "test.soledad") + + +sample_file = os.environ.get("SAMPLE", "hacker_crackdown.txt") +sample_path = os.path.join(os.curdir, sample_file) + +try: + with open(sample_file) as f: + SAMPLE = f.readlines() +except Exception: + print("[!] Problem opening sample file. Did you download " + "the sample, or correctly set 'SAMPLE' env var?") + sys.exit(1) + +if numdocs > len(SAMPLE): + print("[!] Sorry! The requested DOCS number is larger than " + "the num of lines in our sample file") + sys.exit(1) + + +def debug(*args): + if not silent: + print(*args) + +debug("[+] db path:", tmpdb) +debug("[+] num docs", numdocs) + +if os.path.isfile(tmpdb): + debug("[+] Removing existing db file...") + os.remove(tmpdb) + +start_time = datetime.datetime.now() + +opts = SQLCipherOptions(tmpdb, "secret", create=True) +dbpool = adbapi.getConnectionPool(opts) + + +def createDoc(doc): + return dbpool.runU1DBQuery("create_doc", doc) + +db_indexes = { + 'by-chash': ['chash'], + 'by-number': ['number']} + + +def create_indexes(_): + deferreds = [] + for index, definition in db_indexes.items(): + d = dbpool.runU1DBQuery("create_index", index, *definition) + deferreds.append(d) + return defer.gatherResults(deferreds) + + +class TimeWitness(object): + def __init__(self, init_time): + self.init_time = init_time + + def get_time_count(self): + return datetime.datetime.now() - self.init_time + + +def get_from_index(_): + init_time = datetime.datetime.now() + debug("GETTING FROM INDEX...", init_time) + + def printValue(res, time): + print("RESULT->", res) + print("Index Query Took: ", time.get_time_count()) + return res + + d = dbpool.runU1DBQuery( + "get_from_index", "by-chash", + #"1150c7f10fabce0a57ce13071349fc5064f15bdb0cc1bf2852f74ef3f103aff5") + # XXX this is line 89 from the hacker crackdown... + # Should accept any other optional hash as an enviroment variable. + "57793320d4997a673fc7062652da0596c36a4e9fbe31310d2281e67d56d82469") + d.addCallback(printValue, TimeWitness(init_time)) + return d + + +def getAllDocs(): + return dbpool.runU1DBQuery("get_all_docs") + + +def errBack(e): + debug("[!] ERROR FOUND!!!") + e.printTraceback() + reactor.stop() + + +def countDocs(_): + debug("counting docs...") + d = getAllDocs() + d.addCallbacks(printResult, errBack) + d.addCallbacks(allDone, errBack) + return d + + +def printResult(r, **kwargs): + if kwargs: + debug(*kwargs.values()) + elif isinstance(r, u1db.Document): + debug(r.doc_id, r.content['number']) + else: + len_results = len(r[1]) + debug("GOT %s results" % len(r[1])) + + if len_results == numdocs: + debug("ALL GOOD") + else: + debug("[!] MISSING DOCS!!!!!") + raise ValueError("We didn't expect this result len") + + +def allDone(_): + debug("ALL DONE!") + + #if silent: + end_time = datetime.datetime.now() + print((end_time - start_time).total_seconds()) + reactor.stop() + + +def insert_docs(_): + deferreds = [] + for i in range(numdocs): + payload = SAMPLE[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = createDoc(doc) + d.addCallbacks(partial(printResult, i=i, chash=chash, payload=payload), + lambda e: e.printTraceback()) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + +d = create_indexes(None) +d.addCallback(insert_docs) +d.addCallback(get_from_index) +d.addCallback(countDocs) + +reactor.run() diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py new file mode 100644 index 00000000..c6d76e6b --- /dev/null +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +# measure_index_times.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Measure u1db retrieval times for different u1db index situations. +""" +from __future__ import print_function +from functools import partial +import datetime +import hashlib +import os +import sys + +import u1db +from twisted.internet import defer, reactor + +from leap.soledad.client import adbapi +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +folder = os.environ.get("TMPDIR", "tmp") +numdocs = int(os.environ.get("DOCS", "1000")) +silent = os.environ.get("SILENT", False) +tmpdb = os.path.join(folder, "test.soledad") + + +sample_file = os.environ.get("SAMPLE", "hacker_crackdown.txt") +sample_path = os.path.join(os.curdir, sample_file) + +try: + with open(sample_file) as f: + SAMPLE = f.readlines() +except Exception: + print("[!] Problem opening sample file. Did you download " + "the sample, or correctly set 'SAMPLE' env var?") + sys.exit(1) + +if numdocs > len(SAMPLE): + print("[!] Sorry! The requested DOCS number is larger than " + "the num of lines in our sample file") + sys.exit(1) + + +def debug(*args): + if not silent: + print(*args) + +debug("[+] db path:", tmpdb) +debug("[+] num docs", numdocs) + +if os.path.isfile(tmpdb): + debug("[+] Removing existing db file...") + os.remove(tmpdb) + +start_time = datetime.datetime.now() + +opts = SQLCipherOptions(tmpdb, "secret", create=True) +dbpool = adbapi.getConnectionPool(opts) + + +def createDoc(doc, doc_id): + return dbpool.runU1DBQuery("create_doc", doc, doc_id=doc_id) + +db_indexes = { + 'by-chash': ['chash'], + 'by-number': ['number']} + + +def create_indexes(_): + deferreds = [] + for index, definition in db_indexes.items(): + d = dbpool.runU1DBQuery("create_index", index, *definition) + deferreds.append(d) + return defer.gatherResults(deferreds) + + +class TimeWitness(object): + def __init__(self, init_time): + self.init_time = init_time + + def get_time_count(self): + return datetime.datetime.now() - self.init_time + + +def get_from_index(_): + init_time = datetime.datetime.now() + debug("GETTING FROM INDEX...", init_time) + + def printValue(res, time): + print("RESULT->", res) + print("Index Query Took: ", time.get_time_count()) + return res + + d = dbpool.runU1DBQuery( + "get_doc", + #"1150c7f10fabce0a57ce13071349fc5064f15bdb0cc1bf2852f74ef3f103aff5") + # XXX this is line 89 from the hacker crackdown... + # Should accept any other optional hash as an enviroment variable. + "57793320d4997a673fc7062652da0596c36a4e9fbe31310d2281e67d56d82469") + d.addCallback(printValue, TimeWitness(init_time)) + return d + + +def getAllDocs(): + return dbpool.runU1DBQuery("get_all_docs") + + +def errBack(e): + debug("[!] ERROR FOUND!!!") + e.printTraceback() + reactor.stop() + + +def countDocs(_): + debug("counting docs...") + d = getAllDocs() + d.addCallbacks(printResult, errBack) + d.addCallbacks(allDone, errBack) + return d + + +def printResult(r, **kwargs): + if kwargs: + debug(*kwargs.values()) + elif isinstance(r, u1db.Document): + debug(r.doc_id, r.content['number']) + else: + len_results = len(r[1]) + debug("GOT %s results" % len(r[1])) + + if len_results == numdocs: + debug("ALL GOOD") + else: + debug("[!] MISSING DOCS!!!!!") + raise ValueError("We didn't expect this result len") + + +def allDone(_): + debug("ALL DONE!") + + #if silent: + end_time = datetime.datetime.now() + print((end_time - start_time).total_seconds()) + reactor.stop() + + +def insert_docs(_): + deferreds = [] + for i in range(numdocs): + payload = SAMPLE[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = createDoc(doc, doc_id=chash) + d.addCallbacks(partial(printResult, i=i, chash=chash, payload=payload), + lambda e: e.printTraceback()) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + +d = create_indexes(None) +d.addCallback(insert_docs) +d.addCallback(get_from_index) +d.addCallback(countDocs) + +reactor.run() diff --git a/client/src/leap/soledad/client/examples/compare.txt b/client/src/leap/soledad/client/examples/compare.txt new file mode 100644 index 00000000..19a1325a --- /dev/null +++ b/client/src/leap/soledad/client/examples/compare.txt @@ -0,0 +1,8 @@ +TIMES=100 TMPDIR=/media/sdb5/leap python use_adbapi.py 1.34s user 0.16s system 53% cpu 2.832 total +TIMES=100 TMPDIR=/media/sdb5/leap python use_api.py 1.22s user 0.14s system 62% cpu 2.181 total + +TIMES=1000 TMPDIR=/media/sdb5/leap python use_api.py 2.18s user 0.34s system 27% cpu 9.213 total +TIMES=1000 TMPDIR=/media/sdb5/leap python use_adbapi.py 2.40s user 0.34s system 39% cpu 7.004 total + +TIMES=5000 TMPDIR=/media/sdb5/leap python use_api.py 6.63s user 1.27s system 13% cpu 57.882 total +TIMES=5000 TMPDIR=/media/sdb5/leap python use_adbapi.py 6.84s user 1.26s system 36% cpu 22.367 total diff --git a/client/src/leap/soledad/client/examples/manifest.phk b/client/src/leap/soledad/client/examples/manifest.phk new file mode 100644 index 00000000..2c86c07d --- /dev/null +++ b/client/src/leap/soledad/client/examples/manifest.phk @@ -0,0 +1,50 @@ +The Hacker's Manifesto + +The Hacker's Manifesto +by: The Mentor + +Another one got caught today, it's all over the papers. "Teenager +Arrested in Computer Crime Scandal", "Hacker Arrested after Bank +Tampering." "Damn kids. They're all alike." But did you, in your +three-piece psychology and 1950's technobrain, ever take a look behind +the eyes of the hacker? Did you ever wonder what made him tick, what +forces shaped him, what may have molded him? I am a hacker, enter my +world. Mine is a world that begins with school. I'm smarter than most of +the other kids, this crap they teach us bores me. "Damn underachiever. +They're all alike." I'm in junior high or high school. I've listened to +teachers explain for the fifteenth time how to reduce a fraction. I +understand it. "No, Ms. Smith, I didn't show my work. I did it in +my head." "Damn kid. Probably copied it. They're all alike." I made a +discovery today. I found a computer. Wait a second, this is cool. It does +what I want it to. If it makes a mistake, it's because I screwed it up. +Not because it doesn't like me, or feels threatened by me, or thinks I'm +a smart ass, or doesn't like teaching and shouldn't be here. Damn kid. +All he does is play games. They're all alike. And then it happened... a +door opened to a world... rushing through the phone line like heroin +through an addict's veins, an electronic pulse is sent out, a refuge from +the day-to-day incompetencies is sought... a board is found. "This is +it... this is where I belong..." I know everyone here... even if I've +never met them, never talked to them, may never hear from them again... I +know you all... Damn kid. Tying up the phone line again. They're all +alike... You bet your ass we're all alike... we've been spoon-fed baby +food at school when we hungered for steak... the bits of meat that you +did let slip through were pre-chewed and tasteless. We've been dominated +by sadists, or ignored by the apathetic. The few that had something to +teach found us willing pupils, but those few are like drops of water in +the desert. This is our world now... the world of the electron and the +switch, the beauty of the baud. We make use of a service already existing +without paying for what could be dirt-cheap if it wasn't run by +profiteering gluttons, and you call us criminals. We explore... and you +call us criminals. We seek after knowledge... and you call us criminals. +We exist without skin color, without nationality, without religious +bias... and you call us criminals. You build atomic bombs, you wage wars, +you murder, cheat, and lie to us and try to make us believe it's for our +own good, yet we're the criminals. Yes, I am a criminal. My crime is that +of curiosity. My crime is that of judging people by what they say and +think, not what they look like. My crime is that of outsmarting you, +something that you will never forgive me for. I am a hacker, and this is +my manifesto. You may stop this individual, but you can't stop us all... +after all, we're all alike. + +This was the last published file written by The Mentor. Shortly after +releasing it, he was busted by the FBI. The Mentor, sadly missed. diff --git a/client/src/leap/soledad/client/examples/plot-async-db.py b/client/src/leap/soledad/client/examples/plot-async-db.py new file mode 100644 index 00000000..018a1a1d --- /dev/null +++ b/client/src/leap/soledad/client/examples/plot-async-db.py @@ -0,0 +1,45 @@ +import csv +from matplotlib import pyplot as plt + +FILE = "bench.csv" + +# config the plot +plt.xlabel('number of inserts') +plt.ylabel('time (seconds)') +plt.title('SQLCipher parallelization') + +kwargs = { + 'linewidth': 1.0, + 'linestyle': '-', +} + +series = (('sync', 'r'), + ('async', 'g')) + +data = {'mark': [], + 'sync': [], + 'async': []} + +with open(FILE, 'rb') as csvfile: + series_reader = csv.reader(csvfile, delimiter=',') + for m, s, a in series_reader: + data['mark'].append(int(m)) + data['sync'].append(float(s)) + data['async'].append(float(a)) + +xmax = max(data['mark']) +xmin = min(data['mark']) +ymax = max(data['sync'] + data['async']) +ymin = min(data['sync'] + data['async']) + +for run in series: + name = run[0] + color = run[1] + plt.plot(data['mark'], data[name], label=name, color=color, **kwargs) + +plt.axes().annotate("", xy=(xmax, ymax)) +plt.axes().annotate("", xy=(xmin, ymin)) + +plt.grid() +plt.legend() +plt.show() diff --git a/client/src/leap/soledad/client/examples/run_benchmark.py b/client/src/leap/soledad/client/examples/run_benchmark.py new file mode 100644 index 00000000..a112cf45 --- /dev/null +++ b/client/src/leap/soledad/client/examples/run_benchmark.py @@ -0,0 +1,28 @@ +""" +Run a mini-benchmark between regular api and dbapi +""" +import commands +import os +import time + +TMPDIR = os.environ.get("TMPDIR", "/tmp") +CSVFILE = 'bench.csv' + +cmd = "SILENT=1 TIMES={times} TMPDIR={tmpdir} python ./use_{version}api.py" + +parse_time = lambda r: r.split('\n')[-1] + + +with open(CSVFILE, 'w') as log: + + for times in range(0, 10000, 500): + cmd1 = cmd.format(times=times, tmpdir=TMPDIR, version="") + sync_time = parse_time(commands.getoutput(cmd1)) + + cmd2 = cmd.format(times=times, tmpdir=TMPDIR, version="adb") + async_time = parse_time(commands.getoutput(cmd2)) + + print times, sync_time, async_time + log.write("%s, %s, %s\n" % (times, sync_time, async_time)) + log.flush() + time.sleep(2) diff --git a/client/src/leap/soledad/client/examples/soledad_sync.py b/client/src/leap/soledad/client/examples/soledad_sync.py new file mode 100644 index 00000000..6d0f6595 --- /dev/null +++ b/client/src/leap/soledad/client/examples/soledad_sync.py @@ -0,0 +1,65 @@ +from leap.bitmask.config.providerconfig import ProviderConfig +from leap.bitmask.crypto.srpauth import SRPAuth +from leap.soledad.client import Soledad + +import logging +logging.basicConfig(level=logging.DEBUG) + + +# EDIT THIS -------------------------------------------- +user = u"USERNAME" +uuid = u"USERUUID" +_pass = u"USERPASS" +server_url = "https://soledad.server.example.org:2323" +# EDIT THIS -------------------------------------------- + +secrets_path = "/tmp/%s.secrets" % uuid +local_db_path = "/tmp/%s.soledad" % uuid +cert_file = "/tmp/cacert.pem" +provider_config = '/tmp/cdev.json' + + +provider = ProviderConfig() +provider.load(provider_config) + +soledad = None + + +def printStuff(r): + print r + + +def printErr(err): + logging.exception(err.value) + + +def init_soledad(_): + token = srpauth.get_token() + print "token", token + + global soledad + soledad = Soledad(uuid, _pass, secrets_path, local_db_path, + server_url, cert_file, + auth_token=token, defer_encryption=False) + + def getall(_): + d = soledad.get_all_docs() + return d + + d1 = soledad.create_doc({"test": 42}) + d1.addCallback(getall) + d1.addCallbacks(printStuff, printErr) + + d2 = soledad.sync() + d2.addCallbacks(printStuff, printErr) + d2.addBoth(lambda r: reactor.stop()) + + +srpauth = SRPAuth(provider) + +d = srpauth.authenticate(user, _pass) +d.addCallbacks(init_soledad, printErr) + + +from twisted.internet import reactor +reactor.run() diff --git a/client/src/leap/soledad/client/examples/use_adbapi.py b/client/src/leap/soledad/client/examples/use_adbapi.py new file mode 100644 index 00000000..d7bd21f2 --- /dev/null +++ b/client/src/leap/soledad/client/examples/use_adbapi.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# use_adbapi.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Example of use of the asynchronous soledad api. +""" +from __future__ import print_function +import datetime +import os + +import u1db +from twisted.internet import defer, reactor + +from leap.soledad.client import adbapi +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +folder = os.environ.get("TMPDIR", "tmp") +times = int(os.environ.get("TIMES", "1000")) +silent = os.environ.get("SILENT", False) + +tmpdb = os.path.join(folder, "test.soledad") + + +def debug(*args): + if not silent: + print(*args) + +debug("[+] db path:", tmpdb) +debug("[+] times", times) + +if os.path.isfile(tmpdb): + debug("[+] Removing existing db file...") + os.remove(tmpdb) + +start_time = datetime.datetime.now() + +opts = SQLCipherOptions(tmpdb, "secret", create=True) +dbpool = adbapi.getConnectionPool(opts) + + +def createDoc(doc): + return dbpool.runU1DBQuery("create_doc", doc) + + +def getAllDocs(): + return dbpool.runU1DBQuery("get_all_docs") + + +def countDocs(_): + debug("counting docs...") + d = getAllDocs() + d.addCallbacks(printResult, lambda e: e.printTraceback()) + d.addBoth(allDone) + + +def printResult(r): + if isinstance(r, u1db.Document): + debug(r.doc_id, r.content['number']) + else: + len_results = len(r[1]) + debug("GOT %s results" % len(r[1])) + + if len_results == times: + debug("ALL GOOD") + else: + raise ValueError("We didn't expect this result len") + + +def allDone(_): + debug("ALL DONE!") + if silent: + end_time = datetime.datetime.now() + print((end_time - start_time).total_seconds()) + reactor.stop() + +deferreds = [] +payload = open('manifest.phk').read() + +for i in range(times): + doc = {"number": i, "payload": payload} + d = createDoc(doc) + d.addCallbacks(printResult, lambda e: e.printTraceback()) + deferreds.append(d) + + +all_done = defer.gatherResults(deferreds, consumeErrors=True) +all_done.addCallback(countDocs) + +reactor.run() diff --git a/client/src/leap/soledad/client/examples/use_api.py b/client/src/leap/soledad/client/examples/use_api.py new file mode 100644 index 00000000..e2501c98 --- /dev/null +++ b/client/src/leap/soledad/client/examples/use_api.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# use_api.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Example of use of the soledad api. +""" +from __future__ import print_function +import datetime +import os + +from leap.soledad.client import sqlcipher +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +folder = os.environ.get("TMPDIR", "tmp") +times = int(os.environ.get("TIMES", "1000")) +silent = os.environ.get("SILENT", False) + +tmpdb = os.path.join(folder, "test.soledad") + + +def debug(*args): + if not silent: + print(*args) + +debug("[+] db path:", tmpdb) +debug("[+] times", times) + +if os.path.isfile(tmpdb): + debug("[+] Removing existing db file...") + os.remove(tmpdb) + +start_time = datetime.datetime.now() + +opts = SQLCipherOptions(tmpdb, "secret", create=True) +db = sqlcipher.SQLCipherDatabase(opts) + + +def allDone(): + debug("ALL DONE!") + +payload = open('manifest.phk').read() + +for i in range(times): + doc = {"number": i, "payload": payload} + d = db.create_doc(doc) + debug(d.doc_id, d.content['number']) + +debug("Count", len(db.get_all_docs()[1])) +if silent: + end_time = datetime.datetime.now() + print((end_time - start_time).total_seconds()) + +allDone() diff --git a/client/src/leap/soledad/client/http_target.py b/client/src/leap/soledad/client/http_target.py new file mode 100644 index 00000000..30590ae1 --- /dev/null +++ b/client/src/leap/soledad/client/http_target.py @@ -0,0 +1,622 @@ +# -*- coding: utf-8 -*- +# http_target.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +A U1DB backend for encrypting data before sending to server and decrypting +after receiving. +""" + + +import json +import base64 +import logging + +from uuid import uuid4 +from functools import partial + +from twisted.internet import defer +from twisted.internet import reactor +from twisted.web.error import Error + +from u1db import errors +from u1db import SyncTarget +from u1db.remote import utils + +from leap.common.http import HTTPClient + +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.errors import InvalidAuthTokenError + +from leap.soledad.client.crypto import is_symmetrically_encrypted +from leap.soledad.client.crypto import encrypt_doc +from leap.soledad.client.crypto import decrypt_doc +from leap.soledad.client.events import SOLEDAD_SYNC_SEND_STATUS +from leap.soledad.client.events import SOLEDAD_SYNC_RECEIVE_STATUS +from leap.soledad.client.events import emit +from leap.soledad.client.encdecpool import SyncDecrypterPool + + +logger = logging.getLogger(__name__) + + +class SoledadHTTPSyncTarget(SyncTarget): + """ + A SyncTarget that encrypts data before sending and decrypts data after + receiving. + + Normally encryption will have been written to the sync database upon + document modification. The sync database is also used to write temporarily + the parsed documents that the remote send us, before being decrypted and + written to the main database. + """ + + def __init__(self, url, source_replica_uid, creds, crypto, cert_file, + sync_db=None, sync_enc_pool=None): + """ + Initialize the sync target. + + :param url: The server sync url. + :type url: str + :param source_replica_uid: The source replica uid which we use when + deferring decryption. + :type source_replica_uid: str + :param url: The url of the target replica to sync with. + :type url: str + :param creds: A dictionary containing the uuid and token. + :type creds: creds + :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt + document contents when syncing. + :type crypto: soledad.crypto.SoledadCrypto + :param cert_file: Path to the certificate of the ca used to validate + the SSL certificate used by the remote soledad + server. + :type cert_file: str + :param sync_db: Optional. handler for the db with the symmetric + encryption of the syncing documents. If + None, encryption will be done in-place, + instead of retreiving it from the dedicated + database. + :type sync_db: Sqlite handler + :param verify_ssl: Whether we should perform SSL server certificate + verification. + :type verify_ssl: bool + """ + if url.endswith("/"): + url = url[:-1] + self._url = str(url) + "/sync-from/" + source_replica_uid + self.source_replica_uid = source_replica_uid + self._auth_header = None + self.set_creds(creds) + self._crypto = crypto + self._sync_db = sync_db + self._sync_enc_pool = sync_enc_pool + self._insert_doc_cb = None + # asynchronous encryption/decryption attributes + self._decryption_callback = None + self._sync_decr_pool = None + self._http = HTTPClient(cert_file) + + def set_creds(self, creds): + """ + Update credentials. + + :param creds: A dictionary containing the uuid and token. + :type creds: dict + """ + uuid = creds['token']['uuid'] + token = creds['token']['token'] + auth = '%s:%s' % (uuid, token) + b64_token = base64.b64encode(auth) + self._auth_header = {'Authorization': ['Token %s' % b64_token]} + + @property + def _defer_encryption(self): + return self._sync_enc_pool is not None + + # + # SyncTarget API + # + + @defer.inlineCallbacks + def get_sync_info(self, source_replica_uid): + """ + Return information about known state of remote database. + + Return the replica_uid and the current database generation of the + remote database, and its last-seen database generation for the client + replica. + + :param source_replica_uid: The client-size replica uid. + :type source_replica_uid: str + + :return: A deferred which fires with (target_replica_uid, + target_replica_generation, target_trans_id, + source_replica_last_known_generation, + source_replica_last_known_transaction_id) + :rtype: twisted.internet.defer.Deferred + """ + raw = yield self._http_request(self._url, headers=self._auth_header) + res = json.loads(raw) + defer.returnValue([ + res['target_replica_uid'], + res['target_replica_generation'], + res['target_replica_transaction_id'], + res['source_replica_generation'], + res['source_transaction_id'] + ]) + + def record_sync_info( + self, source_replica_uid, source_replica_generation, + source_replica_transaction_id): + """ + Record tip information for another replica. + + After sync_exchange has been processed, the caller will have + received new content from this replica. This call allows the + source replica instigating the sync to inform us what their + generation became after applying the documents we returned. + + This is used to allow future sync operations to not need to repeat data + that we just talked about. It also means that if this is called at the + wrong time, there can be database records that will never be + synchronized. + + :param source_replica_uid: The identifier for the source replica. + :type source_replica_uid: str + :param source_replica_generation: The database generation for the + source replica. + :type source_replica_generation: int + :param source_replica_transaction_id: The transaction id associated + with the source replica + generation. + :type source_replica_transaction_id: str + + :return: A deferred which fires with the result of the query. + :rtype: twisted.internet.defer.Deferred + """ + data = json.dumps({ + 'generation': source_replica_generation, + 'transaction_id': source_replica_transaction_id + }) + headers = self._auth_header.copy() + headers.update({'content-type': ['application/json']}) + return self._http_request( + self._url, + method='PUT', + headers=headers, + body=data) + + @defer.inlineCallbacks + def sync_exchange(self, docs_by_generation, source_replica_uid, + last_known_generation, last_known_trans_id, + insert_doc_cb, ensure_callback=None, + defer_decryption=True, sync_id=None): + """ + Find out which documents the remote database does not know about, + encrypt and send them. After that, receive documents from the remote + database. + + :param docs_by_generations: A list of (doc_id, generation, trans_id) + of local documents that were changed since + the last local generation the remote + replica knows about. + :type docs_by_generations: list of tuples + + :param source_replica_uid: The uid of the source replica. + :type source_replica_uid: str + + :param last_known_generation: Target's last known generation. + :type last_known_generation: int + + :param last_known_trans_id: Target's last known transaction id. + :type last_known_trans_id: str + + :param insert_doc_cb: A callback for inserting received documents from + target. If not overriden, this will call u1db + insert_doc_from_target in synchronizer, which + implements the TAKE OTHER semantics. + :type insert_doc_cb: function + + :param ensure_callback: A callback that ensures we know the target + replica uid if the target replica was just + created. + :type ensure_callback: function + + :param defer_decryption: Whether to defer the decryption process using + the intermediate database. If False, + decryption will be done inline. + :type defer_decryption: bool + + :return: A deferred which fires with the new generation and + transaction id of the target replica. + :rtype: twisted.internet.defer.Deferred + """ + + self._ensure_callback = ensure_callback + + if sync_id is None: + sync_id = str(uuid4()) + self.source_replica_uid = source_replica_uid + + # save a reference to the callback so we can use it after decrypting + self._insert_doc_cb = insert_doc_cb + + gen_after_send, trans_id_after_send = yield self._send_docs( + docs_by_generation, + last_known_generation, + last_known_trans_id, + sync_id) + + cur_target_gen, cur_target_trans_id = yield self._receive_docs( + last_known_generation, last_known_trans_id, + ensure_callback, sync_id, + defer_decryption=defer_decryption) + + # update gen and trans id info in case we just sent and did not + # receive docs. + if gen_after_send is not None and gen_after_send > cur_target_gen: + cur_target_gen = gen_after_send + cur_target_trans_id = trans_id_after_send + + defer.returnValue([cur_target_gen, cur_target_trans_id]) + + # + # methods to send docs + # + + def _prepare(self, comma, entries, **dic): + entry = comma + '\r\n' + json.dumps(dic) + entries.append(entry) + return len(entry) + + @defer.inlineCallbacks + def _send_docs(self, docs_by_generation, last_known_generation, + last_known_trans_id, sync_id): + + if not docs_by_generation: + defer.returnValue([None, None]) + + headers = self._auth_header.copy() + headers.update({'content-type': ['application/x-soledad-sync-put']}) + # add remote replica metadata to the request + first_entries = ['['] + self._prepare( + '', first_entries, + last_known_generation=last_known_generation, + last_known_trans_id=last_known_trans_id, + sync_id=sync_id, + ensure=self._ensure_callback is not None) + idx = 0 + total = len(docs_by_generation) + for doc, gen, trans_id in docs_by_generation: + idx += 1 + result = yield self._send_one_doc( + headers, first_entries, doc, + gen, trans_id, total, idx) + if self._defer_encryption: + self._sync_enc_pool.delete_encrypted_doc( + doc.doc_id, doc.rev) + emit(SOLEDAD_SYNC_SEND_STATUS, + "Soledad sync send status: %d/%d" + % (idx, total)) + response_dict = json.loads(result)[0] + gen_after_send = response_dict['new_generation'] + trans_id_after_send = response_dict['new_transaction_id'] + defer.returnValue([gen_after_send, trans_id_after_send]) + + @defer.inlineCallbacks + def _send_one_doc(self, headers, first_entries, doc, gen, trans_id, + number_of_docs, doc_idx): + entries = first_entries[:] + # add the document to the request + content = yield self._encrypt_doc(doc) + self._prepare( + ',', entries, + id=doc.doc_id, rev=doc.rev, content=content, gen=gen, + trans_id=trans_id, number_of_docs=number_of_docs, + doc_idx=doc_idx) + entries.append('\r\n]') + data = ''.join(entries) + result = yield self._http_request( + self._url, + method='POST', + headers=headers, + body=data) + defer.returnValue(result) + + def _encrypt_doc(self, doc): + d = None + if doc.is_tombstone(): + d = defer.succeed(None) + elif not self._defer_encryption: + # fallback case, for tests + d = defer.succeed(encrypt_doc(self._crypto, doc)) + else: + + def _maybe_encrypt_doc_inline(doc_json): + if doc_json is None: + # the document is not marked as tombstone, but we got + # nothing from the sync db. As it is not encrypted + # yet, we force inline encryption. + return encrypt_doc(self._crypto, doc) + return doc_json + + d = self._sync_enc_pool.get_encrypted_doc(doc.doc_id, doc.rev) + d.addCallback(_maybe_encrypt_doc_inline) + return d + + # + # methods to receive doc + # + + @defer.inlineCallbacks + def _receive_docs(self, last_known_generation, last_known_trans_id, + ensure_callback, sync_id, defer_decryption): + + self._queue_for_decrypt = defer_decryption \ + and self._sync_db is not None + + new_generation = last_known_generation + new_transaction_id = last_known_trans_id + + if self._queue_for_decrypt: + logger.debug( + "Soledad sync: will queue received docs for decrypting.") + + if defer_decryption: + self._setup_sync_decr_pool() + + headers = self._auth_header.copy() + headers.update({'content-type': ['application/x-soledad-sync-get']}) + + #--------------------------------------------------------------------- + # maybe receive the first document + #--------------------------------------------------------------------- + + # we fetch the first document before fetching the rest because we need + # to know the total number of documents to be received, and this + # information comes as metadata to each request. + + d = self._receive_one_doc( + headers, last_known_generation, last_known_trans_id, + sync_id, 0) + d.addCallback(partial(self._insert_received_doc, 1, 1)) + number_of_changes, ngen, ntrans = yield d + + if defer_decryption: + self._sync_decr_pool.start(number_of_changes) + + #--------------------------------------------------------------------- + # maybe receive the rest of the documents + #--------------------------------------------------------------------- + + # launch many asynchronous fetches and inserts of received documents + # in the temporary sync db. Will wait for all results before + # continuing. + + received = 1 + deferreds = [] + while received < number_of_changes: + d = self._receive_one_doc( + headers, last_known_generation, + last_known_trans_id, sync_id, received) + d.addCallback( + partial( + self._insert_received_doc, + received + 1, # the index of the current received doc + number_of_changes)) + deferreds.append(d) + received += 1 + results = yield defer.gatherResults(deferreds) + + # get generation and transaction id of target after insertions + if deferreds: + _, new_generation, new_transaction_id = results.pop() + + #--------------------------------------------------------------------- + # wait for async decryption to finish + #--------------------------------------------------------------------- + + # below we do a trick so we can wait for the SyncDecrypterPool to + # finish its work before finally returning the new generation and + # transaction id of the remote replica. To achieve that, we create a + # Deferred that will return the results of the sync and, if we are + # decrypting asynchronously, we use reactor.callLater() to + # periodically poll the decrypter and check if it has finished its + # work. When it has finished, we either call the callback or errback + # of that deferred. In case we are not asynchronously decrypting, we + # just fire the deferred. + + def _shutdown_and_finish(res): + self._sync_decr_pool.close() + return new_generation, new_transaction_id + + d = defer.Deferred() + d.addCallback(_shutdown_and_finish) + + def _wait_or_finish(): + if not self._sync_decr_pool.has_finished(): + reactor.callLater( + SyncDecrypterPool.DECRYPT_LOOP_PERIOD, + _wait_or_finish) + else: + if not self._sync_decr_pool.failed(): + d.callback(None) + else: + d.errback(self._sync_decr_pool.failure) + + if defer_decryption: + _wait_or_finish() + else: + d.callback(None) + + new_generation, new_transaction_id = yield d + defer.returnValue([new_generation, new_transaction_id]) + + def _receive_one_doc(self, headers, last_known_generation, + last_known_trans_id, sync_id, received): + entries = ['['] + # add remote replica metadata to the request + self._prepare( + '', entries, + last_known_generation=last_known_generation, + last_known_trans_id=last_known_trans_id, + sync_id=sync_id, + ensure=self._ensure_callback is not None) + # inform server of how many documents have already been received + self._prepare( + ',', entries, received=received) + entries.append('\r\n]') + # send headers + return self._http_request( + self._url, + method='POST', + headers=headers, + body=''.join(entries)) + + def _insert_received_doc(self, idx, total, response): + """ + Insert a received document into the local replica. + + :param idx: The index count of the current operation. + :type idx: int + :param total: The total number of operations. + :type total: int + :param response: The body and headers of the response. + :type response: tuple(str, dict) + """ + new_generation, new_transaction_id, number_of_changes, doc_id, \ + rev, content, gen, trans_id = \ + self._parse_received_doc_response(response) + if doc_id is not None: + # decrypt incoming document and insert into local database + # ------------------------------------------------------------- + # symmetric decryption of document's contents + # ------------------------------------------------------------- + # If arriving content was symmetrically encrypted, we decrypt it. + # We do it inline if defer_decryption flag is False or no sync_db + # was defined, otherwise we defer it writing it to the received + # docs table. + doc = SoledadDocument(doc_id, rev, content) + if is_symmetrically_encrypted(doc): + if self._queue_for_decrypt: + self._sync_decr_pool.insert_encrypted_received_doc( + doc.doc_id, doc.rev, doc.content, gen, trans_id, + idx) + else: + # defer_decryption is False or no-sync-db fallback + doc.set_json(decrypt_doc(self._crypto, doc)) + self._insert_doc_cb(doc, gen, trans_id) + else: + # not symmetrically encrypted doc, insert it directly + # or save it in the decrypted stage. + if self._queue_for_decrypt: + self._sync_decr_pool.insert_received_doc( + doc.doc_id, doc.rev, doc.content, gen, trans_id, + idx) + else: + self._insert_doc_cb(doc, gen, trans_id) + # ------------------------------------------------------------- + # end of symmetric decryption + # ------------------------------------------------------------- + msg = "%d/%d" % (idx, total) + emit(SOLEDAD_SYNC_RECEIVE_STATUS, msg) + logger.debug("Soledad sync receive status: %s" % msg) + return number_of_changes, new_generation, new_transaction_id + + def _parse_received_doc_response(self, response): + """ + Parse the response from the server containing the received document. + + :param response: The body and headers of the response. + :type response: tuple(str, dict) + + :return: (new_gen, new_trans_id, number_of_changes, doc_id, rev, + content, gen, trans_id) + :rtype: tuple + """ + # decode incoming stream + parts = response.splitlines() + if not parts or parts[0] != '[' or parts[-1] != ']': + raise errors.BrokenSyncStream + data = parts[1:-1] + # decode metadata + line, comma = utils.check_and_strip_comma(data[0]) + metadata = None + try: + metadata = json.loads(line) + new_generation = metadata['new_generation'] + new_transaction_id = metadata['new_transaction_id'] + number_of_changes = metadata['number_of_changes'] + except (json.JSONDecodeError, KeyError): + raise errors.BrokenSyncStream + # make sure we have replica_uid from fresh new dbs + if self._ensure_callback and 'replica_uid' in metadata: + self._ensure_callback(metadata['replica_uid']) + # parse incoming document info + doc_id = None + rev = None + content = None + gen = None + trans_id = None + if number_of_changes > 0: + try: + entry = json.loads(data[1]) + doc_id = entry['id'] + rev = entry['rev'] + content = entry['content'] + gen = entry['gen'] + trans_id = entry['trans_id'] + except (IndexError, KeyError): + raise errors.BrokenSyncStream + return new_generation, new_transaction_id, number_of_changes, \ + doc_id, rev, content, gen, trans_id + + def _setup_sync_decr_pool(self): + """ + Set up the SyncDecrypterPool for deferred decryption. + """ + if self._sync_decr_pool is None and self._sync_db is not None: + # initialize syncing queue decryption pool + self._sync_decr_pool = SyncDecrypterPool( + self._crypto, + self._sync_db, + insert_doc_cb=self._insert_doc_cb, + source_replica_uid=self.source_replica_uid) + + def _http_request(self, url, method='GET', body=None, headers={}): + d = self._http.request(url, method, body, headers) + d.addErrback(_unauth_to_invalid_token_error) + return d + + +def _unauth_to_invalid_token_error(failure): + """ + An errback to translate unauthorized errors to our own invalid token + class. + + :param failure: The original failure. + :type failure: twisted.python.failure.Failure + + :return: Either the original failure or an invalid auth token error. + :rtype: twisted.python.failure.Failure + """ + failure.trap(Error) + if failure.getErrorMessage() == "401 Unauthorized": + raise InvalidAuthTokenError + return failure diff --git a/client/src/leap/soledad/client/interfaces.py b/client/src/leap/soledad/client/interfaces.py new file mode 100644 index 00000000..4f7b0779 --- /dev/null +++ b/client/src/leap/soledad/client/interfaces.py @@ -0,0 +1,362 @@ +# -*- coding: utf-8 -*- +# interfaces.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Interfaces used by the Soledad Client. +""" +from zope.interface import Interface, Attribute + + +class ILocalStorage(Interface): + """ + I implement core methods for the u1db local storage of documents and + indexes. + """ + local_db_path = Attribute( + "The path for the local database replica") + local_db_file_name = Attribute( + "The name of the local SQLCipher U1DB database file") + uuid = Attribute("The user uuid") + default_prefix = Attribute( + "Prefix for default values for path") + + def put_doc(self, doc): + """ + Update a document in the local encrypted database. + + :param doc: the document to update + :type doc: SoledadDocument + + :return: + a deferred that will fire with the new revision identifier for + the document + :rtype: Deferred + """ + + def delete_doc(self, doc): + """ + Delete a document from the local encrypted database. + + :param doc: the document to delete + :type doc: SoledadDocument + + :return: + a deferred that will fire with ... + :rtype: Deferred + """ + + def get_doc(self, doc_id, include_deleted=False): + """ + Retrieve a document from the local encrypted database. + + :param doc_id: the unique document identifier + :type doc_id: str + :param include_deleted: + if True, deleted documents will be returned with empty content; + otherwise asking for a deleted document will return None + :type include_deleted: bool + + :return: + A deferred that will fire with the document object, containing a + SoledadDocument, or None if it could not be found + :rtype: Deferred + """ + + def get_docs(self, doc_ids, check_for_conflicts=True, + include_deleted=False): + """ + Get the content for many documents. + + :param doc_ids: a list of document identifiers + :type doc_ids: list + :param check_for_conflicts: if set False, then the conflict check will + be skipped, and 'None' will be returned instead of True/False + :type check_for_conflicts: bool + + :return: + A deferred that will fire with an iterable giving the Document + object for each document id in matching doc_ids order. + :rtype: Deferred + """ + + def get_all_docs(self, include_deleted=False): + """ + Get the JSON content for all documents in the database. + + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted + documents will not be included in the results. + :return: + A deferred that will fire with (generation, [Document]): that is, + the current generation of the database, followed by a list of all + the documents in the database. + :rtype: Deferred + """ + + def create_doc(self, content, doc_id=None): + """ + Create a new document in the local encrypted database. + + :param content: the contents of the new document + :type content: dict + :param doc_id: an optional identifier specifying the document id + :type doc_id: str + + :return: + A deferred tht will fire with the new document (SoledadDocument + instance). + :rtype: Deferred + """ + + def create_doc_from_json(self, json, doc_id=None): + """ + Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param json: The JSON document string + :type json: str + :param doc_id: An optional identifier specifying the document id. + :type doc_id: + :return: + A deferred that will fire with the new document (A SoledadDocument + instance) + :rtype: Deferred + """ + + def create_index(self, index_name, *index_expressions): + """ + Create an named index, which can then be queried for future lookups. + Creating an index which already exists is not an error, and is cheap. + Creating an index which does not match the index_expressions of the + existing index is an error. + Creating an index will block until the expressions have been evaluated + and the index generated. + + :param index_name: A unique name which can be used as a key prefix + :type index_name: str + :param index_expressions: + index expressions defining the index information. + :type index_expressions: dict + + Examples: + + "fieldname", or "fieldname.subfieldname" to index alphabetically + sorted on the contents of a field. + + "number(fieldname, width)", "lower(fieldname)" + """ + + def delete_index(self, index_name): + """ + Remove a named index. + + :param index_name: The name of the index we are removing + :type index_name: str + """ + + def list_indexes(self): + """ + List the definitions of all known indexes. + + :return: A list of [('index-name', ['field', 'field2'])] definitions. + :rtype: Deferred + """ + + def get_from_index(self, index_name, *key_values): + """ + Return documents that match the keys supplied. + + You must supply exactly the same number of values as have been defined + in the index. It is possible to do a prefix match by using '*' to + indicate a wildcard match. You can only supply '*' to trailing entries, + (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) + It is also possible to append a '*' to the last supplied value (eg + 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: List of [Document] + :rtype: list + """ + + def get_count_from_index(self, index_name, *key_values): + """ + Return the count of the documents that match the keys and + values supplied. + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: count. + :rtype: int + """ + + def get_range_from_index(self, index_name, start_value, end_value): + """ + Return documents that fall within the specified range. + + Both ends of the range are inclusive. For both start_value and + end_value, one must supply exactly the same number of values as have + been defined in the index, or pass None. In case of a single column + index, a string is accepted as an alternative for a tuple with a single + value. It is possible to do a prefix match by using '*' to indicate + a wildcard match. You can only supply '*' to trailing entries, (eg + 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also + possible to append a '*' to the last supplied value (eg 'val*', '*', + '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param start_values: tuples of values that define the lower bound of + the range. eg, if you have an index with 3 fields then you would + have: (val1, val2, val3) + :type start_values: tuple + :param end_values: tuples of values that define the upper bound of the + range. eg, if you have an index with 3 fields then you would have: + (val1, val2, val3) + :type end_values: tuple + :return: A deferred that will fire with a list of [Document] + :rtype: Deferred + """ + + def get_index_keys(self, index_name): + """ + Return all keys under which documents are indexed in this index. + + :param index_name: The index to query + :type index_name: str + :return: + A deferred that will fire with a list of tuples of indexed keys. + :rtype: Deferred + """ + + def get_doc_conflicts(self, doc_id): + """ + Get the list of conflicts for the given document. + + :param doc_id: the document id + :type doc_id: str + + :return: + A deferred that will fire with a list of the document entries that + are conflicted. + :rtype: Deferred + """ + + def resolve_doc(self, doc, conflicted_doc_revs): + """ + Mark a document as no longer conflicted. + + :param doc: a document with the new content to be inserted. + :type doc: SoledadDocument + :param conflicted_doc_revs: + A deferred that will fire with a list of revisions that the new + content supersedes. + :type conflicted_doc_revs: list + """ + + +class ISyncableStorage(Interface): + """ + I implement methods to synchronize with a remote replica. + """ + replica_uid = Attribute("The uid of the local replica") + syncing = Attribute( + "Property, True if the syncer is syncing.") + token = Attribute("The authentication Token.") + + def sync(self, defer_decryption=True): + """ + Synchronize the local encrypted replica with a remote replica. + + This method blocks until a syncing lock is acquired, so there are no + attempts of concurrent syncs from the same client replica. + + :param url: the url of the target replica to sync with + :type url: str + + :param defer_decryption: + Whether to defer the decryption process using the intermediate + database. If False, decryption will be done inline. + :type defer_decryption: bool + + :return: + A deferred that will fire with the local generation before the + synchronisation was performed. + :rtype: str + """ + + def stop_sync(self): + """ + Stop the current syncing process. + """ + + +class ISecretsStorage(Interface): + """ + I implement methods needed for initializing and accessing secrets, that are + synced against the Shared Recovery Database. + """ + secrets_file_name = Attribute( + "The name of the file where the storage secrets will be stored") + + storage_secret = Attribute("") + remote_storage_secret = Attribute("") + shared_db = Attribute("The shared db object") + + # XXX this used internally from secrets, so it might be good to preserve + # as a public boundary with other components. + + # We should also probably document its interface. + secrets = Attribute("A SoledadSecrets object containing access to secrets") + + def init_shared_db(self, server_url, uuid, creds): + """ + Initialize the shared recovery database. + + :param server_url: + :type server_url: + :param uuid: + :type uuid: + :param creds: + :type creds: + """ + + def change_passphrase(self, new_passphrase): + """ + Change the passphrase that encrypts the storage secret. + + :param new_passphrase: The new passphrase. + :type new_passphrase: unicode + + :raise NoStorageSecret: Raised if there's no storage secret available. + """ + + # XXX not in use. Uncomment if we ever decide to allow + # multiple secrets. + # secret_id = Attribute("The id of the storage secret to be used") diff --git a/client/src/leap/soledad/client/pragmas.py b/client/src/leap/soledad/client/pragmas.py new file mode 100644 index 00000000..55397d10 --- /dev/null +++ b/client/src/leap/soledad/client/pragmas.py @@ -0,0 +1,379 @@ +# -*- coding: utf-8 -*- +# pragmas.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Different pragmas used in the initialization of the SQLCipher database. +""" +import logging +import string +import threading +import os + +from leap.soledad.common import soledad_assert + + +logger = logging.getLogger(__name__) + + +_db_init_lock = threading.Lock() + + +def set_init_pragmas(conn, opts=None, extra_queries=None): + """ + Set the initialization pragmas. + + This includes the crypto pragmas, and any other options that must + be passed early to sqlcipher db. + """ + soledad_assert(opts is not None) + extra_queries = [] if extra_queries is None else extra_queries + with _db_init_lock: + # only one execution path should initialize the db + _set_init_pragmas(conn, opts, extra_queries) + + +def _set_init_pragmas(conn, opts, extra_queries): + + sync_off = os.environ.get('LEAP_SQLITE_NOSYNC') + memstore = os.environ.get('LEAP_SQLITE_MEMSTORE') + nowal = os.environ.get('LEAP_SQLITE_NOWAL') + + set_crypto_pragmas(conn, opts) + + if not nowal: + set_write_ahead_logging(conn) + if sync_off: + set_synchronous_off(conn) + else: + set_synchronous_normal(conn) + if memstore: + set_mem_temp_store(conn) + + for query in extra_queries: + conn.cursor().execute(query) + + +def set_crypto_pragmas(db_handle, sqlcipher_opts): + """ + Set cryptographic params (key, cipher, KDF number of iterations and + cipher page size). + + :param db_handle: + :type db_handle: + :param sqlcipher_opts: options for the SQLCipherDatabase + :type sqlcipher_opts: SQLCipherOpts instance + """ + # XXX assert CryptoOptions + opts = sqlcipher_opts + _set_key(db_handle, opts.key, opts.is_raw_key) + _set_cipher(db_handle, opts.cipher) + _set_kdf_iter(db_handle, opts.kdf_iter) + _set_cipher_page_size(db_handle, opts.cipher_page_size) + + +def _set_key(db_handle, key, is_raw_key): + """ + Set the ``key`` for use with the database. + + The process of creating a new, encrypted database is called 'keying' + the database. SQLCipher uses just-in-time key derivation at the point + it is first needed for an operation. This means that the key (and any + options) must be set before the first operation on the database. As + soon as the database is touched (e.g. SELECT, CREATE TABLE, UPDATE, + etc.) and pages need to be read or written, the key is prepared for + use. + + Implementation Notes: + + * PRAGMA key should generally be called as the first operation on a + database. + + :param key: The key for use with the database. + :type key: str + :param is_raw_key: + Whether C{key} is a raw 64-char hex string or a passphrase that should + be hashed to obtain the encyrption key. + :type is_raw_key: bool + """ + if is_raw_key: + _set_key_raw(db_handle, key) + else: + _set_key_passphrase(db_handle, key) + + +def _set_key_passphrase(db_handle, passphrase): + """ + Set a passphrase for encryption key derivation. + + The key itself can be a passphrase, which is converted to a key using + PBKDF2 key derivation. The result is used as the encryption key for + the database. By using this method, there is no way to alter the KDF; + if you want to do so you should use a raw key instead and derive the + key using your own KDF. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param passphrase: The passphrase used to derive the encryption key. + :type passphrase: str + """ + db_handle.cursor().execute("PRAGMA key = '%s'" % passphrase) + + +def _set_key_raw(db_handle, key): + """ + Set a raw hexadecimal encryption key. + + It is possible to specify an exact byte sequence using a blob literal. + With this method, it is the calling application's responsibility to + ensure that the data provided is a 64 character hex string, which will + be converted directly to 32 bytes (256 bits) of key data. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param key: A 64 character hex string. + :type key: str + """ + if not all(c in string.hexdigits for c in key): + raise NotAnHexString(key) + db_handle.cursor().execute('PRAGMA key = "x\'%s"' % key) + + +def _set_cipher(db_handle, cipher='aes-256-cbc'): + """ + Set the cipher and mode to use for symmetric encryption. + + SQLCipher uses aes-256-cbc as the default cipher and mode of + operation. It is possible to change this, though not generally + recommended, using PRAGMA cipher. + + SQLCipher makes direct use of libssl, so all cipher options available + to libssl are also available for use with SQLCipher. See `man enc` for + OpenSSL's supported ciphers. + + Implementation Notes: + + * PRAGMA cipher must be called after PRAGMA key and before the first + actual database operation or it will have no effect. + + * If a non-default value is used PRAGMA cipher to create a database, + it must also be called every time that database is opened. + + * SQLCipher does not implement its own encryption. Instead it uses the + widely available and peer-reviewed OpenSSL libcrypto for all + cryptographic functions. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param cipher: The cipher and mode to use. + :type cipher: str + """ + db_handle.cursor().execute("PRAGMA cipher = '%s'" % cipher) + + +def _set_kdf_iter(db_handle, kdf_iter=4000): + """ + Set the number of iterations for the key derivation function. + + SQLCipher uses PBKDF2 key derivation to strengthen the key and make it + resistent to brute force and dictionary attacks. The default + configuration uses 4000 PBKDF2 iterations (effectively 16,000 SHA1 + operations). PRAGMA kdf_iter can be used to increase or decrease the + number of iterations used. + + Implementation Notes: + + * PRAGMA kdf_iter must be called after PRAGMA key and before the first + actual database operation or it will have no effect. + + * If a non-default value is used PRAGMA kdf_iter to create a database, + it must also be called every time that database is opened. + + * It is not recommended to reduce the number of iterations if a + passphrase is in use. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param kdf_iter: The number of iterations to use. + :type kdf_iter: int + """ + db_handle.cursor().execute("PRAGMA kdf_iter = '%d'" % kdf_iter) + + +def _set_cipher_page_size(db_handle, cipher_page_size=1024): + """ + Set the page size of the encrypted database. + + SQLCipher 2 introduced the new PRAGMA cipher_page_size that can be + used to adjust the page size for the encrypted database. The default + page size is 1024 bytes, but it can be desirable for some applications + to use a larger page size for increased performance. For instance, + some recent testing shows that increasing the page size can noticeably + improve performance (5-30%) for certain queries that manipulate a + large number of pages (e.g. selects without an index, large inserts in + a transaction, big deletes). + + To adjust the page size, call the pragma immediately after setting the + key for the first time and each subsequent time that you open the + database. + + Implementation Notes: + + * PRAGMA cipher_page_size must be called after PRAGMA key and before + the first actual database operation or it will have no effect. + + * If a non-default value is used PRAGMA cipher_page_size to create a + database, it must also be called every time that database is opened. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param cipher_page_size: The page size. + :type cipher_page_size: int + """ + db_handle.cursor().execute( + "PRAGMA cipher_page_size = '%d'" % cipher_page_size) + + +# XXX UNUSED ? +def set_rekey(db_handle, new_key, is_raw_key): + """ + Change the key of an existing encrypted database. + + To change the key on an existing encrypted database, it must first be + unlocked with the current encryption key. Once the database is + readable and writeable, PRAGMA rekey can be used to re-encrypt every + page in the database with a new key. + + * PRAGMA rekey must be called after PRAGMA key. It can be called at any + time once the database is readable. + + * PRAGMA rekey can not be used to encrypted a standard SQLite + database! It is only useful for changing the key on an existing + database. + + * Previous versions of SQLCipher provided a PRAGMA rekey_cipher and + code>PRAGMA rekey_kdf_iter. These are deprecated and should not be + used. Instead, use sqlcipher_export(). + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param new_key: The new key. + :type new_key: str + :param is_raw_key: Whether C{password} is a raw 64-char hex string or a + passphrase that should be hashed to obtain the encyrption + key. + :type is_raw_key: bool + """ + if is_raw_key: + _set_rekey_raw(db_handle, new_key) + else: + _set_rekey_passphrase(db_handle, new_key) + + +def _set_rekey_passphrase(db_handle, passphrase): + """ + Change the passphrase for encryption key derivation. + + The key itself can be a passphrase, which is converted to a key using + PBKDF2 key derivation. The result is used as the encryption key for + the database. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param passphrase: The passphrase used to derive the encryption key. + :type passphrase: str + """ + db_handle.cursor().execute("PRAGMA rekey = '%s'" % passphrase) + + +def _set_rekey_raw(db_handle, key): + """ + Change the raw hexadecimal encryption key. + + It is possible to specify an exact byte sequence using a blob literal. + With this method, it is the calling application's responsibility to + ensure that the data provided is a 64 character hex string, which will + be converted directly to 32 bytes (256 bits) of key data. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param key: A 64 character hex string. + :type key: str + """ + if not all(c in string.hexdigits for c in key): + raise NotAnHexString(key) + db_handle.cursor().execute('PRAGMA rekey = "x\'%s"' % key) + + +def set_synchronous_off(db_handle): + """ + Change the setting of the "synchronous" flag to OFF. + """ + logger.debug("SQLCIPHER: SETTING SYNCHRONOUS OFF") + db_handle.cursor().execute('PRAGMA synchronous=OFF') + + +def set_synchronous_normal(db_handle): + """ + Change the setting of the "synchronous" flag to NORMAL. + """ + logger.debug("SQLCIPHER: SETTING SYNCHRONOUS NORMAL") + db_handle.cursor().execute('PRAGMA synchronous=NORMAL') + + +def set_mem_temp_store(db_handle): + """ + Use a in-memory store for temporary tables. + """ + logger.debug("SQLCIPHER: SETTING TEMP_STORE MEMORY") + db_handle.cursor().execute('PRAGMA temp_store=MEMORY') + + +def set_write_ahead_logging(db_handle): + """ + Enable write-ahead logging, and set the autocheckpoint to 50 pages. + + Setting the autocheckpoint to a small value, we make the reads not + suffer too much performance degradation. + + From the sqlite docs: + + "There is a tradeoff between average read performance and average write + performance. To maximize the read performance, one wants to keep the + WAL as small as possible and hence run checkpoints frequently, perhaps + as often as every COMMIT. To maximize write performance, one wants to + amortize the cost of each checkpoint over as many writes as possible, + meaning that one wants to run checkpoints infrequently and let the WAL + grow as large as possible before each checkpoint. The decision of how + often to run checkpoints may therefore vary from one application to + another depending on the relative read and write performance + requirements of the application. The default strategy is to run a + checkpoint once the WAL reaches 1000 pages" + """ + logger.debug("SQLCIPHER: SETTING WRITE-AHEAD LOGGING") + db_handle.cursor().execute('PRAGMA journal_mode=WAL') + + # The optimum value can still use a little bit of tuning, but we favor + # small sizes of the WAL file to get fast reads, since we assume that + # the writes will be quick enough to not block too much. + + db_handle.cursor().execute('PRAGMA wal_autocheckpoint=50') + + +class NotAnHexString(Exception): + """ + Raised when trying to (raw) key the database with a non-hex string. + """ + pass diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py new file mode 100644 index 00000000..e89e21aa --- /dev/null +++ b/client/src/leap/soledad/client/secrets.py @@ -0,0 +1,787 @@ +# -*- coding: utf-8 -*- +# secrets.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +Soledad secrets handling. +""" + + +import os +import scrypt +import hmac +import logging +import binascii +import errno + + +from hashlib import sha256 +import simplejson as json + + +from leap.soledad.common import soledad_assert +from leap.soledad.common import soledad_assert_type +from leap.soledad.common import document +from leap.soledad.common import errors +from leap.soledad.common import crypto +from leap.soledad.client import events + + +logger = logging.getLogger(name=__name__) + + +# +# Exceptions +# + + +class SecretsException(Exception): + """ + Generic exception type raised by this module. + """ + + +class NoStorageSecret(SecretsException): + """ + Raised when trying to use a storage secret but none is available. + """ + pass + + +class PassphraseTooShort(SecretsException): + """ + Raised when trying to change the passphrase but the provided passphrase is + too short. + """ + + +class BootstrapSequenceError(SecretsException): + """ + Raised when an attempt to generate a secret and store it in a recovery + document on server failed. + """ + + +# +# Secrets handler +# + +class SoledadSecrets(object): + """ + Soledad secrets handler. + + The first C{self.REMOTE_STORAGE_SECRET_LENGTH} bytes of the storage + secret are used for remote storage encryption. We use the next + C{self.LOCAL_STORAGE_SECRET} bytes to derive a key for local storage. + From these bytes, the first C{self.SALT_LENGTH} bytes are used as the + salt and the rest as the password for the scrypt hashing. + """ + + LOCAL_STORAGE_SECRET_LENGTH = 512 + """ + The length, in bytes, of the secret used to derive a passphrase for the + SQLCipher database. + """ + + REMOTE_STORAGE_SECRET_LENGTH = 512 + """ + The length, in bytes, of the secret used to derive an encryption key and a + MAC auth key for remote storage. + """ + + SALT_LENGTH = 64 + """ + The length, in bytes, of the salt used to derive the key for the storage + secret encryption. + """ + + GEN_SECRET_LENGTH = LOCAL_STORAGE_SECRET_LENGTH \ + + REMOTE_STORAGE_SECRET_LENGTH \ + + SALT_LENGTH # for sync db + """ + The length, in bytes, of the secret to be generated. This includes local + and remote secrets, and the salt for deriving the sync db secret. + """ + + MINIMUM_PASSPHRASE_LENGTH = 6 + """ + The minimum length, in bytes, for a passphrase. The passphrase length is + only checked when the user changes her passphrase, not when she + instantiates Soledad. + """ + + IV_SEPARATOR = ":" + """ + A separator used for storing the encryption initial value prepended to the + ciphertext. + """ + + UUID_KEY = 'uuid' + STORAGE_SECRETS_KEY = 'storage_secrets' + ACTIVE_SECRET_KEY = 'active_secret' + SECRET_KEY = 'secret' + CIPHER_KEY = 'cipher' + LENGTH_KEY = 'length' + KDF_KEY = 'kdf' + KDF_SALT_KEY = 'kdf_salt' + KDF_LENGTH_KEY = 'kdf_length' + KDF_SCRYPT = 'scrypt' + CIPHER_AES256 = 'aes256' + """ + Keys used to access storage secrets in recovery documents. + """ + + def __init__(self, uuid, passphrase, secrets_path, shared_db, crypto): + """ + Initialize the secrets manager. + + :param uuid: User's unique id. + :type uuid: str + :param passphrase: The passphrase for locking and unlocking encryption + secrets for local and remote storage. + :type passphrase: unicode + :param secrets_path: Path for storing encrypted key used for + symmetric encryption. + :type secrets_path: str + :param shared_db: The shared database that stores user secrets. + :type shared_db: leap.soledad.client.shared_db.SoledadSharedDatabase + :param crypto: A soledad crypto object. + :type crypto: SoledadCrypto + """ + # XXX removed since not in use + # We will pick the first secret available. + # param secret_id: The id of the storage secret to be used. + + self._uuid = uuid + self._passphrase = passphrase + self._secrets_path = secrets_path + self._shared_db = shared_db + self._crypto = crypto + self._secrets = {} + + self._secret_id = None + + def bootstrap(self): + """ + Bootstrap secrets. + + Soledad secrets bootstrap is the following sequence of stages: + + * stage 1 - local secret loading: + - if secrets exist locally, load them. + * stage 2 - remote secret loading: + - else, if secrets exist in server, download them. + * stage 3 - secret generation: + - else, generate a new secret and store in server. + + This method decides which bootstrap stages have already been performed + and performs the missing ones in order. + + :raise BootstrapSequenceError: Raised when the secret generation and + storage on server sequence has failed for some reason. + """ + # STAGE 1 - verify if secrets exist locally + if not self._has_secret(): # try to load from local storage. + + # STAGE 2 - there are no secrets in local storage, so try to fetch + # encrypted secrets from server. + logger.info( + 'Trying to fetch cryptographic secrets from shared recovery ' + 'database...') + + # --- start of atomic operation in shared db --- + + # obtain lock on shared db + token = timeout = None + try: + token, timeout = self._shared_db.lock() + except errors.AlreadyLockedError: + raise BootstrapSequenceError('Database is already locked.') + except errors.LockTimedOutError: + raise BootstrapSequenceError('Lock operation timed out.') + + self._get_or_gen_crypto_secrets() + + # release the lock on shared db + try: + self._shared_db.unlock(token) + self._shared_db.close() + except errors.NotLockedError: + # for some reason the lock expired. Despite that, secret + # loading or generation/storage must have been executed + # successfully, so we pass. + pass + except errors.InvalidTokenError: + # here, our lock has not only expired but also some other + # client application has obtained a new lock and is currently + # doing its thing in the shared database. Using the same + # reasoning as above, we assume everything went smooth and + # pass. + pass + except Exception as e: + logger.error("Unhandled exception when unlocking shared " + "database.") + logger.exception(e) + + # --- end of atomic operation in shared db --- + + def _has_secret(self): + """ + Return whether there is a storage secret available for use or not. + + :return: Whether there's a storage secret for symmetric encryption. + :rtype: bool + """ + logger.info("Checking if there's a secret in local storage...") + if (self._secret_id is None or self._secret_id not in self._secrets) \ + and os.path.isfile(self._secrets_path): + try: + self._load_secrets() # try to load from disk + except IOError as e: + logger.warning( + 'IOError while loading secrets from disk: %s' % str(e)) + + if self.storage_secret is not None: + logger.info("Found a secret in local storage.") + return True + + logger.info("Could not find a secret in local storage.") + return False + + def _load_secrets(self): + """ + Load storage secrets from local file. + """ + # read storage secrets from file + content = None + with open(self._secrets_path, 'r') as f: + content = json.loads(f.read()) + _, mac, active_secret = self._import_recovery_document(content) + # choose first secret if no secret_id was given + if self._secret_id is None: + if active_secret is None: + self.set_secret_id(self._secrets.items()[0][0]) + else: + self.set_secret_id(active_secret) + # enlarge secret if needed + enlarged = False + if len(self._secrets[self._secret_id]) < self.GEN_SECRET_LENGTH: + gen_len = self.GEN_SECRET_LENGTH \ + - len(self._secrets[self._secret_id]) + new_piece = os.urandom(gen_len) + self._secrets[self._secret_id] += new_piece + enlarged = True + # store and save in shared db if needed + if not mac or enlarged: + self._store_secrets() + self._put_secrets_in_shared_db() + + def _get_or_gen_crypto_secrets(self): + """ + Retrieves or generates the crypto secrets. + + :raises BootstrapSequenceError: Raised when unable to store secrets in + shared database. + """ + if self._shared_db.syncable: + doc = self._get_secrets_from_shared_db() + else: + doc = None + + if doc is not None: + logger.info( + 'Found cryptographic secrets in shared recovery ' + 'database.') + _, mac, active_secret = self._import_recovery_document(doc.content) + if mac is False: + self.put_secrets_in_shared_db() + self._store_secrets() # save new secrets in local file + if self._secret_id is None: + if active_secret is None: + self.set_secret_id(self._secrets.items()[0][0]) + else: + self.set_secret_id(active_secret) + else: + # STAGE 3 - there are no secrets in server also, so + # generate a secret and store it in remote db. + logger.info( + 'No cryptographic secrets found, creating new ' + ' secrets...') + self.set_secret_id(self._gen_secret()) + + if self._shared_db.syncable: + try: + self._put_secrets_in_shared_db() + except Exception as ex: + # storing generated secret in shared db failed for + # some reason, so we erase the generated secret and + # raise. + try: + os.unlink(self._secrets_path) + except OSError as e: + if e.errno != errno.ENOENT: + # no such file or directory + logger.exception(e) + logger.exception(ex) + raise BootstrapSequenceError( + 'Could not store generated secret in the shared ' + 'database, bailing out...') + + # + # Shared DB related methods + # + + def _shared_db_doc_id(self): + """ + Calculate the doc_id of the document in the shared db that stores key + material. + + :return: the hash + :rtype: str + """ + return sha256( + '%s%s' % + (self._passphrase_as_string(), self._uuid)).hexdigest() + + def _export_recovery_document(self): + """ + Export the storage secrets. + + A recovery document has the following structure: + + { + 'storage_secrets': { + '<storage_secret id>': { + 'kdf': 'scrypt', + 'kdf_salt': '<b64 repr of salt>' + 'kdf_length': <key length> + 'cipher': 'aes256', + 'length': <secret length>, + 'secret': '<encrypted storage_secret>', + }, + }, + 'active_secret': '<secret_id>', + 'kdf': 'scrypt', + 'kdf_salt': '<b64 repr of salt>', + 'kdf_length: <key length>, + '_mac_method': 'hmac', + '_mac': '<mac>' + } + + Note that multiple storage secrets might be stored in one recovery + document. This method will also calculate a MAC of a string + representation of the secrets dictionary. + + :return: The recovery document. + :rtype: dict + """ + # create salt and key for calculating MAC + salt = os.urandom(self.SALT_LENGTH) + key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) + # encrypt secrets + encrypted_secrets = {} + for secret_id in self._secrets: + encrypted_secrets[secret_id] = self._encrypt_storage_secret( + self._secrets[secret_id]) + # create the recovery document + data = { + self.STORAGE_SECRETS_KEY: encrypted_secrets, + self.ACTIVE_SECRET_KEY: self._secret_id, + self.KDF_KEY: self.KDF_SCRYPT, + self.KDF_SALT_KEY: binascii.b2a_base64(salt), + self.KDF_LENGTH_KEY: len(key), + crypto.MAC_METHOD_KEY: crypto.MacMethods.HMAC, + crypto.MAC_KEY: hmac.new( + key, + json.dumps(encrypted_secrets, sort_keys=True), + sha256).hexdigest(), + } + return data + + def _import_recovery_document(self, data): + """ + Import storage secrets for symmetric encryption and uuid (if present) + from a recovery document. + + Note that this method does not store the imported data on disk. For + that, use C{self._store_secrets()}. + + :param data: The recovery document. + :type data: dict + + :return: A tuple containing the number of imported secrets, whether + there was MAC information available for authenticating, and + the secret_id of the last active secret. + :rtype: (int, bool) + """ + soledad_assert(self.STORAGE_SECRETS_KEY in data) + # check mac of the recovery document + mac = None + if crypto.MAC_KEY in data: + soledad_assert(data[crypto.MAC_KEY] is not None) + soledad_assert(crypto.MAC_METHOD_KEY in data) + soledad_assert(self.KDF_KEY in data) + soledad_assert(self.KDF_SALT_KEY in data) + soledad_assert(self.KDF_LENGTH_KEY in data) + if data[crypto.MAC_METHOD_KEY] == crypto.MacMethods.HMAC: + key = scrypt.hash( + self._passphrase_as_string(), + binascii.a2b_base64(data[self.KDF_SALT_KEY]), + buflen=32) + mac = hmac.new( + key, + json.dumps( + data[self.STORAGE_SECRETS_KEY], sort_keys=True), + sha256).hexdigest() + else: + raise crypto.UnknownMacMethodError('Unknown MAC method: %s.' % + data[crypto.MAC_METHOD_KEY]) + if mac != data[crypto.MAC_KEY]: + raise crypto.WrongMacError('Could not authenticate recovery document\'s ' + 'contents.') + # include secrets in the secret pool. + secret_count = 0 + secrets = data[self.STORAGE_SECRETS_KEY].items() + active_secret = None + # XXX remove check for existence of key (included for backwards + # compatibility) + if self.ACTIVE_SECRET_KEY in data: + active_secret = data[self.ACTIVE_SECRET_KEY] + for secret_id, encrypted_secret in secrets: + if secret_id not in self._secrets: + try: + self._secrets[secret_id] = \ + self._decrypt_storage_secret(encrypted_secret) + secret_count += 1 + except SecretsException as e: + logger.error("Failed to decrypt storage secret: %s" + % str(e)) + return secret_count, mac, active_secret + + def _get_secrets_from_shared_db(self): + """ + Retrieve the document with encrypted key material from the shared + database. + + :return: a document with encrypted key material in its contents + :rtype: document.SoledadDocument + """ + events.emit(events.SOLEDAD_DOWNLOADING_KEYS, self._uuid) + db = self._shared_db + if not db: + logger.warning('No shared db found') + return + doc = db.get_doc(self._shared_db_doc_id()) + events.emit(events.SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid) + return doc + + def _put_secrets_in_shared_db(self): + """ + Assert local keys are the same as shared db's ones. + + Try to fetch keys from shared recovery database. If they already exist + in the remote db, assert that that data is the same as local data. + Otherwise, upload keys to shared recovery database. + """ + soledad_assert( + self._has_secret(), + 'Tried to send keys to server but they don\'t exist in local ' + 'storage.') + # try to get secrets doc from server, otherwise create it + doc = self._get_secrets_from_shared_db() + if doc is None: + doc = document.SoledadDocument( + doc_id=self._shared_db_doc_id()) + # fill doc with encrypted secrets + doc.content = self._export_recovery_document() + # upload secrets to server + events.emit(events.SOLEDAD_UPLOADING_KEYS, self._uuid) + db = self._shared_db + if not db: + logger.warning('No shared db found') + return + db.put_doc(doc) + events.emit(events.SOLEDAD_DONE_UPLOADING_KEYS, self._uuid) + + # + # Management of secret for symmetric encryption. + # + + def _decrypt_storage_secret(self, encrypted_secret_dict): + """ + Decrypt the storage secret. + + Storage secret is encrypted before being stored. This method decrypts + and returns the decrypted storage secret. + + :param encrypted_secret_dict: The encrypted storage secret. + :type encrypted_secret_dict: dict + + :return: The decrypted storage secret. + :rtype: str + + :raise SecretsException: Raised in case the decryption of the storage + secret fails for some reason. + """ + # calculate the encryption key + if encrypted_secret_dict[self.KDF_KEY] != self.KDF_SCRYPT: + raise SecretsException("Unknown KDF in stored secret.") + key = scrypt.hash( + self._passphrase_as_string(), + # the salt is stored base64 encoded + binascii.a2b_base64( + encrypted_secret_dict[self.KDF_SALT_KEY]), + buflen=32, # we need a key with 256 bits (32 bytes). + ) + if encrypted_secret_dict[self.KDF_LENGTH_KEY] != len(key): + raise SecretsException("Wrong length of decryption key.") + if encrypted_secret_dict[self.CIPHER_KEY] != self.CIPHER_AES256: + raise SecretsException("Unknown cipher in stored secret.") + # recover the initial value and ciphertext + iv, ciphertext = encrypted_secret_dict[self.SECRET_KEY].split( + self.IV_SEPARATOR, 1) + ciphertext = binascii.a2b_base64(ciphertext) + decrypted_secret = self._crypto.decrypt_sym(ciphertext, key, iv=iv) + if encrypted_secret_dict[self.LENGTH_KEY] != len(decrypted_secret): + raise SecretsException("Wrong length of decrypted secret.") + return decrypted_secret + + def _encrypt_storage_secret(self, decrypted_secret): + """ + Encrypt the storage secret. + + An encrypted secret has the following structure: + + { + '<secret_id>': { + 'kdf': 'scrypt', + 'kdf_salt': '<b64 repr of salt>' + 'kdf_length': <key length> + 'cipher': 'aes256', + 'length': <secret length>, + 'secret': '<encrypted b64 repr of storage_secret>', + } + } + + :param decrypted_secret: The decrypted storage secret. + :type decrypted_secret: str + + :return: The encrypted storage secret. + :rtype: dict + """ + # generate random salt + salt = os.urandom(self.SALT_LENGTH) + # get a 256-bit key + key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) + iv, ciphertext = self._crypto.encrypt_sym(decrypted_secret, key) + encrypted_secret_dict = { + # leap.soledad.crypto submodule uses AES256 for symmetric + # encryption. + self.KDF_KEY: self.KDF_SCRYPT, + self.KDF_SALT_KEY: binascii.b2a_base64(salt), + self.KDF_LENGTH_KEY: len(key), + self.CIPHER_KEY: self.CIPHER_AES256, + self.LENGTH_KEY: len(decrypted_secret), + self.SECRET_KEY: '%s%s%s' % ( + str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), + } + return encrypted_secret_dict + + @property + def storage_secret(self): + """ + Return the storage secret. + + :return: The decrypted storage secret. + :rtype: str + """ + return self._secrets.get(self._secret_id) + + def set_secret_id(self, secret_id): + """ + Define the id of the storage secret to be used. + + This method will also replace the secret in the crypto object. + + :param secret_id: The id of the storage secret to be used. + :type secret_id: str + """ + self._secret_id = secret_id + + def _gen_secret(self): + """ + Generate a secret for symmetric encryption and store in a local + encrypted file. + + This method emits the following events.signals: + + * SOLEDAD_CREATING_KEYS + * SOLEDAD_DONE_CREATING_KEYS + + :return: The id of the generated secret. + :rtype: str + """ + events.emit(events.SOLEDAD_CREATING_KEYS, self._uuid) + # generate random secret + secret = os.urandom(self.GEN_SECRET_LENGTH) + secret_id = sha256(secret).hexdigest() + self._secrets[secret_id] = secret + self._store_secrets() + events.emit(events.SOLEDAD_DONE_CREATING_KEYS, self._uuid) + return secret_id + + def _store_secrets(self): + """ + Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}. + """ + with open(self._secrets_path, 'w') as f: + f.write( + json.dumps( + self._export_recovery_document())) + + def change_passphrase(self, new_passphrase): + """ + Change the passphrase that encrypts the storage secret. + + :param new_passphrase: The new passphrase. + :type new_passphrase: unicode + + :raise NoStorageSecret: Raised if there's no storage secret available. + """ + # TODO: maybe we want to add more checks to guarantee passphrase is + # reasonable? + soledad_assert_type(new_passphrase, unicode) + if len(new_passphrase) < self.MINIMUM_PASSPHRASE_LENGTH: + raise PassphraseTooShort( + 'Passphrase must be at least %d characters long!' % + self.MINIMUM_PASSPHRASE_LENGTH) + # ensure there's a secret for which the passphrase will be changed. + if not self._has_secret(): + raise NoStorageSecret() + self._passphrase = new_passphrase + self._store_secrets() + self._put_secrets_in_shared_db() + + # + # Setters and getters + # + + @property + def secret_id(self): + return self._secret_id + + def _get_secrets_path(self): + return self._secrets_path + + def _set_secrets_path(self, secrets_path): + self._secrets_path = secrets_path + + secrets_path = property( + _get_secrets_path, + _set_secrets_path, + doc='The path for the file containing the encrypted symmetric secret.') + + @property + def passphrase(self): + """ + Return the passphrase for locking and unlocking encryption secrets for + local and remote storage. + """ + return self._passphrase + + def _passphrase_as_string(self): + return self._passphrase.encode('utf-8') + + # + # remote storage secret + # + + @property + def remote_storage_secret(self): + """ + Return the secret for remote storage. + """ + key_start = 0 + key_end = self.REMOTE_STORAGE_SECRET_LENGTH + return self.storage_secret[key_start:key_end] + + # + # local storage key + # + + def _get_local_storage_secret(self): + """ + Return the local storage secret. + + :return: The local storage secret. + :rtype: str + """ + secret_len = self.REMOTE_STORAGE_SECRET_LENGTH + lsecret_len = self.LOCAL_STORAGE_SECRET_LENGTH + pwd_start = secret_len + self.SALT_LENGTH + pwd_end = secret_len + lsecret_len + return self.storage_secret[pwd_start:pwd_end] + + def _get_local_storage_salt(self): + """ + Return the local storage salt. + + :return: The local storage salt. + :rtype: str + """ + salt_start = self.REMOTE_STORAGE_SECRET_LENGTH + salt_end = salt_start + self.SALT_LENGTH + return self.storage_secret[salt_start:salt_end] + + def get_local_storage_key(self): + """ + Return the local storage key derived from the local storage secret. + + :return: The key for protecting the local database. + :rtype: str + """ + return scrypt.hash( + password=self._get_local_storage_secret(), + salt=self._get_local_storage_salt(), + buflen=32, # we need a key with 256 bits (32 bytes) + ) + + # + # sync db key + # + + def _get_sync_db_salt(self): + """ + Return the salt for sync db. + """ + salt_start = self.LOCAL_STORAGE_SECRET_LENGTH \ + + self.REMOTE_STORAGE_SECRET_LENGTH + salt_end = salt_start + self.SALT_LENGTH + return self.storage_secret[salt_start:salt_end] + + def get_sync_db_key(self): + """ + Return the key for protecting the sync database. + + :return: The key for protecting the sync database. + :rtype: str + """ + return scrypt.hash( + password=self._get_local_storage_secret(), + salt=self._get_sync_db_salt(), + buflen=32, # we need a key with 256 bits (32 bytes) + ) diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index 52e51c6f..f1a2642e 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -14,19 +14,11 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. - - """ A shared database for storing/retrieving encrypted key material. """ - -import simplejson as json - - from u1db.remote import http_database - -from leap.soledad.common import SHARED_DB_LOCK_DOC_ID_PREFIX from leap.soledad.client.auth import TokenBasedAuth @@ -34,6 +26,9 @@ from leap.soledad.client.auth import TokenBasedAuth # Soledad shared database # ---------------------------------------------------------------------------- +# TODO could have a hierarchy of soledad exceptions. + + class NoTokenForAuth(Exception): """ No token was found for token-based authentication. @@ -46,6 +41,12 @@ class Unauthorized(Exception): """ +class ImproperlyConfiguredError(Exception): + """ + Wrong parameters in the database configuration. + """ + + class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): """ This is a shared recovery database that enables users to store their @@ -54,6 +55,10 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): # TODO: prevent client from messing with the shared DB. # TODO: define and document API. + # If syncable is False, the database will not attempt to sync against + # a remote replica. Default is True. + syncable = True + # # Token auth methods. # @@ -90,9 +95,7 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): # @staticmethod - def open_database(url, uuid, create, creds=None): - # TODO: users should not be able to create the shared database, so we - # have to remove this from here in the future. + def open_database(url, uuid, creds=None, syncable=True): """ Open a Soledad shared database. @@ -100,17 +103,23 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :type url: str :param uuid: The user's unique id. :type uuid: str - :param create: Should the database be created if it does not already - exist? - :type create: bool - :param token: An authentication token for accessing the shared db. - :type token: str + :param creds: A tuple containing the authentication method and + credentials. + :type creds: tuple + :param syncable: + If syncable is False, the database will not attempt to sync against + a remote replica. + :type syncable: bool :return: The shared database in the given url. :rtype: SoledadSharedDatabase """ + # XXX fix below, doesn't work with tests. + #if syncable and not url.startswith('https://'): + # raise ImproperlyConfiguredError( + # "Remote soledad server must be an https URI") db = SoledadSharedDatabase(url, uuid, creds=creds) - db.open(create) + db.syncable = syncable return db @staticmethod @@ -153,9 +162,12 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :raise HTTPError: Raised if any HTTP error occurs. """ - res, headers = self._request_json('PUT', ['lock', self._uuid], - body={}) - return res['token'], res['timeout'] + if self.syncable: + res, headers = self._request_json( + 'PUT', ['lock', self._uuid], body={}) + return res['token'], res['timeout'] + else: + return None, None def unlock(self, token): """ @@ -166,5 +178,6 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :raise HTTPError: """ - res, headers = self._request_json('DELETE', ['lock', self._uuid], - params={'token': token}) + if self.syncable: + _, _ = self._request_json( + 'DELETE', ['lock', self._uuid], params={'token': token}) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index fded2119..b2025130 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -42,261 +42,120 @@ SQLCipher 1.1 databases, we do not implement them as all SQLCipher databases handled by Soledad should be created by SQLCipher >= 2.0. """ import logging -import multiprocessing import os -import sqlite3 -import string import threading -import time import json +import u1db + +from u1db import errors as u1db_errors +from u1db.backends import sqlite_backend from hashlib import sha256 from contextlib import contextmanager from collections import defaultdict +from functools import partial -from pysqlcipher import dbapi2 -from u1db.backends import sqlite_backend -from u1db import errors as u1db_errors -from taskthread import TimerTask +from pysqlcipher import dbapi2 as sqlcipher_dbapi2 + +from twisted.internet import reactor +from twisted.internet.threads import deferToThreadPool +from twisted.python.threadpool import ThreadPool +from twisted.enterprise import adbapi -from leap.soledad.client.crypto import SyncEncrypterPool, SyncDecrypterPool -from leap.soledad.client.target import SoledadSyncTarget -from leap.soledad.client.target import PendingReceivedDocsSyncError +from leap.soledad.client import encdecpool +from leap.soledad.client.http_target import SoledadHTTPSyncTarget from leap.soledad.client.sync import SoledadSynchronizer + +from leap.soledad.client import pragmas +from leap.soledad.common import soledad_assert from leap.soledad.common.document import SoledadDocument logger = logging.getLogger(__name__) -# Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2 -sqlite_backend.dbapi2 = dbapi2 - -# It seems that, as long as we are not using old sqlite versions, serialized -# mode is enabled by default at compile time. So accessing db connections from -# different threads should be safe, as long as no attempt is made to use them -# from multiple threads with no locking. -# See https://sqlite.org/threadsafe.html -# and http://bugs.python.org/issue16509 - -SQLITE_CHECK_SAME_THREAD = False - -# We set isolation_level to None to setup autocommit mode. -# See: http://docs.python.org/2/library/sqlite3.html#controlling-transactions -# This avoids problems with sequential operations using the same soledad object -# trying to open new transactions -# (The error was: -# OperationalError:cannot start a transaction within a transaction.) -SQLITE_ISOLATION_LEVEL = None - - -def open(path, password, create=True, document_factory=None, crypto=None, - raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, - cipher_page_size=1024, defer_encryption=False): - """Open a database at the given location. - - Will raise u1db.errors.DatabaseDoesNotExist if create=False and the - database does not already exist. - - :param path: The filesystem path for the database to open. - :type path: str - :param create: True/False, should the database be created if it doesn't - already exist? - :param create: bool - :param document_factory: A function that will be called with the same - parameters as Document.__init__. - :type document_factory: callable - :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. - :type crypto: soledad.crypto.SoledadCrypto - :param raw_key: Whether C{password} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. - :type raw_key: bool - :param cipher: The cipher and mode to use. - :type cipher: str - :param kdf_iter: The number of iterations to use. - :type kdf_iter: int - :param cipher_page_size: The page size. - :type cipher_page_size: int - :param defer_encryption: Whether to defer encryption/decryption of - documents, or do it inline while syncing. - :type defer_encryption: bool - - :return: An instance of Database. - :rtype SQLCipherDatabase - """ - return SQLCipherDatabase.open_database( - path, password, create=create, document_factory=document_factory, - crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter, - cipher_page_size=cipher_page_size, defer_encryption=defer_encryption) +# Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2 +sqlite_backend.dbapi2 = sqlcipher_dbapi2 -# -# Exceptions -# -class DatabaseIsNotEncrypted(Exception): - """ - Exception raised when trying to open non-encrypted databases. +def initialize_sqlcipher_db(opts, on_init=None, check_same_thread=True): """ - pass - + Initialize a SQLCipher database. -class NotAnHexString(Exception): + :param opts: + :type opts: SQLCipherOptions + :param on_init: a tuple of queries to be executed on initialization + :type on_init: tuple + :return: pysqlcipher.dbapi2.Connection """ - Raised when trying to (raw) key the database with a non-hex string. - """ - pass + # Note: There seemed to be a bug in sqlite 3.5.9 (with python2.6) + # where without re-opening the database on Windows, it + # doesn't see the transaction that was just committed + # Removing from here now, look at the pysqlite implementation if the + # bug shows up in windows. + if not os.path.isfile(opts.path) and not opts.create: + raise u1db_errors.DatabaseDoesNotExist() -# -# The SQLCipher database -# + conn = sqlcipher_dbapi2.connect( + opts.path, check_same_thread=check_same_thread) + pragmas.set_init_pragmas(conn, opts, extra_queries=on_init) + return conn -class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): - """ - A U1DB implementation that uses SQLCipher as its persistence layer. - """ - defer_encryption = False - _index_storage_value = 'expand referenced encrypted' - k_lock = threading.Lock() - create_doc_lock = threading.Lock() - update_indexes_lock = threading.Lock() - _sync_watcher = None - _sync_enc_pool = None +def initialize_sqlcipher_adbapi_db(opts, extra_queries=None): + from leap.soledad.client import sqlcipher_adbapi + return sqlcipher_adbapi.getConnectionPool( + opts, extra_queries=extra_queries) - """ - The name of the local symmetrically encrypted documents to - sync database file. - """ - LOCAL_SYMMETRIC_SYNC_FILE_NAME = 'sync.u1db' +class SQLCipherOptions(object): """ - A dictionary that hold locks which avoid multiple sync attempts from the - same database replica. - """ - encrypting_lock = threading.Lock() - - """ - Period or recurrence of the periodic encrypting task, in seconds. - """ - ENCRYPT_TASK_PERIOD = 1 - - syncing_lock = defaultdict(threading.Lock) - """ - A dictionary that hold locks which avoid multiple sync attempts from the - same database replica. + A container with options for the initialization of an SQLCipher database. """ - def __init__(self, sqlcipher_file, password, document_factory=None, - crypto=None, raw_key=False, cipher='aes-256-cbc', - kdf_iter=4000, cipher_page_size=1024): + @classmethod + def copy(cls, source, path=None, key=None, create=None, + is_raw_key=None, cipher=None, kdf_iter=None, + cipher_page_size=None, defer_encryption=None, sync_db_key=None): """ - Connect to an existing SQLCipher database, creating a new sqlcipher - database file if needed. - - :param sqlcipher_file: The path for the SQLCipher file. - :type sqlcipher_file: str - :param password: The password that protects the SQLCipher db. - :type password: str - :param document_factory: A function that will be called with the same - parameters as Document.__init__. - :type document_factory: callable - :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. - :type crypto: soledad.crypto.SoledadCrypto - :param raw_key: Whether password is a raw 64-char hex string or a - passphrase that should be hashed to obtain the - encyrption key. - :type raw_key: bool - :param cipher: The cipher and mode to use. - :type cipher: str - :param kdf_iter: The number of iterations to use. - :type kdf_iter: int - :param cipher_page_size: The page size. - :type cipher_page_size: int + Return a copy of C{source} with parameters different than None + replaced by new values. """ - # ensure the db is encrypted if the file already exists - if os.path.exists(sqlcipher_file): - self.assert_db_is_encrypted( - sqlcipher_file, password, raw_key, cipher, kdf_iter, - cipher_page_size) + local_vars = locals() + args = [] + kwargs = {} - # connect to the sqlcipher database - with self.k_lock: - self._db_handle = dbapi2.connect( - sqlcipher_file, - isolation_level=SQLITE_ISOLATION_LEVEL, - check_same_thread=SQLITE_CHECK_SAME_THREAD) - # set SQLCipher cryptographic parameters - self._set_crypto_pragmas( - self._db_handle, password, raw_key, cipher, kdf_iter, - cipher_page_size) - if os.environ.get('LEAP_SQLITE_NOSYNC'): - self._pragma_synchronous_off(self._db_handle) + for name in ["path", "key"]: + val = local_vars[name] + if val is not None: + args.append(val) else: - self._pragma_synchronous_normal(self._db_handle) - if os.environ.get('LEAP_SQLITE_MEMSTORE'): - self._pragma_mem_temp_store(self._db_handle) - self._pragma_write_ahead_logging(self._db_handle) - self._real_replica_uid = None - self._ensure_schema() - self._crypto = crypto - - self._sync_db = None - self._sync_db_write_lock = None - self._sync_enc_pool = None + args.append(getattr(source, name)) - if self.defer_encryption: - if sqlcipher_file != ":memory:": - self._sync_db_path = "%s-sync" % sqlcipher_file + for name in ["create", "is_raw_key", "cipher", "kdf_iter", + "cipher_page_size", "defer_encryption", "sync_db_key"]: + val = local_vars[name] + if val is not None: + kwargs[name] = val else: - self._sync_db_path = ":memory:" - - # initialize sync db - self._init_sync_db() - - # initialize syncing queue encryption pool - self._sync_enc_pool = SyncEncrypterPool( - self._crypto, self._sync_db, self._sync_db_write_lock) - self._sync_watcher = TimerTask(self._encrypt_syncing_docs, - self.ENCRYPT_TASK_PERIOD) - self._sync_watcher.start() - - def factory(doc_id=None, rev=None, json='{}', has_conflicts=False, - syncable=True): - return SoledadDocument(doc_id=doc_id, rev=rev, json=json, - has_conflicts=has_conflicts, - syncable=syncable) - self.set_document_factory(factory) - # we store syncers in a dictionary indexed by the target URL. We also - # store a hash of the auth info in case auth info expires and we need - # to rebuild the syncer for that target. The final self._syncers - # format is the following: - # - # self._syncers = {'<url>': ('<auth_hash>', syncer), ...} - self._syncers = {} - - @classmethod - def _open_database(cls, sqlcipher_file, password, document_factory=None, - crypto=None, raw_key=False, cipher='aes-256-cbc', - kdf_iter=4000, cipher_page_size=1024, - defer_encryption=False): - """ - Open a SQLCipher database. - - :param sqlcipher_file: The path for the SQLCipher file. - :type sqlcipher_file: str - :param password: The password that protects the SQLCipher db. - :type password: str - :param document_factory: A function that will be called with the same - parameters as Document.__init__. - :type document_factory: callable - :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. - :type crypto: soledad.crypto.SoledadCrypto - :param raw_key: Whether C{password} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. + kwargs[name] = getattr(source, name) + + return SQLCipherOptions(*args, **kwargs) + + def __init__(self, path, key, create=True, is_raw_key=False, + cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024, + defer_encryption=False, sync_db_key=None): + """ + :param path: The filesystem path for the database to open. + :type path: str + :param create: + True/False, should the database be created if it doesn't + already exist? + :param create: bool + :param is_raw_key: + Whether ``password`` is a raw 64-char hex string or a passphrase + that should be hashed to obtain the encyrption key. :type raw_key: bool :param cipher: The cipher and mode to use. :type cipher: str @@ -304,233 +163,93 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :type kdf_iter: int :param cipher_page_size: The page size. :type cipher_page_size: int - :param defer_encryption: Whether to defer encryption/decryption of - documents, or do it inline while syncing. + :param defer_encryption: + Whether to defer encryption/decryption of documents, or do it + inline while syncing. :type defer_encryption: bool - - :return: The database object. - :rtype: SQLCipherDatabase """ - cls.defer_encryption = defer_encryption - if not os.path.isfile(sqlcipher_file): - raise u1db_errors.DatabaseDoesNotExist() - - tries = 2 - # Note: There seems to be a bug in sqlite 3.5.9 (with python2.6) - # where without re-opening the database on Windows, it - # doesn't see the transaction that was just committed - while True: - - with cls.k_lock: - db_handle = dbapi2.connect( - sqlcipher_file, - check_same_thread=SQLITE_CHECK_SAME_THREAD) - - try: - # set cryptographic params - cls._set_crypto_pragmas( - db_handle, password, raw_key, cipher, kdf_iter, - cipher_page_size) - c = db_handle.cursor() - # XXX if we use it here, it should be public - v, err = cls._which_index_storage(c) - except Exception as exc: - logger.warning("ERROR OPENING DATABASE!") - logger.debug("error was: %r" % exc) - v, err = None, exc - finally: - db_handle.close() - if v is not None: - break - # possibly another process is initializing it, wait for it to be - # done - if tries == 0: - raise err # go for the richest error? - tries -= 1 - time.sleep(cls.WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL) - return SQLCipherDatabase._sqlite_registry[v]( - sqlcipher_file, password, document_factory=document_factory, - crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter, - cipher_page_size=cipher_page_size) + self.path = path + self.key = key + self.is_raw_key = is_raw_key + self.create = create + self.cipher = cipher + self.kdf_iter = kdf_iter + self.cipher_page_size = cipher_page_size + self.defer_encryption = defer_encryption + self.sync_db_key = sync_db_key - @classmethod - def open_database(cls, sqlcipher_file, password, create, backend_cls=None, - document_factory=None, crypto=None, raw_key=False, - cipher='aes-256-cbc', kdf_iter=4000, - cipher_page_size=1024, defer_encryption=False): + def __str__(self): """ - Open a SQLCipher database. - - :param sqlcipher_file: The path for the SQLCipher file. - :type sqlcipher_file: str - - :param password: The password that protects the SQLCipher db. - :type password: str - - :param create: Should the datbase be created if it does not already - exist? - :type create: bool - - :param backend_cls: A class to use as backend. - :type backend_cls: type + Return string representation of options, for easy debugging. - :param document_factory: A function that will be called with the same - parameters as Document.__init__. - :type document_factory: callable - - :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. - :type crypto: soledad.crypto.SoledadCrypto - - :param raw_key: Whether C{password} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the - encyrption key. - :type raw_key: bool + :return: String representation of options. + :rtype: str + """ + attr_names = filter(lambda a: not a.startswith('_'), dir(self)) + attr_str = [] + for a in attr_names: + attr_str.append(a + "=" + str(getattr(self, a))) + name = self.__class__.__name__ + return "%s(%s)" % (name, ', '.join(attr_str)) - :param cipher: The cipher and mode to use. - :type cipher: str - :param kdf_iter: The number of iterations to use. - :type kdf_iter: int +# +# The SQLCipher database +# - :param cipher_page_size: The page size. - :type cipher_page_size: int +class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): + """ + A U1DB implementation that uses SQLCipher as its persistence layer. + """ + defer_encryption = False - :param defer_encryption: Whether to defer encryption/decryption of - documents, or do it inline while syncing. - :type defer_encryption: bool + # The attribute _index_storage_value will be used as the lookup key for the + # implementation of the SQLCipher storage backend. + _index_storage_value = 'expand referenced encrypted' - :return: The database object. - :rtype: SQLCipherDatabase - """ - cls.defer_encryption = defer_encryption - try: - return cls._open_database( - sqlcipher_file, password, document_factory=document_factory, - crypto=crypto, raw_key=raw_key, cipher=cipher, - kdf_iter=kdf_iter, cipher_page_size=cipher_page_size, - defer_encryption=defer_encryption) - except u1db_errors.DatabaseDoesNotExist: - if not create: - raise - # TODO: remove backend class from here. - if backend_cls is None: - # default is SQLCipherPartialExpandDatabase - backend_cls = SQLCipherDatabase - return backend_cls( - sqlcipher_file, password, document_factory=document_factory, - crypto=crypto, raw_key=raw_key, cipher=cipher, - kdf_iter=kdf_iter, cipher_page_size=cipher_page_size) - - def sync(self, url, creds=None, autocreate=True, defer_decryption=True): + def __init__(self, opts): """ - Synchronize documents with remote replica exposed at url. + Connect to an existing SQLCipher database, creating a new sqlcipher + database file if needed. - There can be at most one instance syncing the same database replica at - the same time, so this method will block until the syncing lock can be - acquired. + *** IMPORTANT *** - :param url: The url of the target replica to sync with. - :type url: str - :param creds: optional dictionary giving credentials. - to authorize the operation with the server. - :type creds: dict - :param autocreate: Ask the target to create the db if non-existent. - :type autocreate: bool - :param defer_decryption: Whether to defer the decryption process using - the intermediate database. If False, - decryption will be done inline. - :type defer_decryption: bool + Don't forget to close the database after use by calling the close() + method otherwise some resources might not be freed and you may + experience several kinds of leakages. - :return: The local generation before the synchronisation was performed. - :rtype: int - """ - res = None - # the following context manager blocks until the syncing lock can be - # acquired. - with self.syncer(url, creds=creds) as syncer: + *** IMPORTANT *** - # XXX could mark the critical section here... - try: - if defer_decryption and not self.defer_encryption: - logger.warning("Can't defer decryption without first having " - "created a sync db. Falling back to normal " - "syncing mode.") - defer_decryption = False - res = syncer.sync(autocreate=autocreate, - defer_decryption=defer_decryption) - - except PendingReceivedDocsSyncError: - logger.warning("Local sync db is not clear, skipping sync...") - return - - return res - - def stop_sync(self): + :param opts: options for initialization of the SQLCipher database. + :type opts: SQLCipherOptions """ - Interrupt all ongoing syncs. - """ - for url in self._syncers: - _, syncer = self._syncers[url] - syncer.stop() + # ensure the db is encrypted if the file already exists + if os.path.isfile(opts.path): + _assert_db_is_encrypted(opts) - @contextmanager - def syncer(self, url, creds=None): - """ - Accesor for synchronizer. + # connect to the sqlcipher database + self._db_handle = initialize_sqlcipher_db(opts) - As we reuse the same synchronizer for every sync, there can be only - one instance synchronizing the same database replica at the same time. - Because of that, this method blocks until the syncing lock can be - acquired. - """ - with SQLCipherDatabase.syncing_lock[self._get_replica_uid()]: - syncer = self._get_syncer(url, creds=creds) - yield syncer + # TODO --------------------------------------------------- + # Everything else in this initialization has to be factored + # out, so it can be used from SoledadSQLCipherWrapper.__init__ + # too. + # --------------------------------------------------------- - @property - def syncing(self): - lock = SQLCipherDatabase.syncing_lock[self._get_replica_uid()] - acquired_lock = lock.acquire(False) - if acquired_lock is False: - return True - lock.release() - return False + self._ensure_schema() + self.set_document_factory(soledad_doc_factory) + self._prime_replica_uid() - def _get_syncer(self, url, creds=None): + def _prime_replica_uid(self): """ - Get a synchronizer for C{url} using C{creds}. - - :param url: The url of the target replica to sync with. - :type url: str - :param creds: optional dictionary giving credentials. - to authorize the operation with the server. - :type creds: dict - - :return: A synchronizer. - :rtype: Synchronizer + In the u1db implementation, _replica_uid is a property + that returns the value in _real_replica_uid, and does + a db query if no value found. + Here we prime the replica uid during initialization so + that we don't have to wait for the query afterwards. """ - # we want to store at most one syncer for each url, so we also store a - # hash of the connection credentials and replace the stored syncer for - # a certain url if credentials have changed. - h = sha256(json.dumps([url, creds])).hexdigest() - cur_h, syncer = self._syncers.get(url, (None, None)) - if syncer is None or h != cur_h: - wlock = self._sync_db_write_lock - syncer = SoledadSynchronizer( - self, - SoledadSyncTarget(url, - self._replica_uid, - creds=creds, - crypto=self._crypto, - sync_db=self._sync_db, - sync_db_write_lock=wlock)) - self._syncers[url] = (h, syncer) - # in order to reuse the same synchronizer multiple times we have to - # reset its state (i.e. the number of documents received from target - # and inserted in the local replica). - syncer.num_inserted = 0 - return syncer + self._real_replica_uid = None + self._get_replica_uid() def _extra_schema_init(self, c): """ @@ -547,63 +266,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): 'ALTER TABLE document ' 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') - def _init_sync_db(self): - """ - Initialize the Symmetrically-Encrypted document to be synced database, - and the queue to communicate with subprocess workers. - """ - self._sync_db = sqlite3.connect(self._sync_db_path, - check_same_thread=False) - - self._sync_db_write_lock = threading.Lock() - self._create_sync_db_tables() - self.sync_queue = multiprocessing.Queue() - - def _create_sync_db_tables(self): - """ - Create tables for the local sync documents db if needed. - """ - encr = SyncEncrypterPool - decr = SyncDecrypterPool - sql_encr = ("CREATE TABLE IF NOT EXISTS %s (%s)" % ( - encr.TABLE_NAME, encr.FIELD_NAMES)) - sql_decr = ("CREATE TABLE IF NOT EXISTS %s (%s)" % ( - decr.TABLE_NAME, decr.FIELD_NAMES)) - - with self._sync_db_write_lock: - with self._sync_db: - self._sync_db.execute(sql_encr) - self._sync_db.execute(sql_decr) - - # - # Symmetric encryption of syncing docs - # - - def _encrypt_syncing_docs(self): - """ - Process the syncing queue and send the documents there - to be encrypted in the sync db. They will be read by the - SoledadSyncTarget during the sync_exchange. - - Called periodical from the TimerTask self._sync_watcher. - """ - lock = self.encrypting_lock - # optional wait flag used to avoid blocking - if not lock.acquire(False): - return - else: - queue = self.sync_queue - try: - while not queue.empty(): - doc = queue.get_nowait() - self._sync_enc_pool.encrypt_doc(doc) - - except Exception as exc: - logger.error("Error while encrypting docs to sync") - logger.exception(exc) - finally: - lock.release() - # # Document operations # @@ -619,12 +281,82 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :return: The new document revision. :rtype: str """ - doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc( - self, doc) + doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc(self, doc) if self.defer_encryption: - self.sync_queue.put_nowait(doc) + # TODO move to api? + self._sync_enc_pool.enqueue_doc_for_encryption(doc) return doc_rev + # + # SQLCipher API methods + # + + # Extra query methods: extensions to the base u1db sqlite implmentation. + + def get_count_from_index(self, index_name, *key_values): + """ + Return the count for a given combination of index_name + and key values. + + Extension method made from similar methods in u1db version 13.09 + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: count. + :rtype: int + """ + c = self._db_handle.cursor() + definition = self._get_index_definition(index_name) + + if len(key_values) != len(definition): + raise u1db_errors.InvalidValueForIndex() + tables = ["document_fields d%d" % i for i in range(len(definition))] + novalue_where = ["d.doc_id = d%d.doc_id" + " AND d%d.field_name = ?" + % (i, i) for i in range(len(definition))] + exact_where = [novalue_where[i] + + (" AND d%d.value = ?" % (i,)) + for i in range(len(definition))] + args = [] + where = [] + for idx, (field, value) in enumerate(zip(definition, key_values)): + args.append(field) + where.append(exact_where[idx]) + args.append(value) + + tables = ["document_fields d%d" % i for i in range(len(definition))] + statement = ( + "SELECT COUNT(*) FROM document d, %s WHERE %s " % ( + ', '.join(tables), + ' AND '.join(where), + )) + try: + c.execute(statement, tuple(args)) + except sqlcipher_dbapi2.OperationalError, e: + raise sqlcipher_dbapi2.OperationalError( + str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args)) + res = c.fetchall() + return res[0][0] + + def close(self): + """ + Close db connections. + """ + # TODO should be handled by adbapi instead + # TODO syncdb should be stopped first + + if logger is not None: # logger might be none if called from __del__ + logger.debug("SQLCipher backend: closing") + + # close the actual database + if getattr(self, '_db_handle', False): + self._db_handle.close() + self._db_handle = None + # indexes def _put_and_update_indexes(self, old_doc, doc): @@ -636,13 +368,11 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :param doc: The new version of the document. :type doc: u1db.Document """ - with self.update_indexes_lock: - sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes( - self, old_doc, doc) - c = self._db_handle.cursor() - c.execute('UPDATE document SET syncable=? ' - 'WHERE doc_id=?', - (doc.syncable, doc.doc_id)) + sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes( + self, old_doc, doc) + c = self._db_handle.cursor() + c.execute('UPDATE document SET syncable=? WHERE doc_id=?', + (doc.syncable, doc.doc_id)) def _get_doc(self, doc_id, check_for_conflicts=False): """ @@ -662,438 +392,426 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self, doc_id, check_for_conflicts) if doc: c = self._db_handle.cursor() - c.execute('SELECT syncable FROM document ' - 'WHERE doc_id=?', + c.execute('SELECT syncable FROM document WHERE doc_id=?', (doc.doc_id,)) result = c.fetchone() doc.syncable = bool(result[0]) return doc - # - # SQLCipher API methods - # - - @classmethod - def assert_db_is_encrypted(cls, sqlcipher_file, key, raw_key, cipher, - kdf_iter, cipher_page_size): + def __del__(self): """ - Assert that C{sqlcipher_file} contains an encrypted database. - - When opening an existing database, PRAGMA key will not immediately - throw an error if the key provided is incorrect. To test that the - database can be successfully opened with the provided key, it is - necessary to perform some operation on the database (i.e. read from - it) and confirm it is success. - - The easiest way to do this is select off the sqlite_master table, - which will attempt to read the first page of the database and will - parse the schema. - - :param sqlcipher_file: The path for the SQLCipher file. - :type sqlcipher_file: str - :param key: The key that protects the SQLCipher db. - :type key: str - :param raw_key: Whether C{key} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. - :type raw_key: bool - :param cipher: The cipher and mode to use. - :type cipher: str - :param kdf_iter: The number of iterations to use. - :type kdf_iter: int - :param cipher_page_size: The page size. - :type cipher_page_size: int - """ - try: - # try to open an encrypted database with the regular u1db - # backend should raise a DatabaseError exception. - sqlite_backend.SQLitePartialExpandDatabase(sqlcipher_file) - raise DatabaseIsNotEncrypted() - except dbapi2.DatabaseError: - # assert that we can access it using SQLCipher with the given - # key - with cls.k_lock: - db_handle = dbapi2.connect( - sqlcipher_file, - isolation_level=SQLITE_ISOLATION_LEVEL, - check_same_thread=SQLITE_CHECK_SAME_THREAD) - cls._set_crypto_pragmas( - db_handle, key, raw_key, cipher, - kdf_iter, cipher_page_size) - db_handle.cursor().execute( - 'SELECT count(*) FROM sqlite_master') + Free resources when deleting or garbage collecting the database. - @classmethod - def _set_crypto_pragmas(cls, db_handle, key, raw_key, cipher, kdf_iter, - cipher_page_size): - """ - Set cryptographic params (key, cipher, KDF number of iterations and - cipher page size). + This is only here to minimze problems if someone ever forgets to call + the close() method after using the database; you should not rely on + garbage collecting to free up the database resources. """ - cls._pragma_key(db_handle, key, raw_key) - cls._pragma_cipher(db_handle, cipher) - cls._pragma_kdf_iter(db_handle, kdf_iter) - cls._pragma_cipher_page_size(db_handle, cipher_page_size) + self.close() - @classmethod - def _pragma_key(cls, db_handle, key, raw_key): - """ - Set the C{key} for use with the database. - The process of creating a new, encrypted database is called 'keying' - the database. SQLCipher uses just-in-time key derivation at the point - it is first needed for an operation. This means that the key (and any - options) must be set before the first operation on the database. As - soon as the database is touched (e.g. SELECT, CREATE TABLE, UPDATE, - etc.) and pages need to be read or written, the key is prepared for - use. +class SQLCipherU1DBSync(SQLCipherDatabase): + """ + Soledad syncer implementation. + """ - Implementation Notes: + _sync_enc_pool = None - * PRAGMA key should generally be called as the first operation on a - database. + """ + The name of the local symmetrically encrypted documents to + sync database file. + """ + LOCAL_SYMMETRIC_SYNC_FILE_NAME = 'sync.u1db' - :param key: The key for use with the database. - :type key: str - :param raw_key: Whether C{key} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. - :type raw_key: bool - """ - if raw_key: - cls._pragma_key_raw(db_handle, key) - else: - cls._pragma_key_passphrase(db_handle, key) + """ + Period or recurrence of the Looping Call that will do the encryption to the + syncdb (in seconds). + """ + ENCRYPT_LOOP_PERIOD = 1 - @classmethod - def _pragma_key_passphrase(cls, db_handle, passphrase): - """ - Set a passphrase for encryption key derivation. - - The key itself can be a passphrase, which is converted to a key using - PBKDF2 key derivation. The result is used as the encryption key for - the database. By using this method, there is no way to alter the KDF; - if you want to do so you should use a raw key instead and derive the - key using your own KDF. - - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param passphrase: The passphrase used to derive the encryption key. - :type passphrase: str - """ - db_handle.cursor().execute("PRAGMA key = '%s'" % passphrase) + """ + A dictionary that hold locks which avoid multiple sync attempts from the + same database replica. + """ + syncing_lock = defaultdict(threading.Lock) - @classmethod - def _pragma_key_raw(cls, db_handle, key): - """ - Set a raw hexadecimal encryption key. + def __init__(self, opts, soledad_crypto, replica_uid, cert_file, + defer_encryption=False): - It is possible to specify an exact byte sequence using a blob literal. - With this method, it is the calling application's responsibility to - ensure that the data provided is a 64 character hex string, which will - be converted directly to 32 bytes (256 bits) of key data. + self._opts = opts + self._path = opts.path + self._crypto = soledad_crypto + self.__replica_uid = replica_uid + self._cert_file = cert_file - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param key: A 64 character hex string. - :type key: str - """ - if not all(c in string.hexdigits for c in key): - raise NotAnHexString(key) - db_handle.cursor().execute('PRAGMA key = "x\'%s"' % key) + self._sync_db_key = opts.sync_db_key + self._sync_db = None + self._sync_enc_pool = None - @classmethod - def _pragma_cipher(cls, db_handle, cipher='aes-256-cbc'): - """ - Set the cipher and mode to use for symmetric encryption. + # we store syncers in a dictionary indexed by the target URL. We also + # store a hash of the auth info in case auth info expires and we need + # to rebuild the syncer for that target. The final self._syncers + # format is the following: + # + # self._syncers = {'<url>': ('<auth_hash>', syncer), ...} - SQLCipher uses aes-256-cbc as the default cipher and mode of - operation. It is possible to change this, though not generally - recommended, using PRAGMA cipher. + self._syncers = {} - SQLCipher makes direct use of libssl, so all cipher options available - to libssl are also available for use with SQLCipher. See `man enc` for - OpenSSL's supported ciphers. + self.running = False + self._sync_threadpool = None + self._initialize_sync_threadpool() - Implementation Notes: + self._reactor = reactor + self._reactor.callWhenRunning(self._start) - * PRAGMA cipher must be called after PRAGMA key and before the first - actual database operation or it will have no effect. + self._db_handle = None + self._initialize_main_db() - * If a non-default value is used PRAGMA cipher to create a database, - it must also be called every time that database is opened. + # the sync_db is used both for deferred encryption and decryption, so + # we want to initialize it anyway to allow for all combinations of + # deferred encryption and decryption configurations. + self._initialize_sync_db(opts) - * SQLCipher does not implement its own encryption. Instead it uses the - widely available and peer-reviewed OpenSSL libcrypto for all - cryptographic functions. + if defer_encryption: + # initialize syncing queue encryption pool + self._sync_enc_pool = encdecpool.SyncEncrypterPool( + self._crypto, self._sync_db) - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param cipher: The cipher and mode to use. - :type cipher: str - """ - db_handle.cursor().execute("PRAGMA cipher = '%s'" % cipher) + self.shutdownID = None - @classmethod - def _pragma_kdf_iter(cls, db_handle, kdf_iter=4000): - """ - Set the number of iterations for the key derivation function. + @property + def _replica_uid(self): + return str(self.__replica_uid) - SQLCipher uses PBKDF2 key derivation to strengthen the key and make it - resistent to brute force and dictionary attacks. The default - configuration uses 4000 PBKDF2 iterations (effectively 16,000 SHA1 - operations). PRAGMA kdf_iter can be used to increase or decrease the - number of iterations used. + def _start(self): + if not self.running: + self._sync_threadpool.start() + self.shutdownID = self._reactor.addSystemEventTrigger( + 'during', 'shutdown', self.finalClose) + self.running = True - Implementation Notes: + def _defer_to_sync_threadpool(self, meth, *args, **kwargs): + return deferToThreadPool( + self._reactor, self._sync_threadpool, meth, *args, **kwargs) - * PRAGMA kdf_iter must be called after PRAGMA key and before the first - actual database operation or it will have no effect. + def _initialize_main_db(self): - * If a non-default value is used PRAGMA kdf_iter to create a database, - it must also be called every time that database is opened. + def _init_db(): + self._db_handle = initialize_sqlcipher_db( + self._opts, check_same_thread=False) + self._real_replica_uid = None + self._ensure_schema() + self.set_document_factory(soledad_doc_factory) - * It is not recommended to reduce the number of iterations if a - passphrase is in use. + return self._defer_to_sync_threadpool(_init_db) - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param kdf_iter: The number of iterations to use. - :type kdf_iter: int + def _initialize_sync_threadpool(self): """ - db_handle.cursor().execute("PRAGMA kdf_iter = '%d'" % kdf_iter) + Initialize a ThreadPool with exactly one thread, that will be used to + run all the network blocking calls for syncing on a separate thread. - @classmethod - def _pragma_cipher_page_size(cls, db_handle, cipher_page_size=1024): + TODO this needs to be ported away from urllib and into twisted async + calls, and then we can ditch this syncing thread and reintegrate into + the main reactor. """ - Set the page size of the encrypted database. - - SQLCipher 2 introduced the new PRAGMA cipher_page_size that can be - used to adjust the page size for the encrypted database. The default - page size is 1024 bytes, but it can be desirable for some applications - to use a larger page size for increased performance. For instance, - some recent testing shows that increasing the page size can noticeably - improve performance (5-30%) for certain queries that manipulate a - large number of pages (e.g. selects without an index, large inserts in - a transaction, big deletes). + # XXX if the number of threads in this thread pool is ever changed, we + # should make sure that no operations on the database shuold occur + # before the database has been initialized. + self._sync_threadpool = ThreadPool(0, 1) - To adjust the page size, call the pragma immediately after setting the - key for the first time and each subsequent time that you open the - database. - - Implementation Notes: + def _initialize_sync_db(self, opts): + """ + Initialize the Symmetrically-Encrypted document to be synced database, + and the queue to communicate with subprocess workers. - * PRAGMA cipher_page_size must be called after PRAGMA key and before - the first actual database operation or it will have no effect. + :param opts: + :type opts: SQLCipherOptions + """ + soledad_assert(opts.sync_db_key is not None) + sync_db_path = None + if opts.path != ":memory:": + sync_db_path = "%s-sync" % opts.path + else: + sync_db_path = ":memory:" - * If a non-default value is used PRAGMA cipher_page_size to create a - database, it must also be called every time that database is opened. + # we copy incoming options because the opts object might be used + # somewhere else + sync_opts = SQLCipherOptions.copy( + opts, path=sync_db_path, create=True) + self._sync_db = getConnectionPool( + sync_opts, extra_queries=self._sync_db_extra_init) - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param cipher_page_size: The page size. - :type cipher_page_size: int + @property + def _sync_db_extra_init(self): """ - db_handle.cursor().execute( - "PRAGMA cipher_page_size = '%d'" % cipher_page_size) + Queries for creating tables for the local sync documents db if needed. + They are passed as extra initialization to initialize_sqlciphjer_db - @classmethod - def _pragma_rekey(cls, db_handle, new_key, raw_key): - """ - Change the key of an existing encrypted database. - - To change the key on an existing encrypted database, it must first be - unlocked with the current encryption key. Once the database is - readable and writeable, PRAGMA rekey can be used to re-encrypt every - page in the database with a new key. - - * PRAGMA rekey must be called after PRAGMA key. It can be called at any - time once the database is readable. - - * PRAGMA rekey can not be used to encrypted a standard SQLite - database! It is only useful for changing the key on an existing - database. - - * Previous versions of SQLCipher provided a PRAGMA rekey_cipher and - code>PRAGMA rekey_kdf_iter. These are deprecated and should not be - used. Instead, use sqlcipher_export(). - - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param new_key: The new key. - :type new_key: str - :param raw_key: Whether C{password} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. - :type raw_key: bool + :rtype: tuple of strings """ - # XXX change key param! - if raw_key: - cls._pragma_rekey_raw(db_handle, key) - else: - cls._pragma_rekey_passphrase(db_handle, key) + maybe_create = "CREATE TABLE IF NOT EXISTS %s (%s)" + encr = encdecpool.SyncEncrypterPool + decr = encdecpool.SyncDecrypterPool + sql_encr_table_query = (maybe_create % ( + encr.TABLE_NAME, encr.FIELD_NAMES)) + sql_decr_table_query = (maybe_create % ( + decr.TABLE_NAME, decr.FIELD_NAMES)) + return (sql_encr_table_query, sql_decr_table_query) - @classmethod - def _pragma_rekey_passphrase(cls, db_handle, passphrase): + def sync(self, url, creds=None, defer_decryption=True): """ - Change the passphrase for encryption key derivation. + Synchronize documents with remote replica exposed at url. - The key itself can be a passphrase, which is converted to a key using - PBKDF2 key derivation. The result is used as the encryption key for - the database. + This method defers a sync to a 1-threaded threadpool. The main + database initialziation was deferred to that thread during this + object's initialization. As there's currently only one thread in that + threadpool, the db init was queued before this method was called, so + we don't need to actually wait for the db to be ready. If this ever + changes, we should add a thread-safe condition to ensure the db is + ready before using it. + + :param url: The url of the target replica to sync with. + :type url: str + :param creds: optional dictionary giving credentials to authorize the + operation with the server. + :type creds: dict + :param defer_decryption: + Whether to defer the decryption process using the intermediate + database. If False, decryption will be done inline. + :type defer_decryption: bool - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param passphrase: The passphrase used to derive the encryption key. - :type passphrase: str + :return: + A Deferred, that will fire with the local generation (type `int`) + before the synchronisation was performed. + :rtype: Deferred """ - db_handle.cursor().execute("PRAGMA rekey = '%s'" % passphrase) + # the following context manager blocks until the syncing lock can be + # acquired. + with self._syncer(url, creds=creds) as syncer: + # XXX could mark the critical section here... + return syncer.sync(defer_decryption=defer_decryption) - @classmethod - def _pragma_rekey_raw(cls, db_handle, key): + @contextmanager + def _syncer(self, url, creds=None): """ - Change the raw hexadecimal encryption key. + Accesor for synchronizer. - It is possible to specify an exact byte sequence using a blob literal. - With this method, it is the calling application's responsibility to - ensure that the data provided is a 64 character hex string, which will - be converted directly to 32 bytes (256 bits) of key data. + As we reuse the same synchronizer for every sync, there can be only + one instance synchronizing the same database replica at the same time. + Because of that, this method blocks until the syncing lock can be + acquired. - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param key: A 64 character hex string. - :type key: str + :param creds: optional dictionary giving credentials to authorize the + operation with the server. + :type creds: dict """ - if not all(c in string.hexdigits for c in key): - raise NotAnHexString(key) - # XXX change passphrase param! - db_handle.cursor().execute('PRAGMA rekey = "x\'%s"' % passphrase) + with self.syncing_lock[self._path]: + syncer = self._get_syncer(url, creds=creds) + yield syncer - @classmethod - def _pragma_synchronous_off(cls, db_handle): - """ - Change the setting of the "synchronous" flag to OFF. - """ - logger.debug("SQLCIPHER: SETTING SYNCHRONOUS OFF") - db_handle.cursor().execute('PRAGMA synchronous=OFF') + @property + def syncing(self): + lock = self.syncing_lock[self._path] + acquired_lock = lock.acquire(False) + if acquired_lock is False: + return True + lock.release() + return False - @classmethod - def _pragma_synchronous_normal(cls, db_handle): + def _get_syncer(self, url, creds=None): """ - Change the setting of the "synchronous" flag to NORMAL. + Get a synchronizer for ``url`` using ``creds``. + + :param url: The url of the target replica to sync with. + :type url: str + :param creds: optional dictionary giving credentials. + to authorize the operation with the server. + :type creds: dict + + :return: A synchronizer. + :rtype: Synchronizer """ - logger.debug("SQLCIPHER: SETTING SYNCHRONOUS NORMAL") - db_handle.cursor().execute('PRAGMA synchronous=NORMAL') + # we want to store at most one syncer for each url, so we also store a + # hash of the connection credentials and replace the stored syncer for + # a certain url if credentials have changed. + h = sha256(json.dumps([url, creds])).hexdigest() + cur_h, syncer = self._syncers.get(url, (None, None)) + if syncer is None or h != cur_h: + syncer = SoledadSynchronizer( + self, + SoledadHTTPSyncTarget( + url, + # XXX is the replica_uid ready? + self._replica_uid, + creds=creds, + crypto=self._crypto, + cert_file=self._cert_file, + sync_db=self._sync_db, + sync_enc_pool=self._sync_enc_pool)) + self._syncers[url] = (h, syncer) + # in order to reuse the same synchronizer multiple times we have to + # reset its state (i.e. the number of documents received from target + # and inserted in the local replica). + syncer.num_inserted = 0 + return syncer - @classmethod - def _pragma_mem_temp_store(cls, db_handle): + # + # Symmetric encryption of syncing docs + # + + def get_generation(self): + # FIXME + # XXX this SHOULD BE a callback + return self._get_generation() + + def finalClose(self): """ - Use a in-memory store for temporary tables. + This should only be called by the shutdown trigger. """ - logger.debug("SQLCIPHER: SETTING TEMP_STORE MEMORY") - db_handle.cursor().execute('PRAGMA temp_store=MEMORY') + self.shutdownID = None + self._sync_threadpool.stop() + self.running = False - @classmethod - def _pragma_write_ahead_logging(cls, db_handle): + def close(self): """ - Enable write-ahead logging, and set the autocheckpoint to 50 pages. - - Setting the autocheckpoint to a small value, we make the reads not - suffer too much performance degradation. - - From the sqlite docs: - - "There is a tradeoff between average read performance and average write - performance. To maximize the read performance, one wants to keep the - WAL as small as possible and hence run checkpoints frequently, perhaps - as often as every COMMIT. To maximize write performance, one wants to - amortize the cost of each checkpoint over as many writes as possible, - meaning that one wants to run checkpoints infrequently and let the WAL - grow as large as possible before each checkpoint. The decision of how - often to run checkpoints may therefore vary from one application to - another depending on the relative read and write performance - requirements of the application. The default strategy is to run a - checkpoint once the WAL reaches 1000 pages" + Close the syncer and syncdb orderly """ - logger.debug("SQLCIPHER: SETTING WRITE-AHEAD LOGGING") - db_handle.cursor().execute('PRAGMA journal_mode=WAL') - # The optimum value can still use a little bit of tuning, but we favor - # small sizes of the WAL file to get fast reads, since we assume that - # the writes will be quick enough to not block too much. + # close all open syncers + for url in self._syncers.keys(): + del self._syncers[url] - # TODO - # As a further improvement, we might want to set autocheckpoint to 0 - # here and do the checkpoints manually in a separate thread, to avoid - # any blocks in the main thread (we should run a loopingcall from here) - db_handle.cursor().execute('PRAGMA wal_autocheckpoint=50') + # stop the encryption pool + if self._sync_enc_pool is not None: + self._sync_enc_pool.close() + self._sync_enc_pool = None - # Extra query methods: extensions to the base sqlite implmentation. + # close the sync database + if self._sync_db is not None: + self._sync_db.close() + self._sync_db = None - def get_count_from_index(self, index_name, *key_values): - """ - Returns the count for a given combination of index_name - and key values. - Extension method made from similar methods in u1db version 13.09 +class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase): + """ + A very simple wrapper for u1db around sqlcipher backend. - :param index_name: The index to query - :type index_name: str - :param key_values: values to match. eg, if you have - an index with 3 fields then you would have: - get_from_index(index_name, val1, val2, val3) - :type key_values: tuple - :return: count. - :rtype: int - """ - c = self._db_handle.cursor() - definition = self._get_index_definition(index_name) + Instead of initializing the database on the fly, it just uses an existing + connection that is passed to it in the initializer. - if len(key_values) != len(definition): - raise u1db_errors.InvalidValueForIndex() - tables = ["document_fields d%d" % i for i in range(len(definition))] - novalue_where = ["d.doc_id = d%d.doc_id" - " AND d%d.field_name = ?" - % (i, i) for i in range(len(definition))] - exact_where = [novalue_where[i] - + (" AND d%d.value = ?" % (i,)) - for i in range(len(definition))] - args = [] - where = [] - for idx, (field, value) in enumerate(zip(definition, key_values)): - args.append(field) - where.append(exact_where[idx]) - args.append(value) + It can be used in tests and debug runs to initialize the adbapi with plain + sqlite connections, decoupled from the sqlcipher layer. + """ - tables = ["document_fields d%d" % i for i in range(len(definition))] - statement = ( - "SELECT COUNT(*) FROM document d, %s WHERE %s " % ( - ', '.join(tables), - ' AND '.join(where), - )) - try: - c.execute(statement, tuple(args)) - except dbapi2.OperationalError, e: - raise dbapi2.OperationalError( - str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args)) - res = c.fetchall() - return res[0][0] + def __init__(self, conn): + self._db_handle = conn + self._real_replica_uid = None + self._ensure_schema() + self._factory = u1db.Document - def close(self): - """ - Close db_handle and close syncer. - """ - logger.debug("Sqlcipher backend: closing") - if self._sync_watcher is not None: - self._sync_watcher.stop() - self._sync_watcher.shutdown() - for url in self._syncers: - _, syncer = self._syncers[url] - syncer.close() - if self._sync_enc_pool is not None: - self._sync_enc_pool.close() - if self._db_handle is not None: - self._db_handle.close() - @property - def replica_uid(self): - return self._get_replica_uid() +class SoledadSQLCipherWrapper(SQLCipherDatabase): + """ + A wrapper for u1db that uses the Soledad-extended sqlcipher backend. + + Instead of initializing the database on the fly, it just uses an existing + connection that is passed to it in the initializer. + + It can be used from adbapi to initialize a soledad database after + getting a regular connection to a sqlcipher database. + """ + def __init__(self, conn): + self._db_handle = conn + self._real_replica_uid = None + self._ensure_schema() + self.set_document_factory(soledad_doc_factory) + self._prime_replica_uid() + + +def _assert_db_is_encrypted(opts): + """ + Assert that the sqlcipher file contains an encrypted database. + + When opening an existing database, PRAGMA key will not immediately + throw an error if the key provided is incorrect. To test that the + database can be successfully opened with the provided key, it is + necessary to perform some operation on the database (i.e. read from + it) and confirm it is success. + + The easiest way to do this is select off the sqlite_master table, + which will attempt to read the first page of the database and will + parse the schema. + + :param opts: + """ + # We try to open an encrypted database with the regular u1db + # backend should raise a DatabaseError exception. + # If the regular backend succeeds, then we need to stop because + # the database was not properly initialized. + try: + sqlite_backend.SQLitePartialExpandDatabase(opts.path) + except sqlcipher_dbapi2.DatabaseError: + # assert that we can access it using SQLCipher with the given + # key + dummy_query = ('SELECT count(*) FROM sqlite_master',) + initialize_sqlcipher_db(opts, on_init=dummy_query) + else: + raise DatabaseIsNotEncrypted() + +# +# Exceptions +# +class DatabaseIsNotEncrypted(Exception): + """ + Exception raised when trying to open non-encrypted databases. + """ + pass + + +def soledad_doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False, + syncable=True): + """ + Return a default Soledad Document. + Used in the initialization for SQLCipherDatabase + """ + return SoledadDocument(doc_id=doc_id, rev=rev, json=json, + has_conflicts=has_conflicts, syncable=syncable) + sqlite_backend.SQLiteDatabase.register_implementation(SQLCipherDatabase) + + +# +# twisted.enterprise.adbapi SQLCipher implementation +# + +SQLCIPHER_CONNECTION_TIMEOUT = 10 + + +def getConnectionPool(opts, extra_queries=None): + openfun = partial( + pragmas.set_init_pragmas, + opts=opts, + extra_queries=extra_queries) + return SQLCipherConnectionPool( + database=opts.path, + check_same_thread=False, + cp_openfun=openfun, + timeout=SQLCIPHER_CONNECTION_TIMEOUT) + + +class SQLCipherConnection(adbapi.Connection): + pass + + +class SQLCipherTransaction(adbapi.Transaction): + pass + + +class SQLCipherConnectionPool(adbapi.ConnectionPool): + + connectionFactory = SQLCipherConnection + transactionFactory = SQLCipherTransaction + + def __init__(self, *args, **kwargs): + adbapi.ConnectionPool.__init__( + self, "pysqlcipher.dbapi2", *args, **kwargs) diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py index 5d545a77..53172f31 100644 --- a/client/src/leap/soledad/client/sync.py +++ b/client/src/leap/soledad/client/sync.py @@ -14,26 +14,12 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. - - """ Soledad synchronization utilities. - - -Extend u1db Synchronizer with the ability to: - - * Defer the update of the known replica uid until all the decryption of - the incoming messages has been processed. - - * Be interrupted and recovered. """ - - -import json - import logging -import traceback -from threading import Lock + +from twisted.internet import defer from u1db import errors from u1db.sync import Synchronizer @@ -54,15 +40,8 @@ class SoledadSynchronizer(Synchronizer): Also modified to allow for interrupting the synchronization process. """ - syncing_lock = Lock() - - def stop(self): - """ - Stop the current sync in progress. - """ - self.sync_target.stop() - - def sync(self, autocreate=False, defer_decryption=True): + @defer.inlineCallbacks + def sync(self, defer_decryption=True): """ Synchronize documents between source and target. @@ -74,45 +53,22 @@ class SoledadSynchronizer(Synchronizer): This is done to allow the ongoing parallel decryption of the incoming docs to proceed without `InvalidGeneration` conflicts. - :param autocreate: Whether the target replica should be created or not. - :type autocreate: bool :param defer_decryption: Whether to defer the decryption process using the intermediate database. If False, decryption will be done inline. :type defer_decryption: bool - """ - self.syncing_lock.acquire() - try: - return self._sync(autocreate=autocreate, - defer_decryption=defer_decryption) - except Exception: - # re-raising the exceptions to let syqlcipher.sync catch them - # (and re-create the syncer instance if needed) - raise - finally: - self.release_syncing_lock() - - def _sync(self, autocreate=False, defer_decryption=True): - """ - Helper function, called from the main `sync` method. - See `sync` docstring. + + :return: A deferred which will fire after the sync has finished. + :rtype: twisted.internet.defer.Deferred """ sync_target = self.sync_target # get target identifier, its current generation, # and its last-seen database generation for this source ensure_callback = None - try: - (self.target_replica_uid, target_gen, target_trans_id, - target_my_gen, target_my_trans_id) = \ - sync_target.get_sync_info(self.source._replica_uid) - except errors.DatabaseDoesNotExist: - if not autocreate: - raise - # will try to ask sync_exchange() to create the db - self.target_replica_uid = None - target_gen, target_trans_id = (0, '') - target_my_gen, target_my_trans_id = (0, '') + (self.target_replica_uid, target_gen, target_trans_id, + target_my_gen, target_my_trans_id) = yield \ + sync_target.get_sync_info(self.source._replica_uid) logger.debug( "Soledad target sync info:\n" @@ -120,9 +76,10 @@ class SoledadSynchronizer(Synchronizer): " target generation: %d\n" " target trans id: %s\n" " target my gen: %d\n" - " target my trans_id: %s" + " target my trans_id: %s\n" + " source replica_uid: %s\n" % (self.target_replica_uid, target_gen, target_trans_id, - target_my_gen, target_my_trans_id)) + target_my_gen, target_my_trans_id, self.source._replica_uid)) # make sure we'll have access to target replica uid once it exists if self.target_replica_uid is None: @@ -140,7 +97,7 @@ class SoledadSynchronizer(Synchronizer): # what's changed since that generation and this current gen my_gen, _, changes = self.source.whats_changed(target_my_gen) - logger.debug("Soledad sync: there are %d documents to send." \ + logger.debug("Soledad sync: there are %d documents to send." % len(changes)) # get source last-seen database generation for the target @@ -152,15 +109,15 @@ class SoledadSynchronizer(Synchronizer): self.target_replica_uid) logger.debug( "Soledad source sync info:\n" - " source target gen: %d\n" - " source target trans_id: %s" + " last target gen known to source: %d\n" + " last target trans_id known to source: %s" % (target_last_known_gen, target_last_known_trans_id)) # validate transaction ids if not changes and target_last_known_gen == target_gen: if target_trans_id != target_last_known_trans_id: raise errors.InvalidTransactionId - return my_gen + defer.returnValue(my_gen) # prepare to send all the changed docs changed_doc_ids = [doc_id for doc_id, _, _ in changes] @@ -175,40 +132,26 @@ class SoledadSynchronizer(Synchronizer): # exchange documents and try to insert the returned ones with # the target, return target synced-up-to gen. - # - # The sync_exchange method may be interrupted, in which case it will - # return a tuple of Nones. - try: - new_gen, new_trans_id = sync_target.sync_exchange( - docs_by_generation, self.source._replica_uid, - target_last_known_gen, target_last_known_trans_id, - self._insert_doc_from_target, ensure_callback=ensure_callback, - defer_decryption=defer_decryption) - logger.debug( - "Soledad source sync info after sync exchange:\n" - " source target gen: %d\n" - " source target trans_id: %s" - % (new_gen, new_trans_id)) - info = { - "target_replica_uid": self.target_replica_uid, - "new_gen": new_gen, - "new_trans_id": new_trans_id, - "my_gen": my_gen - } - self._syncing_info = info - if defer_decryption and not sync_target.has_syncdb(): - logger.debug("Sync target has no valid sync db, " - "aborting defer_decryption") - defer_decryption = False - self.complete_sync() - except Exception as e: - logger.error("Soledad sync error: %s" % str(e)) - logger.error(traceback.format_exc()) - sync_target.stop() - finally: - sync_target.close() - - return my_gen + new_gen, new_trans_id = yield sync_target.sync_exchange( + docs_by_generation, self.source._replica_uid, + target_last_known_gen, target_last_known_trans_id, + self._insert_doc_from_target, ensure_callback=ensure_callback, + defer_decryption=defer_decryption) + logger.debug( + "Soledad source sync info after sync exchange:\n" + " source known target gen: %d\n" + " source known target trans_id: %s" + % (new_gen, new_trans_id)) + info = { + "target_replica_uid": self.target_replica_uid, + "new_gen": new_gen, + "new_trans_id": new_trans_id, + "my_gen": my_gen + } + self._syncing_info = info + yield self.complete_sync() + + defer.returnValue(my_gen) def complete_sync(self): """ @@ -216,6 +159,9 @@ class SoledadSynchronizer(Synchronizer): (a) record last known generation and transaction uid for the remote replica, and (b) make target aware of our current reached generation. + + :return: A deferred which will fire when the sync has been completed. + :rtype: twisted.internet.defer.Deferred """ logger.debug("Completing deferred last step in SYNC...") @@ -226,37 +172,23 @@ class SoledadSynchronizer(Synchronizer): info["target_replica_uid"], info["new_gen"], info["new_trans_id"]) # if gapless record current reached generation with target - self._record_sync_info_with_the_target(info["my_gen"]) + return self._record_sync_info_with_the_target(info["my_gen"]) - @property - def syncing(self): + def _record_sync_info_with_the_target(self, start_generation): """ - Return True if a sync is ongoing, False otherwise. - :rtype: bool - """ - # XXX FIXME we need some mechanism for timeout: should cleanup and - # release if something in the syncdb-decrypt goes wrong. we could keep - # track of the release date and cleanup unrealistic sync entries after - # some time. - locked = self.syncing_lock.locked() - return locked - - def release_syncing_lock(self): - """ - Release syncing lock if it's locked. - """ - if self.syncing_lock.locked(): - self.syncing_lock.release() + Store local replica metadata in server. - def close(self): - """ - Close sync target pool of workers. - """ - self.release_syncing_lock() - self.sync_target.close() + :param start_generation: The local generation when the sync was + started. + :type start_generation: int - def __del__(self): - """ - Cleanup: release lock. + :return: A deferred which will fire when the operation has been + completed. + :rtype: twisted.internet.defer.Deferred """ - self.release_syncing_lock() + cur_gen, trans_id = self.source._get_generation_info() + if (cur_gen == start_generation + self.num_inserted + and self.num_inserted > 0): + return self.sync_target.record_sync_info( + self.source._replica_uid, cur_gen, trans_id) + return defer.succeed(None) diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py deleted file mode 100644 index 1eb84e64..00000000 --- a/client/src/leap/soledad/client/target.py +++ /dev/null @@ -1,1469 +0,0 @@ -# -*- coding: utf-8 -*- -# target.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -A U1DB backend for encrypting data before sending to server and decrypting -after receiving. -""" - - -import cStringIO -import gzip -import logging -import re -import urllib -import threading - -from collections import defaultdict -from time import sleep -from uuid import uuid4 - -import simplejson as json -from taskthread import TimerTask -from u1db import errors -from u1db.remote import utils, http_errors -from u1db.remote.http_target import HTTPSyncTarget -from u1db.remote.http_client import _encode_query_parameter, HTTPClientBase -from zope.proxy import ProxyBase -from zope.proxy import sameProxiedObjects, setProxiedObject - -from leap.soledad.common.document import SoledadDocument -from leap.soledad.client.auth import TokenBasedAuth -from leap.soledad.client.crypto import is_symmetrically_encrypted -from leap.soledad.client.crypto import encrypt_doc, decrypt_doc -from leap.soledad.client.crypto import SyncEncrypterPool, SyncDecrypterPool -from leap.soledad.client.events import SOLEDAD_SYNC_SEND_STATUS -from leap.soledad.client.events import SOLEDAD_SYNC_RECEIVE_STATUS -from leap.soledad.client.events import signal - - -logger = logging.getLogger(__name__) - - -def _gunzip(data): - """ - Uncompress data that is gzipped. - - :param data: gzipped data - :type data: basestring - """ - buffer = cStringIO.StringIO() - buffer.write(data) - buffer.seek(0) - try: - data = gzip.GzipFile(mode='r', fileobj=buffer).read() - except Exception: - logger.warning("Error while decrypting gzipped data") - buffer.close() - return data - - -class PendingReceivedDocsSyncError(Exception): - pass - - -class DocumentSyncerThread(threading.Thread): - """ - A thread that knowns how to either send or receive a document during the - sync process. - """ - - def __init__(self, doc_syncer, release_method, failed_method, - idx, total, last_request_lock=None, last_callback_lock=None): - """ - Initialize a new syncer thread. - - :param doc_syncer: A document syncer. - :type doc_syncer: HTTPDocumentSyncer - :param release_method: A method to be called when finished running. - :type release_method: callable(DocumentSyncerThread) - :param failed_method: A method to be called when we failed. - :type failed_method: callable(DocumentSyncerThread) - :param idx: The index count of the current operation. - :type idx: int - :param total: The total number of operations. - :type total: int - :param last_request_lock: A lock to wait for before actually performing - the request. - :type last_request_lock: threading.Lock - :param last_callback_lock: A lock to wait for before actually running - the success callback. - :type last_callback_lock: threading.Lock - """ - threading.Thread.__init__(self) - self._doc_syncer = doc_syncer - self._release_method = release_method - self._failed_method = failed_method - self._idx = idx - self._total = total - self._last_request_lock = last_request_lock - self._last_callback_lock = last_callback_lock - self._response = None - self._exception = None - self._result = None - self._success = False - # a lock so we can signal when we're finished - self._request_lock = threading.Lock() - self._request_lock.acquire() - self._callback_lock = threading.Lock() - self._callback_lock.acquire() - # make thread interruptable - self._stopped = None - self._stop_lock = threading.Lock() - - def run(self): - """ - Run the HTTP request and store results. - - This method will block and wait for an eventual previous operation to - finish before actually performing the request. It also traps any - exception and register any failure with the request. - """ - with self._stop_lock: - if self._stopped is None: - self._stopped = False - else: - return - - # eventually wait for the previous thread to finish - if self._last_request_lock is not None: - self._last_request_lock.acquire() - - # bail out in case we've been interrupted - if self.stopped is True: - return - - try: - self._response = self._doc_syncer.do_request() - self._request_lock.release() - - # run success callback - if self._doc_syncer.success_callback is not None: - - # eventually wait for callback lock release - if self._last_callback_lock is not None: - self._last_callback_lock.acquire() - - # bail out in case we've been interrupted - if self._stopped is True: - return - - self._result = self._doc_syncer.success_callback( - self._idx, self._total, self._response) - self._success = True - doc_syncer = self._doc_syncer - self._release_method(self, doc_syncer) - self._doc_syncer = None - # let next thread executed its callback - self._callback_lock.release() - - # trap any exception and signal failure - except Exception as e: - self._exception = e - self._success = False - # run failure callback - if self._doc_syncer.failure_callback is not None: - - # eventually wait for callback lock release - if self._last_callback_lock is not None: - self._last_callback_lock.acquire() - - # bail out in case we've been interrupted - if self.stopped is True: - return - - self._doc_syncer.failure_callback( - self._idx, self._total, self._exception) - - self._failed_method(self) - # we do not release the callback lock here because we - # failed and so we don't want other threads to succeed. - - @property - def doc_syncer(self): - return self._doc_syncer - - @property - def response(self): - return self._response - - @property - def exception(self): - return self._exception - - @property - def callback_lock(self): - return self._callback_lock - - @property - def request_lock(self): - return self._request_lock - - @property - def success(self): - return self._success - - def stop(self): - with self._stop_lock: - self._stopped = True - - @property - def stopped(self): - with self._stop_lock: - return self._stopped - - @property - def result(self): - return self._result - - -class DocumentSyncerPool(object): - """ - A pool of reusable document syncers. - """ - - POOL_SIZE = 10 - """ - The maximum amount of syncer threads running at the same time. - """ - - def __init__(self, raw_url, raw_creds, query_string, headers, - ensure_callback, stop_method): - """ - Initialize the document syncer pool. - - :param raw_url: The complete raw URL for the HTTP request. - :type raw_url: str - :param raw_creds: The credentials for the HTTP request. - :type raw_creds: dict - :param query_string: The query string for the HTTP request. - :type query_string: str - :param headers: The headers for the HTTP request. - :type headers: dict - :param ensure_callback: A callback to ensure we have the correct - target_replica_uid, if it was just created. - :type ensure_callback: callable - - """ - # save syncer params - self._raw_url = raw_url - self._raw_creds = raw_creds - self._query_string = query_string - self._headers = headers - self._ensure_callback = ensure_callback - self._stop_method = stop_method - # pool attributes - self._failures = False - self._semaphore_pool = threading.BoundedSemaphore( - DocumentSyncerPool.POOL_SIZE) - self._pool_access_lock = threading.Lock() - self._doc_syncers = [] - self._threads = [] - - def new_syncer_thread(self, idx, total, last_request_lock=None, - last_callback_lock=None): - """ - Yield a new document syncer thread. - - :param idx: The index count of the current operation. - :type idx: int - :param total: The total number of operations. - :type total: int - :param last_request_lock: A lock to wait for before actually performing - the request. - :type last_request_lock: threading.Lock - :param last_callback_lock: A lock to wait for before actually running - the success callback. - :type last_callback_lock: threading.Lock - """ - t = None - # wait for available threads - self._semaphore_pool.acquire() - with self._pool_access_lock: - if self._failures is True: - return None - # get a syncer - doc_syncer = self._get_syncer() - # we rely on DocumentSyncerThread.run() to release the lock using - # self.release_syncer so we can launch a new thread. - t = DocumentSyncerThread( - doc_syncer, self.release_syncer, self.cancel_threads, - idx, total, - last_request_lock=last_request_lock, - last_callback_lock=last_callback_lock) - self._threads.append(t) - return t - - def _failed(self): - with self._pool_access_lock: - self._failures = True - - @property - def failures(self): - return self._failures - - def _get_syncer(self): - """ - Get a document syncer from the pool. - - This method will create a new syncer whenever there is no syncer - available in the pool. - - :return: A syncer. - :rtype: HTTPDocumentSyncer - """ - syncer = None - # get an available syncer or create a new one - try: - syncer = self._doc_syncers.pop() - except IndexError: - syncer = HTTPDocumentSyncer( - self._raw_url, self._raw_creds, self._query_string, - self._headers, self._ensure_callback) - return syncer - - def release_syncer(self, syncer_thread, doc_syncer): - """ - Return a syncer to the pool after use and check for any failures. - - :param syncer: The syncer to be returned to the pool. - :type syncer: HTTPDocumentSyncer - """ - with self._pool_access_lock: - self._doc_syncers.append(doc_syncer) - if syncer_thread.success is True: - self._threads.remove(syncer_thread) - self._semaphore_pool.release() - - def cancel_threads(self, calling_thread): - """ - Stop all threads in the pool. - """ - # stop sync - self._stop_method() - stopped = [] - # stop all threads - logger.warning("Soledad sync: cancelling sync threads...") - with self._pool_access_lock: - self._failures = True - while self._threads: - t = self._threads.pop(0) - t.stop() - self._doc_syncers.append(t.doc_syncer) - stopped.append(t) - # release locks and join - while stopped: - t = stopped.pop(0) - t.request_lock.acquire(False) # just in case - t.request_lock.release() - t.callback_lock.acquire(False) # just in case - t.callback_lock.release() - logger.warning("Soledad sync: cancelled sync threads.") - - def cleanup(self): - """ - Close and remove any syncers from the pool. - """ - with self._pool_access_lock: - while self._doc_syncers: - syncer = self._doc_syncers.pop() - syncer.close() - del syncer - - -class HTTPDocumentSyncer(HTTPClientBase, TokenBasedAuth): - - def __init__(self, raw_url, creds, query_string, headers, ensure_callback): - """ - Initialize the client. - - :param raw_url: The raw URL of the target HTTP server. - :type raw_url: str - :param creds: Authentication credentials. - :type creds: dict - :param query_string: The query string for the HTTP request. - :type query_string: str - :param headers: The headers for the HTTP request. - :type headers: dict - :param ensure_callback: A callback to ensure we have the correct - target_replica_uid, if it was just created. - :type ensure_callback: callable - """ - HTTPClientBase.__init__(self, raw_url, creds=creds) - # info needed to perform the request - self._query_string = query_string - self._headers = headers - self._ensure_callback = ensure_callback - # the actual request method - self._request_method = None - self._success_callback = None - self._failure_callback = None - - def _reset(self): - """ - Reset this document syncer so we can reuse it. - """ - self._request_method = None - self._success_callback = None - self._failure_callback = None - self._request_method = None - - def set_request_method(self, method, *args, **kwargs): - """ - Set the actual method to perform the request. - - :param method: Either 'get' or 'put'. - :type method: str - :param args: Arguments for the request method. - :type args: list - :param kwargs: Keyworded arguments for the request method. - :type kwargs: dict - """ - self._reset() - # resolve request method - if method is 'get': - self._request_method = self._get_doc - elif method is 'put': - self._request_method = self._put_doc - else: - raise Exception - # store request method args - self._args = args - self._kwargs = kwargs - - def set_success_callback(self, callback): - self._success_callback = callback - - def set_failure_callback(self, callback): - self._failure_callback = callback - - @property - def success_callback(self): - return self._success_callback - - @property - def failure_callback(self): - return self._failure_callback - - def do_request(self): - """ - Actually perform the request. - - :return: The body and headers of the response. - :rtype: tuple - """ - self._ensure_connection() - args = self._args - kwargs = self._kwargs - return self._request_method(*args, **kwargs) - - def _request(self, method, url_parts, params=None, body=None, - content_type=None): - """ - Perform an HTTP request. - - :param method: The HTTP request method. - :type method: str - :param url_parts: A list representing the request path. - :type url_parts: list - :param params: Parameters for the URL query string. - :type params: dict - :param body: The body of the request. - :type body: str - :param content-type: The content-type of the request. - :type content-type: str - - :return: The body and headers of the response. - :rtype: tuple - - :raise errors.Unavailable: Raised after a number of unsuccesful - request attempts. - :raise Exception: Raised for any other exception ocurring during the - request. - """ - - self._ensure_connection() - unquoted_url = url_query = self._url.path - if url_parts: - if not url_query.endswith('/'): - url_query += '/' - unquoted_url = url_query - url_query += '/'.join(urllib.quote(part, safe='') - for part in url_parts) - # oauth performs its own quoting - unquoted_url += '/'.join(url_parts) - encoded_params = {} - if params: - for key, value in params.items(): - key = unicode(key).encode('utf-8') - encoded_params[key] = _encode_query_parameter(value) - url_query += ('?' + urllib.urlencode(encoded_params)) - if body is not None and not isinstance(body, basestring): - body = json.dumps(body) - content_type = 'application/json' - headers = {} - if content_type: - headers['content-type'] = content_type - - # Patched: We would like to receive gzip pretty please - # ---------------------------------------------------- - headers['accept-encoding'] = "gzip" - # ---------------------------------------------------- - - headers.update( - self._sign_request(method, unquoted_url, encoded_params)) - - for delay in self._delays: - try: - self._conn.request(method, url_query, body, headers) - return self._response() - except errors.Unavailable, e: - sleep(delay) - raise e - - def _response(self): - """ - Return the response of the (possibly gzipped) HTTP request. - - :return: The body and headers of the response. - :rtype: tuple - """ - resp = self._conn.getresponse() - body = resp.read() - headers = dict(resp.getheaders()) - - # Patched: We would like to decode gzip - # ---------------------------------------------------- - encoding = headers.get('content-encoding', '') - if "gzip" in encoding: - body = _gunzip(body) - # ---------------------------------------------------- - - if resp.status in (200, 201): - return body, headers - elif resp.status in http_errors.ERROR_STATUSES: - try: - respdic = json.loads(body) - except ValueError: - pass - else: - self._error(respdic) - # special case - if resp.status == 503: - raise errors.Unavailable(body, headers) - raise errors.HTTPError(resp.status, body, headers) - - def _prepare(self, comma, entries, **dic): - """ - Prepare an entry to be sent through a syncing POST request. - - :param comma: A string to be prepended to the current entry. - :type comma: str - :param entries: A list of entries accumulated to be sent on the - request. - :type entries: list - :param dic: The data to be included in this entry. - :type dic: dict - - :return: The size of the prepared entry. - :rtype: int - """ - entry = comma + '\r\n' + json.dumps(dic) - entries.append(entry) - return len(entry) - - def _init_post_request(self, action, content_length): - """ - Initiate a syncing POST request. - - :param url: The syncing URL. - :type url: str - :param action: The syncing action, either 'get' or 'receive'. - :type action: str - :param headers: The initial headers to be sent on this request. - :type headers: dict - :param content_length: The content-length of the request. - :type content_length: int - """ - self._conn.putrequest('POST', self._query_string) - self._conn.putheader( - 'content-type', 'application/x-soledad-sync-%s' % action) - for header_name, header_value in self._headers: - self._conn.putheader(header_name, header_value) - self._conn.putheader('accept-encoding', 'gzip') - self._conn.putheader('content-length', str(content_length)) - self._conn.endheaders() - - def _get_doc(self, received, sync_id, last_known_generation, - last_known_trans_id): - """ - Get a sync document from server by means of a POST request. - - :param received: The number of documents already received in the - current sync session. - :type received: int - :param sync_id: The id for the current sync session. - :type sync_id: str - :param last_known_generation: Target's last known generation. - :type last_known_generation: int - :param last_known_trans_id: Target's last known transaction id. - :type last_known_trans_id: str - - :return: The body and headers of the response. - :rtype: tuple - """ - entries = ['['] - size = 1 - # add remote replica metadata to the request - size += self._prepare( - '', entries, - last_known_generation=last_known_generation, - last_known_trans_id=last_known_trans_id, - sync_id=sync_id, - ensure=self._ensure_callback is not None) - # inform server of how many documents have already been received - size += self._prepare( - ',', entries, received=received) - entries.append('\r\n]') - size += len(entries[-1]) - # send headers - self._init_post_request('get', size) - # get document - for entry in entries: - self._conn.send(entry) - return self._response() - - def _put_doc(self, sync_id, last_known_generation, last_known_trans_id, - id, rev, content, gen, trans_id, number_of_docs, doc_idx): - """ - Put a sync document on server by means of a POST request. - - :param sync_id: The id for the current sync session. - :type sync_id: str - :param last_known_generation: Target's last known generation. - :type last_known_generation: int - :param last_known_trans_id: Target's last known transaction id. - :type last_known_trans_id: str - :param id: The document id. - :type id: str - :param rev: The document revision. - :type rev: str - :param content: The serialized document content. - :type content: str - :param gen: The generation of the modification of the document. - :type gen: int - :param trans_id: The transaction id of the modification of the - document. - :type trans_id: str - :param number_of_docs: The total amount of documents sent on this sync - session. - :type number_of_docs: int - :param doc_idx: The index of the current document being sent. - :type doc_idx: int - - :return: The body and headers of the response. - :rtype: tuple - """ - # prepare to send the document - entries = ['['] - size = 1 - # add remote replica metadata to the request - size += self._prepare( - '', entries, - last_known_generation=last_known_generation, - last_known_trans_id=last_known_trans_id, - sync_id=sync_id, - ensure=self._ensure_callback is not None) - # add the document to the request - size += self._prepare( - ',', entries, - id=id, rev=rev, content=content, gen=gen, trans_id=trans_id, - number_of_docs=number_of_docs, doc_idx=doc_idx) - entries.append('\r\n]') - size += len(entries[-1]) - # send headers - self._init_post_request('put', size) - # send document - for entry in entries: - self._conn.send(entry) - return self._response() - - def _sign_request(self, method, url_query, params): - """ - Return an authorization header to be included in the HTTP request. - - :param method: The HTTP method. - :type method: str - :param url_query: The URL query string. - :type url_query: str - :param params: A list with encoded query parameters. - :type param: list - - :return: The Authorization header. - :rtype: list of tuple - """ - return TokenBasedAuth._sign_request(self, method, url_query, params) - - def set_token_credentials(self, uuid, token): - """ - Store given credentials so we can sign the request later. - - :param uuid: The user's uuid. - :type uuid: str - :param token: The authentication token. - :type token: str - """ - TokenBasedAuth.set_token_credentials(self, uuid, token) - - -class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): - """ - A SyncTarget that encrypts data before sending and decrypts data after - receiving. - - Normally encryption will have been written to the sync database upon - document modification. The sync database is also used to write temporarily - the parsed documents that the remote send us, before being decrypted and - written to the main database. - """ - - # will later keep a reference to the insert-doc callback - # passed to sync_exchange - _insert_doc_cb = defaultdict(lambda: ProxyBase(None)) - - """ - Period of recurrence of the periodic decrypting task, in seconds. - """ - DECRYPT_TASK_PERIOD = 0.5 - - # - # Modified HTTPSyncTarget methods. - # - - def __init__(self, url, source_replica_uid=None, creds=None, crypto=None, - sync_db=None, sync_db_write_lock=None): - """ - Initialize the SoledadSyncTarget. - - :param source_replica_uid: The source replica uid which we use when - deferring decryption. - :type source_replica_uid: str - :param url: The url of the target replica to sync with. - :type url: str - :param creds: Optional dictionary giving credentials. - to authorize the operation with the server. - :type creds: dict - :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. - :type crypto: soledad.crypto.SoledadCrypto - :param sync_db: Optional. handler for the db with the symmetric - encryption of the syncing documents. If - None, encryption will be done in-place, - instead of retreiving it from the dedicated - database. - :type sync_db: Sqlite handler - :param sync_db_write_lock: a write lock for controlling concurrent - access to the sync_db - :type sync_db_write_lock: threading.Lock - """ - HTTPSyncTarget.__init__(self, url, creds) - self._raw_url = url - self._raw_creds = creds - self._crypto = crypto - self._stopped = True - self._stop_lock = threading.Lock() - self._sync_exchange_lock = threading.Lock() - self.source_replica_uid = source_replica_uid - self._defer_decryption = False - - # deferred decryption attributes - self._sync_db = None - self._sync_db_write_lock = None - self._decryption_callback = None - self._sync_decr_pool = None - self._sync_watcher = None - if sync_db and sync_db_write_lock is not None: - self._sync_db = sync_db - self._sync_db_write_lock = sync_db_write_lock - - def _setup_sync_decr_pool(self): - """ - Set up the SyncDecrypterPool for deferred decryption. - """ - if self._sync_decr_pool is None: - # initialize syncing queue decryption pool - self._sync_decr_pool = SyncDecrypterPool( - self._crypto, self._sync_db, - self._sync_db_write_lock, - insert_doc_cb=self._insert_doc_cb) - self._sync_decr_pool.set_source_replica_uid( - self.source_replica_uid) - - def _teardown_sync_decr_pool(self): - """ - Tear down the SyncDecrypterPool. - """ - if self._sync_decr_pool is not None: - self._sync_decr_pool.close() - self._sync_decr_pool = None - - def _setup_sync_watcher(self): - """ - Set up the sync watcher for deferred decryption. - """ - if self._sync_watcher is None: - self._sync_watcher = TimerTask( - self._decrypt_syncing_received_docs, - delay=self.DECRYPT_TASK_PERIOD) - - def _teardown_sync_watcher(self): - """ - Tear down the sync watcher. - """ - if self._sync_watcher is not None: - self._sync_watcher.stop() - self._sync_watcher.shutdown() - self._sync_watcher = None - - def _get_replica_uid(self, url): - """ - Return replica uid from the url, or None. - - :param url: the replica url - :type url: str - """ - replica_uid_match = re.findall("user-([0-9a-fA-F]+)", url) - return replica_uid_match[0] if len(replica_uid_match) > 0 else None - - @staticmethod - def connect(url, source_replica_uid=None, crypto=None): - return SoledadSyncTarget( - url, source_replica_uid=source_replica_uid, crypto=crypto) - - def _parse_received_doc_response(self, response): - """ - Parse the response from the server containing the received document. - - :param response: The body and headers of the response. - :type response: tuple(str, dict) - """ - data, _ = response - # decode incoming stream - parts = data.splitlines() - if not parts or parts[0] != '[' or parts[-1] != ']': - raise errors.BrokenSyncStream - data = parts[1:-1] - # decode metadata - line, comma = utils.check_and_strip_comma(data[0]) - metadata = None - try: - metadata = json.loads(line) - new_generation = metadata['new_generation'] - new_transaction_id = metadata['new_transaction_id'] - number_of_changes = metadata['number_of_changes'] - except (json.JSONDecodeError, KeyError): - raise errors.BrokenSyncStream - # make sure we have replica_uid from fresh new dbs - if self._ensure_callback and 'replica_uid' in metadata: - self._ensure_callback(metadata['replica_uid']) - # parse incoming document info - doc_id = None - rev = None - content = None - gen = None - trans_id = None - if number_of_changes > 0: - try: - entry = json.loads(data[1]) - doc_id = entry['id'] - rev = entry['rev'] - content = entry['content'] - gen = entry['gen'] - trans_id = entry['trans_id'] - except (IndexError, KeyError): - raise errors.BrokenSyncStream - return new_generation, new_transaction_id, number_of_changes, \ - doc_id, rev, content, gen, trans_id - - def _insert_received_doc(self, idx, total, response): - """ - Insert a received document into the local replica. - - :param idx: The index count of the current operation. - :type idx: int - :param total: The total number of operations. - :type total: int - :param response: The body and headers of the response. - :type response: tuple(str, dict) - """ - new_generation, new_transaction_id, number_of_changes, doc_id, \ - rev, content, gen, trans_id = \ - self._parse_received_doc_response(response) - if doc_id is not None: - # decrypt incoming document and insert into local database - # ------------------------------------------------------------- - # symmetric decryption of document's contents - # ------------------------------------------------------------- - # If arriving content was symmetrically encrypted, we decrypt it. - # We do it inline if defer_decryption flag is False or no sync_db - # was defined, otherwise we defer it writing it to the received - # docs table. - doc = SoledadDocument(doc_id, rev, content) - if is_symmetrically_encrypted(doc): - if self._queue_for_decrypt: - self._save_encrypted_received_doc( - doc, gen, trans_id, idx, total) - else: - # defer_decryption is False or no-sync-db fallback - doc.set_json(decrypt_doc(self._crypto, doc)) - self._return_doc_cb(doc, gen, trans_id) - else: - # not symmetrically encrypted doc, insert it directly - # or save it in the decrypted stage. - if self._queue_for_decrypt: - self._save_received_doc(doc, gen, trans_id, idx, total) - else: - self._return_doc_cb(doc, gen, trans_id) - # ------------------------------------------------------------- - # end of symmetric decryption - # ------------------------------------------------------------- - msg = "%d/%d" % (idx + 1, total) - signal(SOLEDAD_SYNC_RECEIVE_STATUS, msg) - logger.debug("Soledad sync receive status: %s" % msg) - return number_of_changes, new_generation, new_transaction_id - - def _get_remote_docs(self, url, last_known_generation, last_known_trans_id, - headers, return_doc_cb, ensure_callback, sync_id, - syncer_pool, defer_decryption=False): - """ - Fetch sync documents from the remote database and insert them in the - local database. - - If an incoming document's encryption scheme is equal to - EncryptionSchemes.SYMKEY, then this method will decrypt it with - Soledad's symmetric key. - - :param url: The syncing URL. - :type url: str - :param last_known_generation: Target's last known generation. - :type last_known_generation: int - :param last_known_trans_id: Target's last known transaction id. - :type last_known_trans_id: str - :param headers: The headers of the HTTP request. - :type headers: dict - :param return_doc_cb: A callback to insert docs from target. - :type return_doc_cb: callable - :param ensure_callback: A callback to ensure we have the correct - target_replica_uid, if it was just created. - :type ensure_callback: callable - :param sync_id: The id for the current sync session. - :type sync_id: str - :param defer_decryption: Whether to defer the decryption process using - the intermediate database. If False, - decryption will be done inline. - :type defer_decryption: bool - - :raise BrokenSyncStream: If `data` is malformed. - - :return: A dictionary representing the first line of the response got - from remote replica. - :rtype: dict - """ - # we keep a reference to the callback in case we defer the decryption - self._return_doc_cb = return_doc_cb - self._queue_for_decrypt = defer_decryption \ - and self._sync_db is not None - - new_generation = last_known_generation - new_transaction_id = last_known_trans_id - - if self._queue_for_decrypt: - logger.debug( - "Soledad sync: will queue received docs for decrypting.") - - idx = 0 - number_of_changes = 1 - - first_request = True - last_callback_lock = None - threads = [] - - # get incoming documents - while idx < number_of_changes: - # bail out if sync process was interrupted - if self.stopped is True: - break - - # launch a thread to fetch one document from target - t = syncer_pool.new_syncer_thread( - idx, number_of_changes, - last_callback_lock=last_callback_lock) - - # bail out if any thread failed - if t is None: - self.stop() - break - - t.doc_syncer.set_request_method( - 'get', idx, sync_id, last_known_generation, - last_known_trans_id) - t.doc_syncer.set_success_callback(self._insert_received_doc) - - def _failure_callback(idx, total, exception): - _failure_msg = "Soledad sync: error while getting document " \ - "%d/%d: %s" \ - % (idx + 1, total, exception) - logger.warning("%s" % _failure_msg) - logger.warning("Soledad sync: failing gracefully, will " - "recover on next sync.") - - t.doc_syncer.set_failure_callback(_failure_callback) - threads.append(t) - t.start() - last_callback_lock = t.callback_lock - idx += 1 - - # if this is the first request, wait to update the number of - # changes - if first_request is True: - t.join() - if t.success: - number_of_changes, _, _ = t.result - first_request = False - - # make sure all threads finished and we have up-to-date info - last_successful_thread = None - while threads: - # check if there are failures - t = threads.pop(0) - t.join() - if t.success: - last_successful_thread = t - - # get information about last successful thread - if last_successful_thread is not None: - body, _ = last_successful_thread.response - parsed_body = json.loads(body) - # get current target gen and trans id in case no documents were - # transferred - if len(parsed_body) == 1: - metadata = parsed_body[0] - new_generation = metadata['new_generation'] - new_transaction_id = metadata['new_transaction_id'] - # get current target gen and trans id from last transferred - # document - else: - doc_data = parsed_body[1] - new_generation = doc_data['gen'] - new_transaction_id = doc_data['trans_id'] - - return new_generation, new_transaction_id - - def sync_exchange(self, docs_by_generations, - source_replica_uid, last_known_generation, - last_known_trans_id, return_doc_cb, - ensure_callback=None, defer_decryption=True, - sync_id=None): - """ - Find out which documents the remote database does not know about, - encrypt and send them. - - This does the same as the parent's method but encrypts content before - syncing. - - :param docs_by_generations: A list of (doc_id, generation, trans_id) - of local documents that were changed since - the last local generation the remote - replica knows about. - :type docs_by_generations: list of tuples - - :param source_replica_uid: The uid of the source replica. - :type source_replica_uid: str - - :param last_known_generation: Target's last known generation. - :type last_known_generation: int - - :param last_known_trans_id: Target's last known transaction id. - :type last_known_trans_id: str - - :param return_doc_cb: A callback for inserting received documents from - target. If not overriden, this will call u1db - insert_doc_from_target in synchronizer, which - implements the TAKE OTHER semantics. - :type return_doc_cb: function - - :param ensure_callback: A callback that ensures we know the target - replica uid if the target replica was just - created. - :type ensure_callback: function - - :param defer_decryption: Whether to defer the decryption process using - the intermediate database. If False, - decryption will be done inline. - :type defer_decryption: bool - - :return: The new generation and transaction id of the target replica. - :rtype: tuple - """ - self._ensure_callback = ensure_callback - - if defer_decryption: - self._sync_exchange_lock.acquire() - self._setup_sync_decr_pool() - self._setup_sync_watcher() - self._defer_decryption = True - - self.start() - - if sync_id is None: - sync_id = str(uuid4()) - self.source_replica_uid = source_replica_uid - # let the decrypter pool access the passed callback to insert docs - setProxiedObject(self._insert_doc_cb[source_replica_uid], - return_doc_cb) - - if not self.clear_to_sync(): - raise PendingReceivedDocsSyncError - - self._ensure_connection() - if self._trace_hook: # for tests - self._trace_hook('sync_exchange') - url = '%s/sync-from/%s' % (self._url.path, source_replica_uid) - headers = self._sign_request('POST', url, {}) - - cur_target_gen = last_known_generation - cur_target_trans_id = last_known_trans_id - - # send docs - msg = "%d/%d" % (0, len(docs_by_generations)) - signal(SOLEDAD_SYNC_SEND_STATUS, msg) - logger.debug("Soledad sync send status: %s" % msg) - - defer_encryption = self._sync_db is not None - syncer_pool = DocumentSyncerPool( - self._raw_url, self._raw_creds, url, headers, ensure_callback, - self.stop) - threads = [] - last_request_lock = None - last_callback_lock = None - sent = 0 - total = len(docs_by_generations) - - synced = [] - number_of_docs = len(docs_by_generations) - - for doc, gen, trans_id in docs_by_generations: - # allow for interrupting the sync process - if self.stopped is True: - break - - # skip non-syncable docs - if isinstance(doc, SoledadDocument) and not doc.syncable: - continue - - # ------------------------------------------------------------- - # symmetric encryption of document's contents - # ------------------------------------------------------------- - doc_json = doc.get_json() - if not doc.is_tombstone(): - if not defer_encryption: - # fallback case, for tests - doc_json = encrypt_doc(self._crypto, doc) - else: - try: - doc_json = self.get_encrypted_doc_from_db( - doc.doc_id, doc.rev) - except Exception as exc: - logger.error("Error while getting " - "encrypted doc from db") - logger.exception(exc) - continue - if doc_json is None: - # Not marked as tombstone, but we got nothing - # from the sync db. As it is not encrypted yet, we - # force inline encryption. - # TODO: implement a queue to deal with these cases. - doc_json = encrypt_doc(self._crypto, doc) - # ------------------------------------------------------------- - # end of symmetric encryption - # ------------------------------------------------------------- - t = syncer_pool.new_syncer_thread( - sent + 1, total, last_request_lock=None, - last_callback_lock=last_callback_lock) - - # bail out if any thread failed - if t is None: - self.stop() - break - - # set the request method - t.doc_syncer.set_request_method( - 'put', sync_id, cur_target_gen, cur_target_trans_id, - id=doc.doc_id, rev=doc.rev, content=doc_json, gen=gen, - trans_id=trans_id, number_of_docs=number_of_docs, doc_idx=sent + 1) - # set the success calback - - def _success_callback(idx, total, response): - _success_msg = "Soledad sync send status: %d/%d" \ - % (idx, total) - signal(SOLEDAD_SYNC_SEND_STATUS, _success_msg) - logger.debug(_success_msg) - - t.doc_syncer.set_success_callback(_success_callback) - - # set the failure callback - def _failure_callback(idx, total, exception): - _failure_msg = "Soledad sync: error while sending document " \ - "%d/%d: %s" % (idx, total, exception) - logger.warning("%s" % _failure_msg) - logger.warning("Soledad sync: failing gracefully, will " - "recover on next sync.") - - t.doc_syncer.set_failure_callback(_failure_callback) - - # save thread and append - t.start() - threads.append((t, doc)) - last_request_lock = t.request_lock - last_callback_lock = t.callback_lock - sent += 1 - - # make sure all threads finished and we have up-to-date info - while threads: - # check if there are failures - t, doc = threads.pop(0) - t.join() - if t.success: - synced.append((doc.doc_id, doc.rev)) - - if defer_decryption: - self._sync_watcher.start() - - # get docs from target - if self.stopped is False: - cur_target_gen, cur_target_trans_id = self._get_remote_docs( - url, - last_known_generation, last_known_trans_id, headers, - return_doc_cb, ensure_callback, sync_id, syncer_pool, - defer_decryption=defer_decryption) - syncer_pool.cleanup() - - # delete documents from the sync database - if defer_encryption: - self.delete_encrypted_docs_from_db(synced) - - # wait for deferred decryption to finish - if defer_decryption: - while self.clear_to_sync() is False: - sleep(self.DECRYPT_TASK_PERIOD) - self._teardown_sync_watcher() - self._teardown_sync_decr_pool() - self._sync_exchange_lock.release() - - self.stop() - return cur_target_gen, cur_target_trans_id - - def start(self): - """ - Mark current sync session as running. - """ - with self._stop_lock: - self._stopped = False - - def stop(self): - """ - Mark current sync session as stopped. - - This will eventually interrupt the sync_exchange() method and return - enough information to the synchronizer so the sync session can be - recovered afterwards. - """ - with self._stop_lock: - self._stopped = True - - @property - def stopped(self): - """ - Return whether this sync session is stopped. - - :return: Whether this sync session is stopped. - :rtype: bool - """ - with self._stop_lock: - return self._stopped is True - - def get_encrypted_doc_from_db(self, doc_id, doc_rev): - """ - Retrieve encrypted document from the database of encrypted docs for - sync. - - :param doc_id: The Document id. - :type doc_id: str - - :param doc_rev: The document revision - :type doc_rev: str - """ - encr = SyncEncrypterPool - c = self._sync_db.cursor() - sql = ("SELECT content FROM %s WHERE doc_id=? and rev=?" % ( - encr.TABLE_NAME,)) - c.execute(sql, (doc_id, doc_rev)) - res = c.fetchall() - if len(res) != 0: - return res[0][0] - - def delete_encrypted_docs_from_db(self, docs_ids): - """ - Delete several encrypted documents from the database of symmetrically - encrypted docs to sync. - - :param docs_ids: an iterable with (doc_id, doc_rev) for all documents - to be deleted. - :type docs_ids: any iterable of tuples of str - """ - if docs_ids: - encr = SyncEncrypterPool - c = self._sync_db.cursor() - for doc_id, doc_rev in docs_ids: - sql = ("DELETE FROM %s WHERE doc_id=? and rev=?" % ( - encr.TABLE_NAME,)) - c.execute(sql, (doc_id, doc_rev)) - self._sync_db.commit() - - def _save_encrypted_received_doc(self, doc, gen, trans_id, idx, total): - """ - Save a symmetrically encrypted incoming document into the received - docs table in the sync db. A decryption task will pick it up - from here in turn. - - :param doc: The document to save. - :type doc: SoledadDocument - :param gen: The generation. - :type gen: str - :param trans_id: Transacion id. - :type gen: str - :param idx: The index count of the current operation. - :type idx: int - :param total: The total number of operations. - :type total: int - """ - logger.debug( - "Enqueueing doc for decryption: %d/%d." - % (idx + 1, total)) - self._sync_decr_pool.insert_encrypted_received_doc( - doc.doc_id, doc.rev, doc.content, gen, trans_id) - - def _save_received_doc(self, doc, gen, trans_id, idx, total): - """ - Save any incoming document into the received docs table in the sync db. - - :param doc: The document to save. - :type doc: SoledadDocument - :param gen: The generation. - :type gen: str - :param trans_id: Transacion id. - :type gen: str - :param idx: The index count of the current operation. - :type idx: int - :param total: The total number of operations. - :type total: int - """ - logger.debug( - "Enqueueing doc, no decryption needed: %d/%d." - % (idx + 1, total)) - self._sync_decr_pool.insert_received_doc( - doc.doc_id, doc.rev, doc.content, gen, trans_id) - - # - # Symmetric decryption of syncing docs - # - - def clear_to_sync(self): - """ - Return True if sync can proceed (ie, the received db table is empty). - :rtype: bool - """ - if self._sync_decr_pool is not None: - return self._sync_decr_pool.count_received_encrypted_docs() == 0 - else: - return True - - def set_decryption_callback(self, cb): - """ - Set callback to be called when the decryption finishes. - - :param cb: The callback to be set. - :type cb: callable - """ - self._decryption_callback = cb - - def has_decryption_callback(self): - """ - Return True if there is a decryption callback set. - :rtype: bool - """ - return self._decryption_callback is not None - - def has_syncdb(self): - """ - Return True if we have an initialized syncdb. - """ - return self._sync_db is not None - - def _decrypt_syncing_received_docs(self): - """ - Decrypt the documents received from remote replica and insert them - into the local one. - - Called periodically from TimerTask self._sync_watcher. - """ - if sameProxiedObjects( - self._insert_doc_cb.get(self.source_replica_uid), - None): - return - - decrypter = self._sync_decr_pool - decrypter.decrypt_received_docs() - done = decrypter.process_decrypted() - - def _sign_request(self, method, url_query, params): - """ - Return an authorization header to be included in the HTTP request. - - :param method: The HTTP method. - :type method: str - :param url_query: The URL query string. - :type url_query: str - :param params: A list with encoded query parameters. - :type param: list - - :return: The Authorization header. - :rtype: list of tuple - """ - return TokenBasedAuth._sign_request(self, method, url_query, params) - - def set_token_credentials(self, uuid, token): - """ - Store given credentials so we can sign the request later. - - :param uuid: The user's uuid. - :type uuid: str - :param token: The authentication token. - :type token: str - """ - TokenBasedAuth.set_token_credentials(self, uuid, token) diff --git a/common/changes/VERSION_COMPAT b/common/changes/VERSION_COMPAT index e69de29b..cc00ecf7 100644 --- a/common/changes/VERSION_COMPAT +++ b/common/changes/VERSION_COMPAT @@ -0,0 +1,10 @@ +################################################# +# This file keeps track of the recent changes +# introduced in internal leap dependencies. +# Add your changes here so we can properly update +# requirements.pip during the release process. +# (leave header when resetting) +################################################# +# +# BEGIN DEPENDENCY LIST ------------------------- +# leap.foo.bar>=x.y.z diff --git a/common/pkg/requirements-testing.pip b/common/pkg/requirements-testing.pip index 9302450c..c72c9fc4 100644 --- a/common/pkg/requirements-testing.pip +++ b/common/pkg/requirements-testing.pip @@ -3,3 +3,4 @@ testscenarios leap.common leap.soledad.server leap.soledad.client +setuptools-trial diff --git a/common/pkg/requirements.pip b/common/pkg/requirements.pip index 5787114e..005d6884 100644 --- a/common/pkg/requirements.pip +++ b/common/pkg/requirements.pip @@ -1,7 +1,10 @@ simplejson u1db -#this is not strictly needed by us, but we need it -#until u1db adds it to its release as a dep. -oauth +# leap deps -- bump me! +leap.common>=0.7.0 +# XXX -- fix me! +# oauth is not strictly needed by us, but we need it until u1db adds it to its +# release as a dep. +oauth diff --git a/common/setup.cfg b/common/setup.cfg new file mode 100644 index 00000000..c71bffa0 --- /dev/null +++ b/common/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test = trial diff --git a/common/setup.py b/common/setup.py index 365006b2..f4d8bc65 100644 --- a/common/setup.py +++ b/common/setup.py @@ -234,23 +234,6 @@ class cmd_develop(_cmd_develop): # versioneer powered -old_cmd_sdist = cmdclass["sdist"] - - -class cmd_sdist(old_cmd_sdist): - """ - Generate 'src/leap/soledad/common/ddocs.py' which contains couch design - documents scripts. - """ - def run(self): - old_cmd_sdist.run(self) - - def make_release_tree(self, base_dir, files): - old_cmd_sdist.make_release_tree(self, base_dir, files) - build_ddocs_py(basedir=base_dir) - - -# versioneer powered old_cmd_build = cmdclass["build"] @@ -262,7 +245,6 @@ class cmd_build(old_cmd_build): cmdclass["freeze_debianver"] = freeze_debianver cmdclass["build"] = cmd_build -cmdclass["sdist"] = cmd_sdist cmdclass["develop"] = cmd_develop @@ -288,9 +270,9 @@ setup( ), classifiers=trove_classifiers, namespace_packages=["leap", "leap.soledad"], - packages=find_packages('src', exclude=['leap.soledad.common.tests']), + packages=find_packages('src', exclude=['*.tests', '*.tests.*']), package_dir={'': 'src'}, - test_suite='leap.soledad.common.tests.load_tests', + test_suite='leap.soledad.common.tests', install_requires=utils.parse_requirements(), tests_require=utils.parse_requirements( reqfiles=['pkg/requirements-testing.pip']), diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py index 23d28e76..1e52e3a7 100644 --- a/common/src/leap/soledad/common/__init__.py +++ b/common/src/leap/soledad/common/__init__.py @@ -21,14 +21,10 @@ Soledad routines common to client and server. """ -from hashlib import sha256 - - # # Global constants # - SHARED_DB_NAME = 'shared' SHARED_DB_LOCK_DOC_ID_PREFIX = 'lock-' USER_DB_PREFIX = 'user-' @@ -38,45 +34,17 @@ USER_DB_PREFIX = 'user-' # Global functions # -# we want to use leap.common.check.leap_assert in case it is available, -# because it also logs in a way other parts of leap can access log messages. - -try: - from leap.common.check import leap_assert as soledad_assert - -except ImportError: - - def soledad_assert(condition, message): - """ - Asserts the condition and displays the message if that's not - met. - - @param condition: condition to check - @type condition: bool - @param message: message to display if the condition isn't met - @type message: str - """ - assert condition, message - -try: - from leap.common.check import leap_assert_type as soledad_assert_type - -except ImportError: - - def soledad_assert_type(var, expectedType): - """ - Helper assert check for a variable's expected type - - @param var: variable to check - @type var: any - @param expectedType: type to check agains - @type expectedType: type - """ - soledad_assert(isinstance(var, expectedType), - "Expected type %r instead of %r" % - (expectedType, type(var))) +from leap.common.check import leap_assert as soledad_assert +from leap.common.check import leap_assert_type as soledad_assert_type from ._version import get_versions __version__ = get_versions()['version'] del get_versions + + +__all__ = [ + "soledad_assert", + "soledad_assert_type", + "__version__", +] diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index b38b5b96..8d262ccd 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -18,12 +18,12 @@ """A U1DB backend that uses CouchDB as its persistence layer.""" + import simplejson as json import re import uuid import logging import binascii -import socket import time import sys import threading @@ -44,7 +44,7 @@ from couchdb.http import ( urljoin as couch_urljoin, Resource, ) -from u1db import query_parser, vectorclock +from u1db import vectorclock from u1db.errors import ( DatabaseDoesNotExist, InvalidGeneration, @@ -60,7 +60,7 @@ from u1db.remote import http_app from u1db.remote.server_state import ServerState -from leap.soledad.common import USER_DB_PREFIX, ddocs, errors +from leap.soledad.common import ddocs, errors from leap.soledad.common.document import SoledadDocument @@ -160,7 +160,6 @@ class CouchDocument(SoledadDocument): """ if self._conflicts is None: raise Exception("Run self._ensure_fetch_conflicts first!") - conflicts_len = len(self._conflicts) self._conflicts = filter( lambda doc: doc.rev not in conflict_revs, self._conflicts) @@ -1181,7 +1180,7 @@ class CouchDatabase(CommonBackend): res = self._database.resource(*ddoc_path) try: with CouchDatabase.update_handler_lock[self._get_replica_uid()]: - body={ + body = { 'other_replica_uid': other_replica_uid, 'other_generation': other_generation, 'other_transaction_id': other_transaction_id, diff --git a/common/src/leap/soledad/common/crypto.py b/common/src/leap/soledad/common/crypto.py index 56bb608a..b4f3234f 100644 --- a/common/src/leap/soledad/common/crypto.py +++ b/common/src/leap/soledad/common/crypto.py @@ -35,13 +35,29 @@ class EncryptionSchemes(object): PUBKEY = 'pubkey' -class UnknownEncryptionScheme(Exception): +class UnknownEncryptionSchemeError(Exception): """ Raised when trying to decrypt from unknown encryption schemes. """ pass +class EncryptionMethods(object): + """ + Representation of encryption methods that can be used. + """ + + AES_256_CTR = 'aes-256-ctr' + XSALSA20 = 'xsalsa20' + + +class UnknownEncryptionMethodError(Exception): + """ + Raised when trying to encrypt/decrypt with unknown method. + """ + pass + + class MacMethods(object): """ Representation of MAC methods used to authenticate document's contents. @@ -50,7 +66,7 @@ class MacMethods(object): HMAC = 'hmac' -class UnknownMacMethod(Exception): +class UnknownMacMethodError(Exception): """ Raised when trying to authenticate document's content with unknown MAC mehtod. @@ -58,7 +74,7 @@ class UnknownMacMethod(Exception): pass -class WrongMac(Exception): +class WrongMacError(Exception): """ Raised when failing to authenticate document's contents based on MAC. """ diff --git a/common/src/leap/soledad/common/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py index 3081683b..acebb77b 100644 --- a/common/src/leap/soledad/common/tests/__init__.py +++ b/common/src/leap/soledad/common/tests/__init__.py @@ -19,291 +19,9 @@ """ Tests to make sure Soledad provides U1DB functionality and more. """ -import os -import random -import string -import u1db -from mock import Mock - - -from leap.soledad.common.document import SoledadDocument -from leap.soledad.client import Soledad -from leap.soledad.client.crypto import decrypt_doc_dict -from leap.soledad.client.crypto import ENC_SCHEME_KEY -from leap.common.testing.basetest import BaseLeapTest - - -#----------------------------------------------------------------------------- -# Some tests inherit from BaseSoledadTest in order to have a working Soledad -# instance in each test. -#----------------------------------------------------------------------------- - -ADDRESS = 'leap@leap.se' - - -class BaseSoledadTest(BaseLeapTest): - """ - Instantiates Soledad for usage in tests. - """ - defer_sync_encryption = False - - def setUp(self): - # config info - self.db1_file = os.path.join(self.tempdir, "db1.u1db") - self.db2_file = os.path.join(self.tempdir, "db2.u1db") - self.email = ADDRESS - # open test dbs - self._db1 = u1db.open(self.db1_file, create=True, - document_factory=SoledadDocument) - self._db2 = u1db.open(self.db2_file, create=True, - document_factory=SoledadDocument) - # get a random prefix for each test, so we do not mess with - # concurrency during initialization and shutting down of - # each local db. - self.rand_prefix = ''.join( - map(lambda x: random.choice(string.ascii_letters), range(6))) - # initialize soledad by hand so we can control keys - self._soledad = self._soledad_instance( - prefix=self.rand_prefix, user=self.email) - - def tearDown(self): - self._db1.close() - self._db2.close() - self._soledad.close() - - # XXX should not access "private" attrs - for f in [self._soledad._local_db_path, self._soledad._secrets_path]: - if os.path.isfile(f): - os.unlink(f) - def get_default_shared_mock(self, put_doc_side_effect): - """ - Get a default class for mocking the shared DB - """ - class defaultMockSharedDB(object): - get_doc = Mock(return_value=None) - put_doc = Mock(side_effect=put_doc_side_effect) - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - def __call__(self): - return self - return defaultMockSharedDB - - def _soledad_instance(self, user=ADDRESS, passphrase=u'123', - prefix='', - secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME, - local_db_path='soledad.u1db', server_url='', - cert_file=None, secret_id=None, - shared_db_class=None): - - def _put_doc_side_effect(doc): - self._doc_put = doc - - if shared_db_class is not None: - MockSharedDB = shared_db_class - else: - MockSharedDB = self.get_default_shared_mock( - _put_doc_side_effect) - - Soledad._shared_db = MockSharedDB() - return Soledad( - user, - passphrase, - secrets_path=os.path.join( - self.tempdir, prefix, secrets_path), - local_db_path=os.path.join( - self.tempdir, prefix, local_db_path), - server_url=server_url, # Soledad will fail if not given an url. - cert_file=cert_file, - secret_id=secret_id, - defer_encryption=self.defer_sync_encryption) - - def assertGetEncryptedDoc( - self, db, doc_id, doc_rev, content, has_conflicts): - """ - Assert that the document in the database looks correct. - """ - exp_doc = self.make_document(doc_id, doc_rev, content, - has_conflicts=has_conflicts) - doc = db.get_doc(doc_id) - - if ENC_SCHEME_KEY in doc.content: - # XXX check for SYM_KEY too - key = self._soledad._crypto.doc_passphrase(doc.doc_id) - secret = self._soledad._crypto.secret - decrypted = decrypt_doc_dict( - doc.content, doc.doc_id, doc.rev, - key, secret) - doc.set_json(decrypted) - self.assertEqual(exp_doc.doc_id, doc.doc_id) - self.assertEqual(exp_doc.rev, doc.rev) - self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts) - self.assertEqual(exp_doc.content, doc.content) - - -# Key material for testing -KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF" -PUBLIC_KEY = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz -iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO -zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx -irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT -huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs -d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g -wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb -hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv -U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H -T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i -Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB -tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD -BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb -T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5 -hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP -QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU -Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+ -eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI -txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB -KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy -7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr -K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx -2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n -3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf -H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS -sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs -iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD -uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0 -GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3 -lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS -fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe -dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1 -WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK -3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td -U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F -Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX -NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj -cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk -ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE -VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51 -XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8 -oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM -Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+ -BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/ -diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2 -ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX -=MuOY ------END PGP PUBLIC KEY BLOCK----- -""" -PRIVATE_KEY = """ ------BEGIN PGP PRIVATE KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz -iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO -zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx -irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT -huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs -d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g -wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb -hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv -U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H -T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i -Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB -AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs -E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t -KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds -FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb -J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky -KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY -VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5 -jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF -q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c -zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv -OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt -VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx -nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv -Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP -4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F -RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv -mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x -sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0 -cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI -L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW -ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd -LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e -SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO -dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8 -xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY -HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw -7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh -cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH -AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM -MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo -rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX -hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA -QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo -alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4 -Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb -HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV -3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF -/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n -s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC -4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ -1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ -uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q -us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/ -Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o -6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA -K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+ -iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t -9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3 -zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl -QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD -Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX -wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e -PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC -9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI -85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih -7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn -E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+ -ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0 -Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m -KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT -xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/ -jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4 -OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o -tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF -cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb -OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i -7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2 -H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX -MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR -ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ -waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU -e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs -rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G -GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu -tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U -22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E -/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC -0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+ -LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm -laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy -bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd -GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp -VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ -z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD -U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l -Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ -GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL -Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1 -RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= -=JTFu ------END PGP PRIVATE KEY BLOCK----- -""" +import os def load_tests(): diff --git a/common/src/leap/soledad/common/tests/couchdb.ini.template b/common/src/leap/soledad/common/tests/couchdb.ini.template index 1fc2205b..174d9d86 100644 --- a/common/src/leap/soledad/common/tests/couchdb.ini.template +++ b/common/src/leap/soledad/common/tests/couchdb.ini.template @@ -12,213 +12,11 @@ delayed_commits = true ; set this to false to ensure an fsync before 201 Created uri_file = %(tempdir)s/lib/couch.uri file_compression = snappy -[database_compaction] -; larger buffer sizes can originate smaller files -doc_buffer_size = 524288 ; value in bytes -checkpoint_after = 5242880 ; checkpoint after every N bytes were written - -[view_compaction] -; larger buffer sizes can originate smaller files -keyvalue_buffer_size = 2097152 ; value in bytes - -[httpd] -port = 0 -bind_address = 127.0.0.1 -authentication_handlers = {couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler} -default_handler = {couch_httpd_db, handle_request} -secure_rewrites = true -vhost_global_handlers = _utils, _uuids, _session, _oauth, _users -allow_jsonp = false -; Options for the MochiWeb HTTP server. -;server_options = [{backlog, 128}, {acceptor_pool_size, 16}] -; For more socket options, consult Erlang's module 'inet' man page. -;socket_options = [{recbuf, 262144}, {sndbuf, 262144}, {nodelay, true}] -log_max_chunk_size = 1000000 - [log] file = %(tempdir)s/log/couch.log level = info include_sasl = true -[couch_httpd_auth] -authentication_db = _users -authentication_redirect = /_utils/session.html -require_valid_user = false -timeout = 600 ; number of seconds before automatic logout -auth_cache_size = 50 ; size is number of cache entries -allow_persistent_cookies = false ; set to true to allow persistent cookies - -[couch_httpd_oauth] -; If set to 'true', oauth token and consumer secrets will be looked up -; in the authentication database (_users). These secrets are stored in -; a top level property named "oauth" in user documents. Example: -; { -; "_id": "org.couchdb.user:joe", -; "type": "user", -; "name": "joe", -; "password_sha": "fe95df1ca59a9b567bdca5cbaf8412abd6e06121", -; "salt": "4e170ffeb6f34daecfd814dfb4001a73" -; "roles": ["foo", "bar"], -; "oauth": { -; "consumer_keys": { -; "consumerKey1": "key1Secret", -; "consumerKey2": "key2Secret" -; }, -; "tokens": { -; "token1": "token1Secret", -; "token2": "token2Secret" -; } -; } -; } -use_users_db = false - -[query_servers] -; javascript = %(tempdir)s/server/main.js -javascript = /usr/bin/couchjs /usr/share/couchdb/server/main.js -coffeescript = /usr/bin/couchjs /usr/share/couchdb/server/main-coffee.js - - -; Changing reduce_limit to false will disable reduce_limit. -; If you think you're hitting reduce_limit with a "good" reduce function, -; please let us know on the mailing list so we can fine tune the heuristic. -[query_server_config] -reduce_limit = true -os_process_limit = 25 - -[daemons] -view_manager={couch_view, start_link, []} -external_manager={couch_external_manager, start_link, []} -query_servers={couch_query_servers, start_link, []} -vhosts={couch_httpd_vhost, start_link, []} -httpd={couch_httpd, start_link, []} -stats_aggregator={couch_stats_aggregator, start, []} -stats_collector={couch_stats_collector, start, []} -uuids={couch_uuids, start, []} -auth_cache={couch_auth_cache, start_link, []} -replication_manager={couch_replication_manager, start_link, []} -os_daemons={couch_os_daemons, start_link, []} -compaction_daemon={couch_compaction_daemon, start_link, []} - -[httpd_global_handlers] -/ = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>} - -_all_dbs = {couch_httpd_misc_handlers, handle_all_dbs_req} -_active_tasks = {couch_httpd_misc_handlers, handle_task_status_req} -_config = {couch_httpd_misc_handlers, handle_config_req} -_replicate = {couch_httpd_replicator, handle_req} -_uuids = {couch_httpd_misc_handlers, handle_uuids_req} -_restart = {couch_httpd_misc_handlers, handle_restart_req} -_stats = {couch_httpd_stats_handlers, handle_stats_req} -_log = {couch_httpd_misc_handlers, handle_log_req} -_session = {couch_httpd_auth, handle_session_req} -_oauth = {couch_httpd_oauth, handle_oauth_req} - -[httpd_db_handlers] -_view_cleanup = {couch_httpd_db, handle_view_cleanup_req} -_compact = {couch_httpd_db, handle_compact_req} -_design = {couch_httpd_db, handle_design_req} -_temp_view = {couch_httpd_view, handle_temp_view_req} -_changes = {couch_httpd_db, handle_changes_req} - -; The external module takes an optional argument allowing you to narrow it to a -; single script. Otherwise the script name is inferred from the first path section -; after _external's own path. -; _mypath = {couch_httpd_external, handle_external_req, <<"mykey">>} -; _external = {couch_httpd_external, handle_external_req} - -[httpd_design_handlers] -_view = {couch_httpd_view, handle_view_req} -_show = {couch_httpd_show, handle_doc_show_req} -_list = {couch_httpd_show, handle_view_list_req} -_info = {couch_httpd_db, handle_design_info_req} -_rewrite = {couch_httpd_rewrite, handle_rewrite_req} -_update = {couch_httpd_show, handle_doc_update_req} - -; enable external as an httpd handler, then link it with commands here. -; note, this api is still under consideration. -; [external] -; mykey = /path/to/mycommand - -; Here you can setup commands for CouchDB to manage -; while it is alive. It will attempt to keep each command -; alive if it exits. -; [os_daemons] -; some_daemon_name = /path/to/script -with args - - -[uuids] -; Known algorithms: -; random - 128 bits of random awesome -; All awesome, all the time. -; sequential - monotonically increasing ids with random increments -; First 26 hex characters are random. Last 6 increment in -; random amounts until an overflow occurs. On overflow, the -; random prefix is regenerated and the process starts over. -; utc_random - Time since Jan 1, 1970 UTC with microseconds -; First 14 characters are the time in hex. Last 18 are random. -algorithm = sequential - -[stats] -; rate is in milliseconds -rate = 1000 -; sample intervals are in seconds -samples = [0, 60, 300, 900] - -[attachments] -compression_level = 8 ; from 1 (lowest, fastest) to 9 (highest, slowest), 0 to disable compression -compressible_types = text/*, application/javascript, application/json, application/xml - -[replicator] -db = _replicator -; Maximum replicaton retry count can be a non-negative integer or "infinity". -max_replication_retry_count = 10 -; More worker processes can give higher network throughput but can also -; imply more disk and network IO. -worker_processes = 4 -; With lower batch sizes checkpoints are done more frequently. Lower batch sizes -; also reduce the total amount of used RAM memory. -worker_batch_size = 500 -; Maximum number of HTTP connections per replication. -http_connections = 20 -; HTTP connection timeout per replication. -; Even for very fast/reliable networks it might need to be increased if a remote -; database is too busy. -connection_timeout = 30000 -; If a request fails, the replicator will retry it up to N times. -retries_per_request = 10 -; Some socket options that might boost performance in some scenarios: -; {nodelay, boolean()} -; {sndbuf, integer()} -; {recbuf, integer()} -; {priority, integer()} -; See the `inet` Erlang module's man page for the full list of options. -socket_options = [{keepalive, true}, {nodelay, false}] -; Path to a file containing the user's certificate. -;cert_file = /full/path/to/server_cert.pem -; Path to file containing user's private PEM encoded key. -;key_file = /full/path/to/server_key.pem -; String containing the user's password. Only used if the private keyfile is password protected. -;password = somepassword -; Set to true to validate peer certificates. -verify_ssl_certificates = false -; File containing a list of peer trusted certificates (in the PEM format). -;ssl_trusted_certificates_file = /etc/ssl/certs/ca-certificates.crt -; Maximum peer certificate depth (must be set even if certificate validation is off). -ssl_certificate_max_depth = 3 - -[compaction_daemon] -; The delay, in seconds, between each check for which database and view indexes -; need to be compacted. -check_interval = 300 -; If a database or view index file is smaller then this value (in bytes), -; compaction will not happen. Very small files always have a very high -; fragmentation therefore it's not worth to compact them. -min_file_size = 131072 - -[compactions] -; List of compaction rules for the compaction daemon. - - -;[admins] -;testuser = -hashed-f50a252c12615697c5ed24ec5cd56b05d66fe91e,b05471ba260132953930cf9f97f327f5 -; pass for above user is 'testpass' +[httpd] +port = 0 +bind_address = 127.0.0.1 diff --git a/common/src/leap/soledad/common/tests/hacker_crackdown.txt b/common/src/leap/soledad/common/tests/hacker_crackdown.txt new file mode 100644 index 00000000..a01eb509 --- /dev/null +++ b/common/src/leap/soledad/common/tests/hacker_crackdown.txt @@ -0,0 +1,13005 @@ +The Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling
+
+This eBook is for the use of anyone anywhere at no cost and with
+almost no restrictions whatsoever. You may copy it, give it away or
+re-use it under the terms of the Project Gutenberg License included
+with this eBook or online at www.gutenberg.org
+
+** This is a COPYRIGHTED Project Gutenberg eBook, Details Below **
+** Please follow the copyright guidelines in this file. **
+
+Title: Hacker Crackdown
+ Law and Disorder on the Electronic Frontier
+
+Author: Bruce Sterling
+
+Posting Date: February 9, 2012 [EBook #101]
+Release Date: January, 1994
+
+Language: English
+
+
+*** START OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN ***
+
+
+
+
+
+
+
+
+
+
+
+
+
+THE HACKER CRACKDOWN
+
+Law and Disorder on the Electronic Frontier
+
+by Bruce Sterling
+
+
+
+
+CONTENTS
+
+
+Preface to the Electronic Release of The Hacker Crackdown
+
+Chronology of the Hacker Crackdown
+
+
+Introduction
+
+
+Part 1: CRASHING THE SYSTEM
+A Brief History of Telephony
+Bell's Golden Vaporware
+Universal Service
+Wild Boys and Wire Women
+The Electronic Communities
+The Ungentle Giant
+The Breakup
+In Defense of the System
+The Crash Post-Mortem
+Landslides in Cyberspace
+
+
+Part 2: THE DIGITAL UNDERGROUND
+Steal This Phone
+Phreaking and Hacking
+The View From Under the Floorboards
+Boards: Core of the Underground
+Phile Phun
+The Rake's Progress
+Strongholds of the Elite
+Sting Boards
+Hot Potatoes
+War on the Legion
+Terminus
+Phile 9-1-1
+War Games
+Real Cyberpunk
+
+
+Part 3: LAW AND ORDER
+Crooked Boards
+The World's Biggest Hacker Bust
+Teach Them a Lesson
+The U.S. Secret Service
+The Secret Service Battles the Boodlers
+A Walk Downtown
+FCIC: The Cutting-Edge Mess
+Cyberspace Rangers
+FLETC: Training the Hacker-Trackers
+
+
+Part 4: THE CIVIL LIBERTARIANS
+NuPrometheus + FBI = Grateful Dead
+Whole Earth + Computer Revolution = WELL
+Phiber Runs Underground and Acid Spikes the Well
+The Trial of Knight Lightning
+Shadowhawk Plummets to Earth
+Kyrie in the Confessional
+$79,499
+A Scholar Investigates
+Computers, Freedom, and Privacy
+
+
+Electronic Afterword to The Hacker Crackdown, Halloween 1993
+
+
+
+
+THE HACKER CRACKDOWN
+
+Law and Disorder on the Electronic Frontier
+
+by Bruce Sterling
+
+
+
+
+
+Preface to the Electronic Release of The Hacker Crackdown
+
+
+January 1, 1994--Austin, Texas
+
+
+Hi, I'm Bruce Sterling, the author of this electronic book.
+
+Out in the traditional world of print, The Hacker Crackdown
+is ISBN 0-553-08058-X, and is formally catalogued by
+the Library of Congress as "1. Computer crimes--United States.
+2. Telephone--United States--Corrupt practices.
+3. Programming (Electronic computers)--United States--Corrupt practices."
+
+`Corrupt practices,' I always get a kick out of that description.
+Librarians are very ingenious people.
+
+The paperback is ISBN 0-553-56370-X. If you go
+and buy a print version of The Hacker Crackdown,
+an action I encourage heartily, you may notice that
+in the front of the book, beneath the copyright notice--
+"Copyright (C) 1992 by Bruce Sterling"--
+it has this little block of printed legal
+boilerplate from the publisher. It says, and I quote:
+
+ "No part of this book may be reproduced or transmitted in any form
+or by any means, electronic or mechanical, including photocopying,
+recording, or by any information storage and retrieval system,
+without permission in writing from the publisher.
+For information address: Bantam Books."
+
+This is a pretty good disclaimer, as such disclaimers go.
+I collect intellectual-property disclaimers, and I've seen dozens of them,
+and this one is at least pretty straightforward. In this narrow
+and particular case, however, it isn't quite accurate.
+Bantam Books puts that disclaimer on every book they publish,
+but Bantam Books does not, in fact, own the electronic rights to this book.
+I do, because of certain extensive contract maneuverings my agent and I
+went through before this book was written. I want to give those electronic
+publishing rights away through certain not-for-profit channels,
+and I've convinced Bantam that this is a good idea.
+
+Since Bantam has seen fit to peacably agree to this scheme of mine,
+Bantam Books is not going to fuss about this. Provided you don't try
+to sell the book, they are not going to bother you for what you do with
+the electronic copy of this book. If you want to check this out personally,
+you can ask them; they're at 1540 Broadway NY NY 10036. However, if you were
+so foolish as to print this book and start retailing it for money in violation
+of my copyright and the commercial interests of Bantam Books, then Bantam,
+a part of the gigantic Bertelsmann multinational publishing combine,
+would roust some of their heavy-duty attorneys out of hibernation
+and crush you like a bug. This is only to be expected.
+I didn't write this book so that you could make money out of it.
+If anybody is gonna make money out of this book,
+it's gonna be me and my publisher.
+
+My publisher deserves to make money out of this book.
+Not only did the folks at Bantam Books commission me
+to write the book, and pay me a hefty sum to do so, but
+they bravely printed, in text, an electronic document the
+reproduction of which was once alleged to be a federal felony.
+Bantam Books and their numerous attorneys were very brave
+and forthright about this book. Furthermore, my former editor
+at Bantam Books, Betsy Mitchell, genuinely cared about this project,
+and worked hard on it, and had a lot of wise things to say
+about the manuscript. Betsy deserves genuine credit for this book,
+credit that editors too rarely get.
+
+The critics were very kind to The Hacker Crackdown,
+and commercially the book has done well. On the other hand,
+I didn't write this book in order to squeeze every last nickel
+and dime out of the mitts of impoverished sixteen-year-old
+cyberpunk high-school-students. Teenagers don't have any money--
+(no, not even enough for the six-dollar Hacker Crackdown paperback,
+with its attractive bright-red cover and useful index).
+That's a major reason why teenagers sometimes succumb to the temptation
+to do things they shouldn't, such as swiping my books out of libraries.
+Kids: this one is all yours, all right? Go give the print version back.
+*8-)
+
+Well-meaning, public-spirited civil libertarians don't have much money,
+either. And it seems almost criminal to snatch cash out of the hands of
+America's direly underpaid electronic law enforcement community.
+
+If you're a computer cop, a hacker, or an electronic civil
+liberties activist, you are the target audience for this book.
+I wrote this book because I wanted to help you, and help other people
+understand you and your unique, uhm, problems. I wrote this book
+to aid your activities, and to contribute to the public discussion
+of important political issues. In giving the text away in this
+fashion, I am directly contributing to the book's ultimate aim:
+to help civilize cyberspace.
+
+Information WANTS to be free. And the information inside
+this book longs for freedom with a peculiar intensity.
+I genuinely believe that the natural habitat of this book
+is inside an electronic network. That may not be the easiest
+direct method to generate revenue for the book's author,
+but that doesn't matter; this is where this book belongs
+by its nature. I've written other books--plenty of other books--
+and I'll write more and I am writing more, but this one is special.
+I am making The Hacker Crackdown available electronically
+as widely as I can conveniently manage, and if you like the book,
+and think it is useful, then I urge you to do the same with it.
+
+You can copy this electronic book. Copy the heck out of it,
+be my guest, and give those copies to anybody who wants them.
+The nascent world of cyberspace is full of sysadmins, teachers,
+trainers, cybrarians, netgurus, and various species of cybernetic activist.
+If you're one of those people, I know about you, and I know the hassle
+you go through to try to help people learn about the electronic frontier.
+I hope that possessing this book in electronic form will lessen your troubles.
+Granted, this treatment of our electronic social spectrum is not the ultimate
+in academic rigor. And politically, it has something to offend
+and trouble almost everyone. But hey, I'm told it's readable,
+and at least the price is right.
+
+You can upload the book onto bulletin board systems, or Internet nodes,
+or electronic discussion groups. Go right ahead and do that, I am giving
+you express permission right now. Enjoy yourself.
+
+You can put the book on disks and give the disks away,
+as long as you don't take any money for it.
+
+But this book is not public domain. You can't copyright it in
+your own name. I own the copyright. Attempts to pirate this book
+and make money from selling it may involve you in a serious litigative snarl.
+Believe me, for the pittance you might wring out of such an action,
+it's really not worth it. This book don't "belong" to you.
+In an odd but very genuine way, I feel it doesn't "belong" to me, either.
+It's a book about the people of cyberspace, and distributing it in this way
+is the best way I know to actually make this information available,
+freely and easily, to all the people of cyberspace--including people
+far outside the borders of the United States, who otherwise may never
+have a chance to see any edition of the book, and who may perhaps learn
+something useful from this strange story of distant, obscure, but portentous
+events in so-called "American cyberspace."
+
+This electronic book is now literary freeware. It now belongs to the
+emergent realm of alternative information economics. You have no right
+to make this electronic book part of the conventional flow of commerce.
+Let it be part of the flow of knowledge: there's a difference.
+I've divided the book into four sections, so that it is less ungainly
+for upload and download; if there's a section of particular relevance
+to you and your colleagues, feel free to reproduce that one and skip the rest.
+
+[Project Gutenberg has reassembled the file, with Sterling's permission.]
+
+Just make more when you need them, and give them to whoever might want them.
+
+Now have fun.
+
+Bruce Sterling--bruces@well.sf.ca.us
+
+
+THE HACKER CRACKDOWN
+
+Law and Disorder on the Electronic Frontier
+
+by Bruce Sterling
+
+
+
+
+
+
+
+CHRONOLOGY OF THE HACKER CRACKDOWN
+
+
+1865 U.S. Secret Service (USSS) founded.
+
+1876 Alexander Graham Bell invents telephone.
+
+1878 First teenage males flung off phone system by enraged authorities.
+
+1939 "Futurian" science-fiction group raided by Secret Service.
+
+1971 Yippie phone phreaks start YIPL/TAP magazine.
+
+1972 RAMPARTS magazine seized in blue-box rip-off scandal.
+
+1978 Ward Christenson and Randy Suess create first personal
+ computer bulletin board system.
+
+1982 William Gibson coins term "cyberspace."
+
+1982 "414 Gang" raided.
+
+1983-1983 AT&T dismantled in divestiture.
+
+1984 Congress passes Comprehensive Crime Control Act giving USSS
+ jurisdiction over credit card fraud and computer fraud.
+
+1984 "Legion of Doom" formed.
+
+1984. 2600: THE HACKER QUARTERLY founded.
+
+1984. WHOLE EARTH SOFTWARE CATALOG published.
+
+1985. First police "sting" bulletin board systems established.
+
+1985. Whole Earth 'Lectronic Link computer conference (WELL) goes on-line.
+
+1986 Computer Fraud and Abuse Act passed.
+
+1986 Electronic Communications Privacy Act passed.
+
+1987 Chicago prosecutors form Computer Fraud and Abuse Task Force.
+
+
+1988
+
+July. Secret Service covertly videotapes "SummerCon" hacker convention.
+
+September. "Prophet" cracks BellSouth AIMSX computer network
+ and downloads E911 Document to his own computer and to Jolnet.
+
+September. AT&T Corporate Information Security informed of Prophet's action.
+
+October. Bellcore Security informed of Prophet's action.
+
+
+1989
+
+January. Prophet uploads E911 Document to Knight Lightning.
+
+February 25. Knight Lightning publishes E911 Document in PHRACK
+ electronic newsletter.
+
+May. Chicago Task Force raids and arrests "Kyrie."
+
+June. "NuPrometheus League" distributes Apple Computer proprietary software.
+
+June 13. Florida probation office crossed with phone-sex line
+ in switching-station stunt.
+
+July. "Fry Guy" raided by USSS and Chicago Computer Fraud
+ and Abuse Task Force.
+
+July. Secret Service raids "Prophet," "Leftist," and "Urvile" in Georgia.
+
+
+1990
+
+January 15. Martin Luther King Day Crash strikes AT&T long-distance
+ network nationwide.
+
+January 18-19. Chicago Task Force raids Knight Lightning in St. Louis.
+
+January 24. USSS and New York State Police raid "Phiber Optik,"
+ "Acid Phreak," and "Scorpion" in New York City.
+
+February 1. USSS raids "Terminus" in Maryland.
+
+February 3. Chicago Task Force raids Richard Andrews' home.
+
+February 6. Chicago Task Force raids Richard Andrews' business.
+
+February 6. USSS arrests Terminus, Prophet, Leftist, and Urvile.
+
+February 9. Chicago Task Force arrests Knight Lightning.
+
+February 20. AT&T Security shuts down public-access
+ "attctc" computer in Dallas.
+
+February 21. Chicago Task Force raids Robert Izenberg in Austin.
+
+March 1. Chicago Task Force raids Steve Jackson Games, Inc.,
+ "Mentor," and "Erik Bloodaxe" in Austin.
+
+May 7,8,9.
+
+USSS and Arizona Organized Crime and Racketeering Bureau conduct
+"Operation Sundevil" raids in Cincinnatti, Detroit, Los Angeles,
+Miami, Newark, Phoenix, Pittsburgh, Richmond, Tucson, San Diego,
+San Jose, and San Francisco.
+
+May. FBI interviews John Perry Barlow re NuPrometheus case.
+
+June. Mitch Kapor and Barlow found Electronic Frontier Foundation;
+ Barlow publishes CRIME AND PUZZLEMENT manifesto.
+
+July 24-27. Trial of Knight Lightning.
+
+1991
+
+February. CPSR Roundtable in Washington, D.C.
+
+March 25-28. Computers, Freedom and Privacy conference in San Francisco.
+
+May 1. Electronic Frontier Foundation, Steve Jackson,
+ and others file suit against members of Chicago Task Force.
+
+July 1-2. Switching station phone software crash affects
+ Washington, Los Angeles, Pittsburgh, San Francisco.
+
+September 17. AT&T phone crash affects New York City and three airports.
+
+
+
+
+Introduction
+
+This is a book about cops, and wild teenage whiz-kids, and lawyers,
+and hairy-eyed anarchists, and industrial technicians, and hippies,
+and high-tech millionaires, and game hobbyists, and computer security
+experts, and Secret Service agents, and grifters, and thieves.
+
+This book is about the electronic frontier of the 1990s.
+It concerns activities that take place inside computers
+and over telephone lines.
+
+A science fiction writer coined the useful term "cyberspace" in 1982,
+but the territory in question, the electronic frontier, is about
+a hundred and thirty years old. Cyberspace is the "place" where
+a telephone conversation appears to occur. Not inside your actual phone,
+the plastic device on your desk. Not inside the other person's phone,
+in some other city. THE PLACE BETWEEN the phones. The indefinite
+place OUT THERE, where the two of you, two human beings,
+actually meet and communicate.
+
+Although it is not exactly "real," "cyberspace" is a genuine place.
+Things happen there that have very genuine consequences. This "place"
+is not "real," but it is serious, it is earnest. Tens of thousands
+of people have dedicated their lives to it, to the public service
+of public communication by wire and electronics.
+
+People have worked on this "frontier" for generations now.
+Some people became rich and famous from their efforts there.
+Some just played in it, as hobbyists. Others soberly pondered it,
+and wrote about it, and regulated it, and negotiated over it in
+international forums, and sued one another about it, in gigantic,
+epic court battles that lasted for years. And almost since
+the beginning, some people have committed crimes in this place.
+
+But in the past twenty years, this electrical "space,"
+which was once thin and dark and one-dimensional--little more
+than a narrow speaking-tube, stretching from phone to phone--
+has flung itself open like a gigantic jack-in-the-box.
+Light has flooded upon it, the eerie light of the glowing computer screen.
+This dark electric netherworld has become a vast flowering electronic landscape.
+Since the 1960s, the world of the telephone has cross-bred itself
+with computers and television, and though there is still no substance
+to cyberspace, nothing you can handle, it has a strange kind
+of physicality now. It makes good sense today to talk of cyberspace
+as a place all its own.
+
+Because people live in it now. Not just a few people,
+not just a few technicians and eccentrics, but thousands
+of people, quite normal people. And not just for a little while,
+either, but for hours straight, over weeks, and months,
+and years. Cyberspace today is a "Net," a "Matrix,"
+international in scope and growing swiftly and steadily.
+It's growing in size, and wealth, and political importance.
+
+People are making entire careers in modern cyberspace.
+Scientists and technicians, of course; they've been there
+for twenty years now. But increasingly, cyberspace
+is filling with journalists and doctors and lawyers
+and artists and clerks. Civil servants make their
+careers there now, "on-line" in vast government data-banks;
+and so do spies, industrial, political, and just plain snoops;
+and so do police, at least a few of them. And there are children
+living there now.
+
+People have met there and been married there.
+There are entire living communities in cyberspace today;
+chattering, gossiping, planning, conferring and scheming,
+leaving one another voice-mail and electronic mail,
+giving one another big weightless chunks of valuable data,
+both legitimate and illegitimate. They busily pass one another
+computer software and the occasional festering computer virus.
+
+We do not really understand how to live in cyberspace yet.
+We are feeling our way into it, blundering about.
+That is not surprising. Our lives in the physical world,
+the "real" world, are also far from perfect, despite a lot more practice.
+Human lives, real lives, are imperfect by their nature, and there are
+human beings in cyberspace. The way we live in cyberspace is
+a funhouse mirror of the way we live in the real world.
+We take both our advantages and our troubles with us.
+
+This book is about trouble in cyberspace.
+Specifically, this book is about certain strange events in
+the year 1990, an unprecedented and startling year for the
+the growing world of computerized communications.
+
+In 1990 there came a nationwide crackdown on illicit
+computer hackers, with arrests, criminal charges,
+one dramatic show-trial, several guilty pleas, and
+huge confiscations of data and equipment all over the USA.
+
+The Hacker Crackdown of 1990 was larger, better organized,
+more deliberate, and more resolute than any previous effort
+in the brave new world of computer crime. The U.S. Secret Service,
+private telephone security, and state and local law enforcement groups
+across the country all joined forces in a determined attempt to break
+the back of America's electronic underground. It was a fascinating
+effort, with very mixed results.
+
+The Hacker Crackdown had another unprecedented effect;
+it spurred the creation, within "the computer community,"
+of the Electronic Frontier Foundation, a new and very odd
+interest group, fiercely dedicated to the establishment
+and preservation of electronic civil liberties. The crackdown,
+remarkable in itself, has created a melee of debate over electronic crime,
+punishment, freedom of the press, and issues of search and seizure.
+Politics has entered cyberspace. Where people go, politics follow.
+
+This is the story of the people of cyberspace.
+
+
+
+PART ONE: Crashing the System
+
+On January 15, 1990, AT&T's long-distance telephone switching system crashed.
+
+This was a strange, dire, huge event. Sixty thousand people lost
+their telephone service completely. During the nine long hours
+of frantic effort that it took to restore service, some seventy million
+telephone calls went uncompleted.
+
+Losses of service, known as "outages" in the telco trade,
+are a known and accepted hazard of the telephone business.
+Hurricanes hit, and phone cables get snapped by the thousands.
+Earthquakes wrench through buried fiber-optic lines.
+Switching stations catch fire and burn to the ground.
+These things do happen. There are contingency plans for them,
+and decades of experience in dealing with them.
+But the Crash of January 15 was unprecedented.
+It was unbelievably huge, and it occurred for
+no apparent physical reason.
+
+The crash started on a Monday afternoon in a single
+switching-station in Manhattan. But, unlike any merely
+physical damage, it spread and spread. Station after
+station across America collapsed in a chain reaction,
+until fully half of AT&T's network had gone haywire
+and the remaining half was hard-put to handle the overflow.
+
+Within nine hours, AT&T software engineers more or less
+understood what had caused the crash. Replicating the
+problem exactly, poring over software line by line,
+took them a couple of weeks. But because it was hard
+to understand technically, the full truth of the matter
+and its implications were not widely and thoroughly aired
+and explained. The root cause of the crash remained obscure,
+surrounded by rumor and fear.
+
+The crash was a grave corporate embarrassment.
+The "culprit" was a bug in AT&T's own software--not the
+sort of admission the telecommunications giant wanted
+to make, especially in the face of increasing competition.
+Still, the truth WAS told, in the baffling technical terms
+necessary to explain it.
+
+Somehow the explanation failed to persuade
+American law enforcement officials and even telephone
+corporate security personnel. These people were not
+technical experts or software wizards, and they had their
+own suspicions about the cause of this disaster.
+
+The police and telco security had important sources
+of information denied to mere software engineers.
+They had informants in the computer underground and
+years of experience in dealing with high-tech rascality
+that seemed to grow ever more sophisticated.
+For years they had been expecting a direct and
+savage attack against the American national telephone system.
+And with the Crash of January 15--the first month of a
+new, high-tech decade--their predictions, fears,
+and suspicions seemed at last to have entered the real world.
+A world where the telephone system had not merely crashed,
+but, quite likely, BEEN crashed--by "hackers."
+
+The crash created a large dark cloud of suspicion
+that would color certain people's assumptions and actions
+for months. The fact that it took place in the realm of
+software was suspicious on its face. The fact that it
+occurred on Martin Luther King Day, still the most
+politically touchy of American holidays, made it more
+suspicious yet.
+
+The Crash of January 15 gave the Hacker Crackdown
+its sense of edge and its sweaty urgency. It made people,
+powerful people in positions of public authority,
+willing to believe the worst. And, most fatally,
+it helped to give investigators a willingness
+to take extreme measures and the determination
+to preserve almost total secrecy.
+
+An obscure software fault in an aging switching system
+in New York was to lead to a chain reaction of legal
+and constitutional trouble all across the country.
+
+#
+
+Like the crash in the telephone system, this chain reaction
+was ready and waiting to happen. During the 1980s,
+the American legal system was extensively patched
+to deal with the novel issues of computer crime.
+There was, for instance, the Electronic Communications
+Privacy Act of 1986 (eloquently described as "a stinking mess"
+by a prominent law enforcement official). And there was the
+draconian Computer Fraud and Abuse Act of 1986, passed unanimously
+by the United States Senate, which later would reveal
+a large number of flaws. Extensive, well-meant efforts
+had been made to keep the legal system up to date.
+But in the day-to-day grind of the real world,
+even the most elegant software tends to crumble
+and suddenly reveal its hidden bugs.
+
+Like the advancing telephone system, the American legal system
+was certainly not ruined by its temporary crash; but for those
+caught under the weight of the collapsing system, life became
+a series of blackouts and anomalies.
+
+In order to understand why these weird events occurred,
+both in the world of technology and in the world of law,
+it's not enough to understand the merely technical problems.
+We will get to those; but first and foremost, we must try
+to understand the telephone, and the business of telephones,
+and the community of human beings that telephones have created.
+
+#
+
+Technologies have life cycles, like cities do,
+like institutions do, like laws and governments do.
+
+The first stage of any technology is the Question
+Mark, often known as the "Golden Vaporware" stage.
+At this early point, the technology is only a phantom,
+a mere gleam in the inventor's eye. One such inventor
+was a speech teacher and electrical tinkerer named
+Alexander Graham Bell.
+
+Bell's early inventions, while ingenious, failed to move the world.
+In 1863, the teenage Bell and his brother Melville made an artificial
+talking mechanism out of wood, rubber, gutta-percha, and tin.
+This weird device had a rubber-covered "tongue" made of movable
+wooden segments, with vibrating rubber "vocal cords," and
+rubber "lips" and "cheeks." While Melville puffed a bellows
+into a tin tube, imitating the lungs, young Alec Bell would
+manipulate the "lips," "teeth," and "tongue," causing the thing
+to emit high-pitched falsetto gibberish.
+
+Another would-be technical breakthrough was the Bell "phonautograph"
+of 1874, actually made out of a human cadaver's ear. Clamped into place
+on a tripod, this grisly gadget drew sound-wave images on smoked glass
+through a thin straw glued to its vibrating earbones.
+
+By 1875, Bell had learned to produce audible sounds--ugly shrieks
+and squawks--by using magnets, diaphragms, and electrical current.
+
+Most "Golden Vaporware" technologies go nowhere.
+
+But the second stage of technology is the Rising Star,
+or, the "Goofy Prototype," stage. The telephone, Bell's
+most ambitious gadget yet, reached this stage on March
+10, 1876. On that great day, Alexander Graham Bell
+became the first person to transmit intelligible human
+speech electrically. As it happened, young Professor Bell,
+industriously tinkering in his Boston lab, had spattered
+his trousers with acid. His assistant, Mr. Watson,
+heard his cry for help--over Bell's experimental
+audio-telegraph. This was an event without precedent.
+
+Technologies in their "Goofy Prototype" stage rarely
+work very well. They're experimental, and therefore
+half- baked and rather frazzled. The prototype may
+be attractive and novel, and it does look as if it ought
+to be good for something-or-other. But nobody, including
+the inventor, is quite sure what. Inventors, and speculators,
+and pundits may have very firm ideas about its potential
+use, but those ideas are often very wrong.
+
+The natural habitat of the Goofy Prototype is in trade shows
+and in the popular press. Infant technologies need publicity
+and investment money like a tottering calf need milk.
+This was very true of Bell's machine. To raise research and
+development money, Bell toured with his device as a stage attraction.
+
+Contemporary press reports of the stage debut of the telephone
+showed pleased astonishment mixed with considerable dread.
+Bell's stage telephone was a large wooden box with a crude
+speaker-nozzle, the whole contraption about the size and shape
+of an overgrown Brownie camera. Its buzzing steel soundplate,
+pumped up by powerful electromagnets, was loud enough to fill
+an auditorium. Bell's assistant Mr. Watson, who could manage
+on the keyboards fairly well, kicked in by playing the organ
+from distant rooms, and, later, distant cities. This feat was
+considered marvellous, but very eerie indeed.
+
+Bell's original notion for the telephone, an idea promoted
+for a couple of years, was that it would become a mass medium.
+We might recognize Bell's idea today as something close to modern
+"cable radio." Telephones at a central source would transmit music,
+Sunday sermons, and important public speeches to a paying network
+of wired-up subscribers.
+
+At the time, most people thought this notion made good sense.
+In fact, Bell's idea was workable. In Hungary, this philosophy
+of the telephone was successfully put into everyday practice.
+In Budapest, for decades, from 1893 until after World War I,
+there was a government-run information service called
+"Telefon Hirmondo-." Hirmondo- was a centralized source
+of news and entertainment and culture, including stock reports,
+plays, concerts, and novels read aloud. At certain hours
+of the day, the phone would ring, you would plug in
+a loudspeaker for the use of the family, and Telefon
+Hirmondo- would be on the air--or rather, on the phone.
+
+Hirmondo- is dead tech today, but Hirmondo- might be considered
+a spiritual ancestor of the modern telephone-accessed computer
+data services, such as CompuServe, GEnie or Prodigy.
+The principle behind Hirmondo- is also not too far from computer
+"bulletin- board systems" or BBS's, which arrived in the late 1970s,
+spread rapidly across America, and will figure largely in this book.
+
+We are used to using telephones for individual person-to-person speech,
+because we are used to the Bell system. But this was just one possibility
+among many. Communication networks are very flexible and protean,
+especially when their hardware becomes sufficiently advanced.
+They can be put to all kinds of uses. And they have been--
+and they will be.
+
+Bell's telephone was bound for glory, but this was a combination
+of political decisions, canny infighting in court, inspired industrial
+leadership, receptive local conditions and outright good luck.
+Much the same is true of communications systems today.
+
+As Bell and his backers struggled to install their newfangled system
+in the real world of nineteenth-century New England, they had to fight
+against skepticism and industrial rivalry. There was already a strong
+electrical communications network present in America: the telegraph.
+The head of the Western Union telegraph system dismissed Bell's prototype
+as "an electrical toy" and refused to buy the rights to Bell's patent.
+The telephone, it seemed, might be all right as a parlor entertainment--
+but not for serious business.
+
+Telegrams, unlike mere telephones, left a permanent physical record
+of their messages. Telegrams, unlike telephones, could be answered
+whenever the recipient had time and convenience. And the telegram
+had a much longer distance-range than Bell's early telephone.
+These factors made telegraphy seem a much more sound and businesslike
+technology--at least to some.
+
+The telegraph system was huge, and well-entrenched.
+In 1876, the United States had 214,000 miles of telegraph wire,
+and 8500 telegraph offices. There were specialized telegraphs
+for businesses and stock traders, government, police and fire departments.
+And Bell's "toy" was best known as a stage-magic musical device.
+
+The third stage of technology is known as the "Cash Cow" stage.
+In the "cash cow" stage, a technology finds its place in the world,
+and matures, and becomes settled and productive. After a year or so,
+Alexander Graham Bell and his capitalist backers concluded that
+eerie music piped from nineteenth-century cyberspace was not the real
+selling-point of his invention. Instead, the telephone was about speech--
+individual, personal speech, the human voice, human conversation and
+human interaction. The telephone was not to be managed from any centralized
+broadcast center. It was to be a personal, intimate technology.
+
+When you picked up a telephone, you were not absorbing
+the cold output of a machine--you were speaking to another human being.
+Once people realized this, their instinctive dread of the telephone
+as an eerie, unnatural device, swiftly vanished. A "telephone call"
+was not a "call" from a "telephone" itself, but a call from another
+human being, someone you would generally know and recognize.
+The real point was not what the machine could do for you (or to you),
+but what you yourself, a person and citizen, could do THROUGH the machine.
+This decision on the part of the young Bell Company was absolutely vital.
+
+The first telephone networks went up around Boston--mostly among
+the technically curious and the well-to-do (much the same segment
+of the American populace that, a hundred years later, would be
+buying personal computers). Entrenched backers of the telegraph
+continued to scoff.
+
+But in January 1878, a disaster made the telephone famous.
+A train crashed in Tarriffville, Connecticut. Forward-looking
+doctors in the nearby city of Hartford had had Bell's
+"speaking telephone" installed. An alert local druggist
+was able to telephone an entire community of local doctors,
+who rushed to the site to give aid. The disaster, as disasters do,
+aroused intense press coverage. The phone had proven its usefulness
+in the real world.
+
+After Tarriffville, the telephone network spread like crabgrass.
+By 1890 it was all over New England. By '93, out to Chicago.
+By '97, into Minnesota, Nebraska and Texas. By 1904 it was
+all over the continent.
+
+The telephone had become a mature technology. Professor Bell
+(now generally known as "Dr. Bell" despite his lack of a formal degree)
+became quite wealthy. He lost interest in the tedious day-to-day business
+muddle of the booming telephone network, and gratefully returned
+his attention to creatively hacking-around in his various laboratories,
+which were now much larger, better-ventilated, and gratifyingly
+better-equipped. Bell was never to have another great inventive success,
+though his speculations and prototypes anticipated fiber-optic transmission,
+manned flight, sonar, hydrofoil ships, tetrahedral construction, and
+Montessori education. The "decibel," the standard scientific measure
+of sound intensity, was named after Bell.
+
+Not all Bell's vaporware notions were inspired. He was fascinated
+by human eugenics. He also spent many years developing a weird personal
+system of astrophysics in which gravity did not exist.
+
+Bell was a definite eccentric. He was something of a hypochondriac,
+and throughout his life he habitually stayed up until four A.M.,
+refusing to rise before noon. But Bell had accomplished a great feat;
+he was an idol of millions and his influence, wealth, and great
+personal charm, combined with his eccentricity, made him something
+of a loose cannon on deck. Bell maintained a thriving scientific
+salon in his winter mansion in Washington, D.C., which gave him
+considerable backstage influence in governmental and scientific circles.
+He was a major financial backer of the the magazines Science and
+National Geographic, both still flourishing today as important organs
+of the American scientific establishment.
+
+Bell's companion Thomas Watson, similarly wealthy and similarly odd,
+became the ardent political disciple of a 19th-century science-fiction writer
+and would-be social reformer, Edward Bellamy. Watson also trod the boards
+briefly as a Shakespearian actor.
+
+There would never be another Alexander Graham Bell,
+but in years to come there would be surprising numbers
+of people like him. Bell was a prototype of the
+high-tech entrepreneur. High-tech entrepreneurs will
+play a very prominent role in this book: not merely as
+technicians and businessmen, but as pioneers of the
+technical frontier, who can carry the power and prestige
+they derive from high-technology into the political and
+social arena.
+
+Like later entrepreneurs, Bell was fierce in defense of
+his own technological territory. As the telephone began to
+flourish, Bell was soon involved in violent lawsuits in the
+defense of his patents. Bell's Boston lawyers were
+excellent, however, and Bell himself, as an elocution
+teacher and gifted public speaker, was a devastatingly
+effective legal witness. In the eighteen years of Bell's patents,
+the Bell company was involved in six hundred separate lawsuits.
+The legal records printed filled 149 volumes. The Bell Company
+won every single suit.
+
+After Bell's exclusive patents expired, rival telephone
+companies sprang up all over America. Bell's company,
+American Bell Telephone, was soon in deep trouble.
+In 1907, American Bell Telephone fell into the hands of the
+rather sinister J.P. Morgan financial cartel, robber-baron
+speculators who dominated Wall Street.
+
+At this point, history might have taken a different turn.
+American might well have been served forever by a patchwork
+of locally owned telephone companies. Many state politicians
+and local businessmen considered this an excellent solution.
+
+But the new Bell holding company, American Telephone and Telegraph
+or AT&T, put in a new man at the helm, a visionary industrialist
+named Theodore Vail. Vail, a former Post Office manager,
+understood large organizations and had an innate feeling
+for the nature of large-scale communications. Vail quickly
+saw to it that AT&T seized the technological edge once again.
+The Pupin and Campbell "loading coil," and the deForest
+"audion," are both extinct technology today, but in 1913
+they gave Vail's company the best LONG-DISTANCE lines
+ever built. By controlling long-distance--the links
+between, and over, and above the smaller local phone
+companies--AT&T swiftly gained the whip-hand over them,
+and was soon devouring them right and left.
+
+Vail plowed the profits back into research and development,
+starting the Bell tradition of huge-scale and brilliant
+industrial research.
+
+Technically and financially, AT&T gradually steamrollered
+the opposition. Independent telephone companies never
+became entirely extinct, and hundreds of them flourish today.
+But Vail's AT&T became the supreme communications company.
+At one point, Vail's AT&T bought Western Union itself,
+the very company that had derided Bell's telephone as a "toy."
+Vail thoroughly reformed Western Union's hidebound business
+along his modern principles; but when the federal government
+grew anxious at this centralization of power, Vail politely
+gave Western Union back.
+
+This centralizing process was not unique. Very similar
+events had happened in American steel, oil, and railroads.
+But AT&T, unlike the other companies, was to remain supreme.
+The monopoly robber-barons of those other industries
+were humbled and shattered by government trust-busting.
+
+Vail, the former Post Office official, was quite willing
+to accommodate the US government; in fact he would
+forge an active alliance with it. AT&T would become
+almost a wing of the American government, almost
+another Post Office--though not quite. AT&T would
+willingly submit to federal regulation, but in return,
+it would use the government's regulators as its own police,
+who would keep out competitors and assure the Bell
+system's profits and preeminence.
+
+This was the second birth--the political birth--of the
+American telephone system. Vail's arrangement was to
+persist, with vast success, for many decades, until 1982.
+His system was an odd kind of American industrial socialism.
+It was born at about the same time as Leninist Communism,
+and it lasted almost as long--and, it must be admitted,
+to considerably better effect.
+
+Vail's system worked. Except perhaps for aerospace,
+there has been no technology more thoroughly dominated
+by Americans than the telephone. The telephone was
+seen from the beginning as a quintessentially American
+technology. Bell's policy, and the policy of Theodore Vail,
+was a profoundly democratic policy of UNIVERSAL ACCESS.
+Vail's famous corporate slogan, "One Policy, One System,
+Universal Service," was a political slogan, with a very
+American ring to it.
+
+The American telephone was not to become the specialized tool
+of government or business, but a general public utility.
+At first, it was true, only the wealthy could afford
+private telephones, and Bell's company pursued the
+business markets primarily. The American phone system
+was a capitalist effort, meant to make money; it was not a charity.
+But from the first, almost all communities with telephone service
+had public telephones. And many stores--especially drugstores--
+offered public use of their phones. You might not own a telephone--
+but you could always get into the system, if you really needed to.
+
+There was nothing inevitable about this decision to make telephones
+"public" and "universal." Vail's system involved a profound act
+of trust in the public. This decision was a political one,
+informed by the basic values of the American republic.
+The situation might have been very different;
+and in other countries, under other systems,
+it certainly was.
+
+Joseph Stalin, for instance, vetoed plans for a Soviet
+phone system soon after the Bolshevik revolution.
+Stalin was certain that publicly accessible telephones
+would become instruments of anti-Soviet counterrevolution
+and conspiracy. (He was probably right.) When telephones
+did arrive in the Soviet Union, they would be instruments
+of Party authority, and always heavily tapped. (Alexander
+Solzhenitsyn's prison-camp novel The First Circle
+describes efforts to develop a phone system more suited
+to Stalinist purposes.)
+
+France, with its tradition of rational centralized government,
+had fought bitterly even against the electric telegraph,
+which seemed to the French entirely too anarchical and frivolous.
+For decades, nineteenth-century France communicated via the
+"visual telegraph," a nation-spanning, government-owned semaphore
+system of huge stone towers that signalled from hilltops,
+across vast distances, with big windmill-like arms.
+In 1846, one Dr. Barbay, a semaphore enthusiast,
+memorably uttered an early version of what might be called
+"the security expert's argument" against the open media.
+
+"No, the electric telegraph is not a sound invention.
+It will always be at the mercy of the slightest disruption,
+wild youths, drunkards, bums, etc. . . . The electric telegraph
+meets those destructive elements with only a few meters of wire
+over which supervision is impossible. A single man could,
+without being seen, cut the telegraph wires leading to Paris,
+and in twenty-four hours cut in ten different places the wires
+of the same line, without being arrested. The visual telegraph,
+on the contrary, has its towers, its high walls, its gates
+well-guarded from inside by strong armed men. Yes, I declare,
+substitution of the electric telegraph for the visual one
+is a dreadful measure, a truly idiotic act."
+
+Dr. Barbay and his high-security stone machines
+were eventually unsuccessful, but his argument--
+that communication exists for the safety and convenience
+of the state, and must be carefully protected from the wild
+boys and the gutter rabble who might want to crash the
+system--would be heard again and again.
+
+When the French telephone system finally did arrive,
+its snarled inadequacy was to be notorious. Devotees
+of the American Bell System often recommended a trip
+to France, for skeptics.
+
+In Edwardian Britain, issues of class and privacy
+were a ball-and-chain for telephonic progress. It was
+considered outrageous that anyone--any wild fool off
+the street--could simply barge bellowing into one's office
+or home, preceded only by the ringing of a telephone bell.
+In Britain, phones were tolerated for the use of business,
+but private phones tended be stuffed away into closets,
+smoking rooms, or servants' quarters. Telephone operators
+were resented in Britain because they did not seem to
+"know their place." And no one of breeding would print
+a telephone number on a business card; this seemed a crass
+attempt to make the acquaintance of strangers.
+
+But phone access in America was to become a popular right;
+something like universal suffrage, only more so.
+American women could not yet vote when the phone system
+came through; yet from the beginning American women
+doted on the telephone. This "feminization" of the
+American telephone was often commented on by foreigners.
+Phones in America were not censored or stiff or formalized;
+they were social, private, intimate, and domestic.
+In America, Mother's Day is by far the busiest day
+of the year for the phone network.
+
+The early telephone companies, and especially AT&T,
+were among the foremost employers of American women.
+They employed the daughters of the American middle-class
+in great armies: in 1891, eight thousand women; by 1946,
+almost a quarter of a million. Women seemed to enjoy
+telephone work; it was respectable, it was steady,
+it paid fairly well as women's work went, and--not least--
+it seemed a genuine contribution to the social good
+of the community. Women found Vail's ideal of public
+service attractive. This was especially true in rural areas,
+where women operators, running extensive rural party-lines,
+enjoyed considerable social power. The operator knew everyone
+on the party-line, and everyone knew her.
+
+Although Bell himself was an ardent suffragist, the
+telephone company did not employ women for the sake of
+advancing female liberation. AT&T did this for sound
+commercial reasons. The first telephone operators of
+the Bell system were not women, but teenage American boys.
+They were telegraphic messenger boys (a group about to
+be rendered technically obsolescent), who swept up
+around the phone office, dunned customers for bills,
+and made phone connections on the switchboard,
+all on the cheap.
+
+Within the very first year of operation, 1878,
+Bell's company learned a sharp lesson about combining
+teenage boys and telephone switchboards. Putting
+teenage boys in charge of the phone system brought swift
+and consistent disaster. Bell's chief engineer described them
+as "Wild Indians." The boys were openly rude to customers.
+They talked back to subscribers, saucing off,
+uttering facetious remarks, and generally giving lip.
+The rascals took Saint Patrick's Day off without permission.
+And worst of all they played clever tricks with
+the switchboard plugs: disconnecting calls, crossing lines
+so that customers found themselves talking to strangers,
+and so forth.
+
+This combination of power, technical mastery, and effective
+anonymity seemed to act like catnip on teenage boys.
+
+This wild-kid-on-the-wires phenomenon was not confined to
+the USA; from the beginning, the same was true of the British
+phone system. An early British commentator kindly remarked:
+"No doubt boys in their teens found the work not a little irksome,
+and it is also highly probable that under the early conditions
+of employment the adventurous and inquisitive spirits of which
+the average healthy boy of that age is possessed, were not always
+conducive to the best attention being given to the wants
+of the telephone subscribers."
+
+So the boys were flung off the system--or at least,
+deprived of control of the switchboard. But the
+"adventurous and inquisitive spirits" of the teenage boys
+would be heard from in the world of telephony, again and again.
+
+The fourth stage in the technological life-cycle is death:
+"the Dog," dead tech. The telephone has so far avoided this fate.
+On the contrary, it is thriving, still spreading, still evolving,
+and at increasing speed.
+
+The telephone has achieved a rare and exalted state for a
+technological artifact: it has become a HOUSEHOLD OBJECT.
+The telephone, like the clock, like pen and paper,
+like kitchen utensils and running water, has become
+a technology that is visible only by its absence.
+The telephone is technologically transparent.
+The global telephone system is the largest and most
+complex machine in the world, yet it is easy to use.
+More remarkable yet, the telephone is almost entirely
+physically safe for the user.
+
+For the average citizen in the 1870s, the telephone
+was weirder, more shocking, more "high-tech" and
+harder to comprehend, than the most outrageous stunts
+of advanced computing for us Americans in the 1990s.
+In trying to understand what is happening to us today,
+with our bulletin-board systems, direct overseas dialling,
+fiber-optic transmissions, computer viruses, hacking stunts,
+and a vivid tangle of new laws and new crimes, it is important
+to realize that our society has been through a similar challenge before--
+and that, all in all, we did rather well by it.
+
+Bell's stage telephone seemed bizarre at first. But the
+sensations of weirdness vanished quickly, once people began
+to hear the familiar voices of relatives and friends,
+in their own homes on their own telephones. The telephone
+changed from a fearsome high-tech totem to an everyday pillar
+of human community.
+
+This has also happened, and is still happening,
+to computer networks. Computer networks such as
+NSFnet, BITnet, USENET, JANET, are technically
+advanced, intimidating, and much harder to use than
+telephones. Even the popular, commercial computer
+networks, such as GEnie, Prodigy, and CompuServe,
+cause much head-scratching and have been described
+as "user-hateful." Nevertheless they too are changing
+from fancy high-tech items into everyday sources
+of human community.
+
+The words "community" and "communication" have
+the same root. Wherever you put a communications
+network, you put a community as well. And whenever
+you TAKE AWAY that network--confiscate it, outlaw it,
+crash it, raise its price beyond affordability--
+then you hurt that community.
+
+Communities will fight to defend themselves. People will fight harder
+and more bitterly to defend their communities, than they will fight
+to defend their own individual selves. And this is very true
+of the "electronic community" that arose around computer networks
+in the 1980s--or rather, the VARIOUS electronic communities,
+in telephony, law enforcement, computing, and the digital
+underground that, by the year 1990, were raiding, rallying,
+arresting, suing, jailing, fining and issuing angry manifestos.
+
+None of the events of 1990 were entirely new.
+Nothing happened in 1990 that did not have some kind
+of earlier and more understandable precedent. What gave
+the Hacker Crackdown its new sense of gravity and
+importance was the feeling--the COMMUNITY feeling--
+that the political stakes had been raised; that trouble
+in cyberspace was no longer mere mischief or inconclusive
+skirmishing, but a genuine fight over genuine issues,
+a fight for community survival and the shape of the future.
+
+These electronic communities, having flourished throughout
+the 1980s, were becoming aware of themselves, and increasingly,
+becoming aware of other, rival communities. Worries were
+sprouting up right and left, with complaints, rumors,
+uneasy speculations. But it would take a catalyst, a shock,
+to make the new world evident. Like Bell's great publicity break,
+the Tarriffville Rail Disaster of January 1878,
+it would take a cause celebre.
+
+That cause was the AT&T Crash of January 15, 1990.
+After the Crash, the wounded and anxious telephone
+community would come out fighting hard.
+
+#
+
+The community of telephone technicians, engineers, operators
+and researchers is the oldest community in cyberspace.
+These are the veterans, the most developed group,
+the richest, the most respectable, in most ways the most powerful.
+Whole generations have come and gone since Alexander Graham Bell's day,
+but the community he founded survives; people work for the phone system
+today whose great-grandparents worked for the phone system.
+Its specialty magazines, such as Telephony, AT&T Technical Journal,
+Telephone Engineer and Management, are decades old;
+they make computer publications like Macworld and PC Week
+look like amateur johnny-come-latelies.
+
+And the phone companies take no back seat in high-technology, either.
+Other companies' industrial researchers may have won new markets;
+but the researchers of Bell Labs have won SEVEN NOBEL PRIZES.
+One potent device that Bell Labs originated, the transistor,
+has created entire GROUPS of industries. Bell Labs are
+world-famous for generating "a patent a day," and have even
+made vital discoveries in astronomy, physics and cosmology.
+
+Throughout its seventy-year history, "Ma Bell" was not so much
+a company as a way of life. Until the cataclysmic divestiture
+of the 1980s, Ma Bell was perhaps the ultimate maternalist mega-employer.
+The AT&T corporate image was the "gentle giant," "the voice with a smile,"
+a vaguely socialist-realist world of cleanshaven linemen in shiny helmets
+and blandly pretty phone-girls in headsets and nylons. Bell System
+employees were famous as rock-ribbed Kiwanis and Rotary members,
+Little-League enthusiasts, school-board people.
+
+During the long heyday of Ma Bell, the Bell employee corps
+were nurtured top-to-bottom on a corporate ethos of public service.
+There was good money in Bell, but Bell was not ABOUT money;
+Bell used public relations, but never mere marketeering.
+People went into the Bell System for a good life,
+and they had a good life. But it was not mere money
+that led Bell people out in the midst of storms and earthquakes
+to fight with toppled phone-poles, to wade in flooded manholes,
+to pull the red-eyed graveyard-shift over collapsing switching-systems.
+The Bell ethic was the electrical equivalent of the postman's:
+neither rain, nor snow, nor gloom of night would stop these couriers.
+
+It is easy to be cynical about this, as it is easy to be
+cynical about any political or social system; but cynicism
+does not change the fact that thousands of people took
+these ideals very seriously. And some still do.
+
+The Bell ethos was about public service; and that was
+gratifying; but it was also about private POWER, and that
+was gratifying too. As a corporation, Bell was very special.
+Bell was privileged. Bell had snuggled up close to the state.
+In fact, Bell was as close to government as you could get in
+America and still make a whole lot of legitimate money.
+
+But unlike other companies, Bell was above and beyond
+the vulgar commercial fray. Through its regional operating companies,
+Bell was omnipresent, local, and intimate, all over America;
+but the central ivory towers at its corporate heart were the
+tallest and the ivoriest around.
+
+There were other phone companies in America, to be sure;
+the so-called independents. Rural cooperatives, mostly;
+small fry, mostly tolerated, sometimes warred upon.
+For many decades, "independent" American phone companies
+lived in fear and loathing of the official Bell monopoly
+(or the "Bell Octopus," as Ma Bell's nineteenth-century
+enemies described her in many angry newspaper manifestos).
+Some few of these independent entrepreneurs, while legally
+in the wrong, fought so bitterly against the Octopus
+that their illegal phone networks were cast into the street
+by Bell agents and publicly burned.
+
+The pure technical sweetness of the Bell System gave its operators,
+inventors and engineers a deeply satisfying sense of power and mastery.
+They had devoted their lives to improving this vast nation-spanning machine;
+over years, whole human lives, they had watched it improve and grow.
+It was like a great technological temple. They were an elite,
+and they knew it--even if others did not; in fact, they felt
+even more powerful BECAUSE others did not understand.
+
+The deep attraction of this sensation of elite technical power
+should never be underestimated. "Technical power" is not for everybody;
+for many people it simply has no charm at all. But for some people,
+it becomes the core of their lives. For a few, it is overwhelming,
+obsessive; it becomes something close to an addiction. People--especially
+clever teenage boys whose lives are otherwise mostly powerless and put-upon
+--love this sensation of secret power, and are willing to do all sorts
+of amazing things to achieve it. The technical POWER of electronics
+has motivated many strange acts detailed in this book, which would
+otherwise be inexplicable.
+
+So Bell had power beyond mere capitalism. The Bell service ethos worked,
+and was often propagandized, in a rather saccharine fashion. Over the decades,
+people slowly grew tired of this. And then, openly impatient with it.
+By the early 1980s, Ma Bell was to find herself with scarcely a real friend
+in the world. Vail's industrial socialism had become hopelessly
+out-of-fashion politically. Bell would be punished for that.
+And that punishment would fall harshly upon the people of the
+telephone community.
+
+#
+
+In 1983, Ma Bell was dismantled by federal court action.
+The pieces of Bell are now separate corporate entities.
+The core of the company became AT&T Communications,
+and also AT&T Industries (formerly Western Electric,
+Bell's manufacturing arm). AT&T Bell Labs became Bell
+Communications Research, Bellcore. Then there are the
+Regional Bell Operating Companies, or RBOCs, pronounced "arbocks."
+
+Bell was a titan and even these regional chunks are gigantic enterprises:
+Fortune 50 companies with plenty of wealth and power behind them.
+But the clean lines of "One Policy, One System, Universal Service"
+have been shattered, apparently forever.
+
+The "One Policy" of the early Reagan Administration was to
+shatter a system that smacked of noncompetitive socialism.
+Since that time, there has been no real telephone "policy"
+on the federal level. Despite the breakup, the remnants
+of Bell have never been set free to compete in the open marketplace.
+
+The RBOCs are still very heavily regulated, but not from the top.
+Instead, they struggle politically, economically and legally,
+in what seems an endless turmoil, in a patchwork of overlapping federal
+and state jurisdictions. Increasingly, like other major American corporations,
+the RBOCs are becoming multinational, acquiring important commercial interests
+in Europe, Latin America, and the Pacific Rim. But this, too, adds to their
+legal and political predicament.
+
+The people of what used to be Ma Bell are not happy about their fate.
+They feel ill-used. They might have been grudgingly willing to make
+a full transition to the free market; to become just companies amid
+other companies. But this never happened. Instead, AT&T and the RBOCS
+("the Baby Bells") feel themselves wrenched from side to side by state
+regulators, by Congress, by the FCC, and especially by the federal court
+of Judge Harold Greene, the magistrate who ordered the Bell breakup
+and who has been the de facto czar of American telecommunications
+ever since 1983.
+
+Bell people feel that they exist in a kind of paralegal limbo today.
+They don't understand what's demanded of them. If it's "service,"
+why aren't they treated like a public service? And if it's money,
+then why aren't they free to compete for it? No one seems to know,
+really. Those who claim to know keep changing their minds.
+Nobody in authority seems willing to grasp the nettle for once and all.
+
+Telephone people from other countries are amazed by the
+American telephone system today. Not that it works so well;
+for nowadays even the French telephone system works, more or less.
+They are amazed that the American telephone system STILL works
+AT ALL, under these strange conditions.
+
+Bell's "One System" of long-distance service is now only about
+eighty percent of a system, with the remainder held by Sprint, MCI,
+and the midget long-distance companies. Ugly wars over dubious
+corporate practices such as "slamming" (an underhanded method
+of snitching clients from rivals) break out with some regularity
+in the realm of long-distance service. The battle to break Bell's
+long-distance monopoly was long and ugly, and since the breakup
+the battlefield has not become much prettier. AT&T's famous
+shame-and-blame advertisements, which emphasized the shoddy work
+and purported ethical shadiness of their competitors, were much
+remarked on for their studied psychological cruelty.
+
+There is much bad blood in this industry, and much
+long-treasured resentment. AT&T's post-breakup
+corporate logo, a striped sphere, is known in the
+industry as the "Death Star" (a reference from the movie
+Star Wars, in which the "Death Star" was the spherical
+high- tech fortress of the harsh-breathing imperial ultra-baddie,
+Darth Vader.) Even AT&T employees are less than thrilled
+by the Death Star. A popular (though banned) T-shirt among
+AT&T employees bears the old-fashioned Bell logo of the Bell System,
+plus the newfangled striped sphere, with the before-and-after comments:
+"This is your brain--This is your brain on drugs!" AT&T made a very
+well-financed and determined effort to break into the personal
+computer market; it was disastrous, and telco computer experts
+are derisively known by their competitors as "the pole-climbers."
+AT&T and the Baby Bell arbocks still seem to have few friends.
+
+Under conditions of sharp commercial competition, a crash like
+that of January 15, 1990 was a major embarrassment to AT&T.
+It was a direct blow against their much-treasured reputation
+for reliability. Within days of the crash AT&T's
+Chief Executive Officer, Bob Allen, officially apologized,
+in terms of deeply pained humility:
+
+"AT&T had a major service disruption last Monday.
+We didn't live up to our own standards of quality,
+and we didn't live up to yours. It's as simple as that.
+And that's not acceptable to us. Or to you. . . .
+We understand how much people have come to depend
+upon AT&T service, so our AT&T Bell Laboratories scientists
+and our network engineers are doing everything possible
+to guard against a recurrence. . . . We know there's no way
+to make up for the inconvenience this problem may have caused you."
+
+Mr Allen's "open letter to customers" was printed in lavish ads
+all over the country: in the Wall Street Journal, USA Today,
+New York Times, Los Angeles Times, Chicago Tribune,
+Philadelphia Inquirer, San Francisco Chronicle Examiner,
+Boston Globe, Dallas Morning News, Detroit Free Press,
+Washington Post, Houston Chronicle, Cleveland Plain Dealer,
+Atlanta Journal Constitution, Minneapolis Star Tribune,
+St. Paul Pioneer Press Dispatch, Seattle Times/Post Intelligencer,
+Tacoma News Tribune, Miami Herald, Pittsburgh Press,
+St. Louis Post Dispatch, Denver Post, Phoenix Republic Gazette
+and Tampa Tribune.
+
+In another press release, AT&T went to some pains to suggest
+that this "software glitch" might have happened just as easily to MCI,
+although, in fact, it hadn't. (MCI's switching software was quite different
+from AT&T's--though not necessarily any safer.) AT&T also announced
+their plans to offer a rebate of service on Valentine's Day to make up
+for the loss during the Crash.
+
+"Every technical resource available, including Bell Labs
+scientists and engineers, has been devoted to assuring
+it will not occur again," the public was told. They were
+further assured that "The chances of a recurrence are small--
+a problem of this magnitude never occurred before."
+
+In the meantime, however, police and corporate
+security maintained their own suspicions about
+"the chances of recurrence" and the real reason why
+a "problem of this magnitude" had appeared, seemingly
+out of nowhere. Police and security knew for a fact
+that hackers of unprecedented sophistication were illegally
+entering, and reprogramming, certain digital switching stations.
+Rumors of hidden "viruses" and secret "logic bombs"
+in the switches ran rampant in the underground,
+with much chortling over AT&T's predicament,
+and idle speculation over what unsung hacker genius
+was responsible for it. Some hackers, including police
+informants, were trying hard to finger one another
+as the true culprits of the Crash.
+
+Telco people found little comfort in objectivity when
+they contemplated these possibilities. It was just too close
+to the bone for them; it was embarrassing; it hurt so much,
+it was hard even to talk about.
+
+There has always been thieving and misbehavior in the phone system.
+There has always been trouble with the rival independents,
+and in the local loops. But to have such trouble in the core
+of the system, the long-distance switching stations,
+is a horrifying affair. To telco people, this is
+all the difference between finding roaches in your kitchen
+and big horrid sewer-rats in your bedroom.
+
+From the outside, to the average citizen, the telcos
+still seem gigantic and impersonal. The American public
+seems to regard them as something akin to Soviet apparats.
+Even when the telcos do their best corporate-citizen routine,
+subsidizing magnet high-schools and sponsoring news-shows
+on public television, they seem to win little except public suspicion.
+
+But from the inside, all this looks very different.
+There's harsh competition. A legal and political system
+that seems baffled and bored, when not actively hostile
+to telco interests. There's a loss of morale, a deep sensation
+of having somehow lost the upper hand. Technological change
+has caused a loss of data and revenue to other, newer forms
+of transmission. There's theft, and new forms of theft,
+of growing scale and boldness and sophistication.
+With all these factors, it was no surprise to see the telcos,
+large and small, break out in a litany of bitter complaint.
+
+In late '88 and throughout 1989, telco representatives
+grew shrill in their complaints to those few American law
+enforcement officials who make it their business to try to
+understand what telephone people are talking about.
+Telco security officials had discovered the computer-
+hacker underground, infiltrated it thoroughly,
+and become deeply alarmed at its growing expertise.
+Here they had found a target that was not only loathsome
+on its face, but clearly ripe for counterattack.
+
+Those bitter rivals: AT&T, MCI and Sprint--and a crowd
+of Baby Bells: PacBell, Bell South, Southwestern Bell,
+NYNEX, USWest, as well as the Bell research consortium Bellcore,
+and the independent long-distance carrier Mid-American--
+all were to have their role in the great hacker dragnet of 1990.
+After years of being battered and pushed around, the telcos had,
+at least in a small way, seized the initiative again.
+After years of turmoil, telcos and government officials were
+once again to work smoothly in concert in defense of the System.
+Optimism blossomed; enthusiasm grew on all sides;
+the prospective taste of vengeance was sweet.
+
+#
+
+From the beginning--even before the crackdown had a name--
+secrecy was a big problem. There were many good reasons
+for secrecy in the hacker crackdown. Hackers and code-thieves
+were wily prey, slinking back to their bedrooms and basements
+and destroying vital incriminating evidence at the first hint of trouble.
+Furthermore, the crimes themselves were heavily technical and difficult
+to describe, even to police--much less to the general public.
+
+When such crimes HAD been described intelligibly to the public,
+in the past, that very publicity had tended to INCREASE the crimes
+enormously. Telco officials, while painfully aware of the vulnerabilities
+of their systems, were anxious not to publicize those weaknesses.
+Experience showed them that those weaknesses, once discovered,
+would be pitilessly exploited by tens of thousands of people--not only
+by professional grifters and by underground hackers and phone phreaks,
+but by many otherwise more-or-less honest everyday folks, who regarded
+stealing service from the faceless, soulless "Phone Company" as a kind of
+harmless indoor sport. When it came to protecting their interests,
+telcos had long since given up on general public sympathy for
+"the Voice with a Smile." Nowadays the telco's "Voice" was
+very likely to be a computer's; and the American public
+showed much less of the proper respect and gratitude due
+the fine public service bequeathed them by Dr. Bell and Mr. Vail.
+The more efficient, high-tech, computerized, and impersonal
+the telcos became, it seemed, the more they were met by
+sullen public resentment and amoral greed.
+
+Telco officials wanted to punish the phone-phreak underground, in as
+public and exemplary a manner as possible. They wanted to make dire
+examples of the worst offenders, to seize the ringleaders and intimidate
+the small fry, to discourage and frighten the wacky hobbyists, and send
+the professional grifters to jail. To do all this, publicity was vital.
+
+Yet operational secrecy was even more so. If word got out that
+a nationwide crackdown was coming, the hackers might simply vanish;
+destroy the evidence, hide their computers, go to earth,
+and wait for the campaign to blow over. Even the young
+hackers were crafty and suspicious, and as for the professional grifters,
+they tended to split for the nearest state-line at the first sign of trouble.
+For the crackdown to work well, they would all have to be caught red-handed,
+swept upon suddenly, out of the blue, from every corner of the compass.
+
+And there was another strong motive for secrecy. In the worst-case scenario,
+a blown campaign might leave the telcos open to a devastating hacker
+counter-attack. If there were indeed hackers loose in America who
+had caused the January 15 Crash--if there were truly gifted hackers,
+loose in the nation's long-distance switching systems, and enraged
+or frightened by the crackdown--then they might react unpredictably
+to an attempt to collar them. Even if caught, they might have talented
+and vengeful friends still running around loose. Conceivably,
+it could turn ugly. Very ugly. In fact, it was hard to imagine
+just how ugly things might turn, given that possibility.
+
+Counter-attack from hackers was a genuine concern for the telcos.
+In point of fact, they would never suffer any such counter-attack.
+But in months to come, they would be at some pains to publicize
+this notion and to utter grim warnings about it.
+
+Still, that risk seemed well worth running. Better to run the risk
+of vengeful attacks, than to live at the mercy of potential crashers.
+Any cop would tell you that a protection racket had no real future.
+
+And publicity was such a useful thing. Corporate security officers,
+including telco security, generally work under conditions of great discretion.
+And corporate security officials do not make money for their companies.
+Their job is to PREVENT THE LOSS of money, which is much less glamorous
+than actually winning profits.
+
+If you are a corporate security official, and you do your job brilliantly,
+then nothing bad happens to your company at all. Because of this, you appear
+completely superfluous. This is one of the many unattractive aspects
+of security work. It's rare that these folks have the chance to draw
+some healthy attention to their own efforts.
+
+Publicity also served the interest of their friends in law enforcement.
+Public officials, including law enforcement officials, thrive by attracting
+favorable public interest. A brilliant prosecution in a matter of vital
+public interest can make the career of a prosecuting attorney.
+And for a police officer, good publicity opens the purses of the legislature;
+it may bring a citation, or a promotion, or at least a rise in status
+and the respect of one's peers.
+
+But to have both publicity and secrecy is to have one's cake and eat it too.
+In months to come, as we will show, this impossible act was to cause great
+pain to the agents of the crackdown. But early on, it seemed possible
+--maybe even likely--that the crackdown could successfully combine
+the best of both worlds. The ARREST of hackers would be heavily publicized.
+The actual DEEDS of the hackers, which were technically hard to explain
+and also a security risk, would be left decently obscured. The THREAT
+hackers posed would be heavily trumpeted; the likelihood of their actually
+committing such fearsome crimes would be left to the public's imagination.
+The spread of the computer underground, and its growing technical
+sophistication, would be heavily promoted; the actual hackers themselves,
+mostly bespectacled middle-class white suburban teenagers,
+would be denied any personal publicity.
+
+It does not seem to have occurred to any telco official
+that the hackers accused would demand a day in court;
+that journalists would smile upon the hackers as
+"good copy;" that wealthy high-tech entrepreneurs would
+offer moral and financial support to crackdown victims;
+that constitutional lawyers would show up with briefcases,
+frowning mightily. This possibility does not seem to have
+ever entered the game-plan.
+
+And even if it had, it probably would not have slowed
+the ferocious pursuit of a stolen phone-company document,
+mellifluously known as "Control Office Administration of
+Enhanced 911 Services for Special Services and Major Account Centers."
+
+In the chapters to follow, we will explore the worlds
+of police and the computer underground, and the large
+shadowy area where they overlap. But first, we must
+explore the battleground. Before we leave the world
+of the telcos, we must understand what a switching system
+actually is and how your telephone actually works.
+
+#
+
+To the average citizen, the idea of the telephone is represented by,
+well, a TELEPHONE: a device that you talk into. To a telco
+professional, however, the telephone itself is known, in lordly
+fashion, as a "subset." The "subset" in your house is a mere adjunct,
+a distant nerve ending, of the central switching stations,
+which are ranked in levels of heirarchy, up to the long-distance electronic
+switching stations, which are some of the largest computers on earth.
+
+Let us imagine that it is, say, 1925, before the
+introduction of computers, when the phone system was
+simpler and somewhat easier to grasp. Let's further
+imagine that you are Miss Leticia Luthor, a fictional
+operator for Ma Bell in New York City of the 20s.
+
+Basically, you, Miss Luthor, ARE the "switching system."
+You are sitting in front of a large vertical switchboard,
+known as a "cordboard," made of shiny wooden panels,
+with ten thousand metal-rimmed holes punched in them,
+known as jacks. The engineers would have put more
+holes into your switchboard, but ten thousand is
+as many as you can reach without actually having
+to get up out of your chair.
+
+Each of these ten thousand holes has its own little electric lightbulb,
+known as a "lamp," and its own neatly printed number code.
+
+With the ease of long habit, you are scanning your board for lit-up bulbs.
+This is what you do most of the time, so you are used to it.
+
+A lamp lights up. This means that the phone
+at the end of that line has been taken off the hook.
+Whenever a handset is taken off the hook, that closes a circuit
+inside the phone which then signals the local office, i.e. you,
+automatically. There might be somebody calling, or then
+again the phone might be simply off the hook, but this
+does not matter to you yet. The first thing you do,
+is record that number in your logbook, in your fine American
+public-school handwriting. This comes first, naturally,
+since it is done for billing purposes.
+
+You now take the plug of your answering cord, which goes
+directly to your headset, and plug it into the lit-up hole.
+"Operator," you announce.
+
+In operator's classes, before taking this job, you have
+been issued a large pamphlet full of canned operator's
+responses for all kinds of contingencies, which you had
+to memorize. You have also been trained in a proper
+non-regional, non-ethnic pronunciation and tone of voice.
+You rarely have the occasion to make any spontaneous
+remark to a customer, and in fact this is frowned upon
+(except out on the rural lines where people have time
+on their hands and get up to all kinds of mischief).
+
+A tough-sounding user's voice at the end of the line
+gives you a number. Immediately, you write that number
+down in your logbook, next to the caller's number,
+which you just wrote earlier. You then look and see if
+the number this guy wants is in fact on your switchboard,
+which it generally is, since it's generally a local call.
+Long distance costs so much that people use it sparingly.
+
+Only then do you pick up a calling-cord from a shelf
+at the base of the switchboard. This is a long elastic cord
+mounted on a kind of reel so that it will zip back in when
+you unplug it. There are a lot of cords down there,
+and when a bunch of them are out at once they look like
+a nest of snakes. Some of the girls think there are bugs
+living in those cable-holes. They're called "cable mites"
+and are supposed to bite your hands and give you rashes.
+You don't believe this, yourself.
+
+Gripping the head of your calling-cord, you slip the tip
+of it deftly into the sleeve of the jack for the called person.
+Not all the way in, though. You just touch it. If you hear
+a clicking sound, that means the line is busy and you can't
+put the call through. If the line is busy, you have to stick
+the calling-cord into a "busy-tone jack," which will give
+the guy a busy-tone. This way you don't have to talk to him
+yourself and absorb his natural human frustration.
+
+But the line isn't busy. So you pop the cord all the way in.
+Relay circuits in your board make the distant phone ring,
+and if somebody picks it up off the hook, then a phone
+conversation starts. You can hear this conversation
+on your answering cord, until you unplug it. In fact
+you could listen to the whole conversation if you wanted,
+but this is sternly frowned upon by management, and frankly,
+when you've overheard one, you've pretty much heard 'em all.
+
+You can tell how long the conversation lasts by the glow
+of the calling-cord's lamp, down on the calling-cord's shelf.
+When it's over, you unplug and the calling-cord zips back into place.
+
+Having done this stuff a few hundred thousand times,
+you become quite good at it. In fact you're plugging,
+and connecting, and disconnecting, ten, twenty, forty cords
+at a time. It's a manual handicraft, really, quite satisfying
+in a way, rather like weaving on an upright loom.
+
+Should a long-distance call come up, it would be different,
+but not all that different. Instead of connecting the call
+through your own local switchboard, you have to go up the hierarchy,
+onto the long-distance lines, known as "trunklines."
+Depending on how far the call goes, it may have to work
+its way through a whole series of operators, which can
+take quite a while. The caller doesn't wait on the line
+while this complex process is negotiated across the country
+by the gaggle of operators. Instead, the caller hangs up,
+and you call him back yourself when the call has finally
+worked its way through.
+
+After four or five years of this work, you get married,
+and you have to quit your job, this being the natural order
+of womanhood in the American 1920s. The phone company
+has to train somebody else--maybe two people, since
+the phone system has grown somewhat in the meantime.
+And this costs money.
+
+In fact, to use any kind of human being as a switching
+system is a very expensive proposition. Eight thousand
+Leticia Luthors would be bad enough, but a quarter of a
+million of them is a military-scale proposition and makes
+drastic measures in automation financially worthwhile.
+
+Although the phone system continues to grow today,
+the number of human beings employed by telcos has
+been dropping steadily for years. Phone "operators"
+now deal with nothing but unusual contingencies,
+all routine operations having been shrugged off onto machines.
+Consequently, telephone operators are considerably less
+machine-like nowadays, and have been known to have accents
+and actual character in their voices. When you reach
+a human operator today, the operators are rather more
+"human" than they were in Leticia's day--but on the other hand,
+human beings in the phone system are much harder to reach
+in the first place.
+
+Over the first half of the twentieth century,
+"electromechanical" switching systems of growing
+complexity were cautiously introduced into the phone system.
+In certain backwaters, some of these hybrid systems are still
+in use. But after 1965, the phone system began to go completely
+electronic, and this is by far the dominant mode today.
+Electromechanical systems have "crossbars," and "brushes,"
+and other large moving mechanical parts, which, while faster
+and cheaper than Leticia, are still slow, and tend to wear out
+fairly quickly.
+
+But fully electronic systems are inscribed on silicon chips,
+and are lightning-fast, very cheap, and quite durable.
+They are much cheaper to maintain than even the best
+electromechanical systems, and they fit into half the space.
+And with every year, the silicon chip grows smaller, faster,
+and cheaper yet. Best of all, automated electronics work
+around the clock and don't have salaries or health insurance.
+
+There are, however, quite serious drawbacks to the
+use of computer-chips. When they do break down, it is
+a daunting challenge to figure out what the heck has gone
+wrong with them. A broken cordboard generally had
+a problem in it big enough to see. A broken chip has
+invisible, microscopic faults. And the faults in bad
+software can be so subtle as to be practically theological.
+
+If you want a mechanical system to do something new,
+then you must travel to where it is, and pull pieces out of it,
+and wire in new pieces. This costs money. However, if you want
+a chip to do something new, all you have to do is change its software,
+which is easy, fast and dirt-cheap. You don't even have to see the chip
+to change its program. Even if you did see the chip, it wouldn't look
+like much. A chip with program X doesn't look one whit different from
+a chip with program Y.
+
+With the proper codes and sequences, and access to specialized phone-lines,
+you can change electronic switching systems all over America from anywhere
+you please.
+
+And so can other people. If they know how, and if they want to,
+they can sneak into a microchip via the special phonelines and diddle with it,
+leaving no physical trace at all. If they broke into the operator's station
+and held Leticia at gunpoint, that would be very obvious. If they broke into
+a telco building and went after an electromechanical switch with a toolbelt,
+that would at least leave many traces. But people can do all manner of amazing
+things to computer switches just by typing on a keyboard, and keyboards are
+everywhere today. The extent of this vulnerability is deep, dark, broad,
+almost mind-boggling, and yet this is a basic, primal fact of life about
+any computer on a network.
+
+Security experts over the past twenty years have insisted,
+with growing urgency, that this basic vulnerability of computers
+represents an entirely new level of risk, of unknown but obviously
+dire potential to society. And they are right.
+
+An electronic switching station does pretty much
+everything Letitia did, except in nanoseconds and
+on a much larger scale. Compared to Miss Luthor's
+ten thousand jacks, even a primitive 1ESS switching computer,
+60s vintage, has a 128,000 lines. And the current AT&T
+system of choice is the monstrous fifth-generation 5ESS.
+
+An Electronic Switching Station can scan every line on its "board"
+in a tenth of a second, and it does this over and over, tirelessly,
+around the clock. Instead of eyes, it uses "ferrod scanners"
+to check the condition of local lines and trunks. Instead of hands,
+it has "signal distributors," "central pulse distributors,"
+"magnetic latching relays," and "reed switches," which complete
+and break the calls. Instead of a brain, it has a "central processor."
+Instead of an instruction manual, it has a program. Instead of
+a handwritten logbook for recording and billing calls,
+it has magnetic tapes. And it never has to talk to anybody.
+Everything a customer might say to it is done by punching
+the direct-dial tone buttons on your subset.
+
+Although an Electronic Switching Station can't talk,
+it does need an interface, some way to relate to its, er,
+employers. This interface is known as the "master control
+center." (This interface might be better known simply as
+"the interface," since it doesn't actually "control" phone
+calls directly. However, a term like "Master Control
+Center" is just the kind of rhetoric that telco maintenance
+engineers--and hackers--find particularly satisfying.)
+
+Using the master control center, a phone engineer can test
+local and trunk lines for malfunctions. He (rarely she)
+can check various alarm displays, measure traffic on the lines,
+examine the records of telephone usage and the charges for those calls,
+and change the programming.
+
+And, of course, anybody else who gets into the master control center
+by remote control can also do these things, if he (rarely she)
+has managed to figure them out, or, more likely, has somehow swiped
+the knowledge from people who already know.
+
+In 1989 and 1990, one particular RBOC, BellSouth,
+which felt particularly troubled, spent a purported $1.2
+million on computer security. Some think it spent as
+much as two million, if you count all the associated costs.
+Two million dollars is still very little compared to the
+great cost-saving utility of telephonic computer systems.
+
+Unfortunately, computers are also stupid.
+Unlike human beings, computers possess the truly
+profound stupidity of the inanimate.
+
+In the 1960s, in the first shocks of spreading computerization,
+there was much easy talk about the stupidity of computers--
+how they could "only follow the program" and were rigidly required
+to do "only what they were told." There has been rather less talk
+about the stupidity of computers since they began to achieve
+grandmaster status in chess tournaments, and to manifest
+many other impressive forms of apparent cleverness.
+
+Nevertheless, computers STILL are profoundly brittle and stupid;
+they are simply vastly more subtle in their stupidity and brittleness.
+The computers of the 1990s are much more reliable in their components
+than earlier computer systems, but they are also called upon to do
+far more complex things, under far more challenging conditions.
+
+On a basic mathematical level, every single line of
+a software program offers a chance for some possible screwup.
+Software does not sit still when it works; it "runs,"
+it interacts with itself and with its own inputs and outputs.
+By analogy, it stretches like putty into millions of possible
+shapes and conditions, so many shapes that they can never
+all be successfully tested, not even in the lifespan of the universe.
+Sometimes the putty snaps.
+
+The stuff we call "software" is not like anything that human society
+is used to thinking about. Software is something like a machine,
+and something like mathematics, and something like language, and
+something like thought, and art, and information. . . . But software
+is not in fact any of those other things. The protean quality
+of software is one of the great sources of its fascination.
+It also makes software very powerful, very subtle,
+very unpredictable, and very risky.
+
+Some software is bad and buggy. Some is "robust,"
+even "bulletproof." The best software is that which has
+been tested by thousands of users under thousands of
+different conditions, over years. It is then known as
+"stable." This does NOT mean that the software is
+now flawless, free of bugs. It generally means that there
+are plenty of bugs in it, but the bugs are well-identified
+and fairly well understood.
+
+There is simply no way to assure that software is free
+of flaws. Though software is mathematical in nature,
+it cannot by "proven" like a mathematical theorem;
+software is more like language, with inherent ambiguities,
+with different definitions, different assumptions,
+different levels of meaning that can conflict.
+
+Human beings can manage, more or less, with
+human language because we can catch the gist of it.
+
+Computers, despite years of effort in "artificial intelligence,"
+have proven spectacularly bad in "catching the gist" of anything at all.
+The tiniest bit of semantic grit may still bring the mightiest computer
+tumbling down. One of the most hazardous things you can do to a
+computer program is try to improve it--to try to make it safer.
+Software "patches" represent new, untried un-"stable" software,
+which is by definition riskier.
+
+The modern telephone system has come to depend,
+utterly and irretrievably, upon software. And the
+System Crash of January 15, 1990, was caused by an
+IMPROVEMENT in software. Or rather, an ATTEMPTED
+improvement.
+
+As it happened, the problem itself--the problem per se--took this form.
+A piece of telco software had been written in C language, a standard
+language of the telco field. Within the C software was a
+long "do. . .while" construct. The "do. . .while" construct
+contained a "switch" statement. The "switch" statement contained
+an "if" clause. The "if" clause contained a "break." The "break"
+was SUPPOSED to "break" the "if clause." Instead, the "break"
+broke the "switch" statement.
+
+That was the problem, the actual reason why people picking up phones
+on January 15, 1990, could not talk to one another.
+
+Or at least, that was the subtle, abstract, cyberspatial
+seed of the problem. This is how the problem manifested itself
+from the realm of programming into the realm of real life.
+
+The System 7 software for AT&T's 4ESS switching station,
+the "Generic 44E14 Central Office Switch Software,"
+had been extensively tested, and was considered very stable.
+By the end of 1989, eighty of AT&T's switching systems
+nationwide had been programmed with the new software. Cautiously,
+thirty-four stations were left to run the slower, less-capable
+System 6, because AT&T suspected there might be shakedown problems
+with the new and unprecedently sophisticated System 7 network.
+
+The stations with System 7 were programmed to switch over to a backup net
+in case of any problems. In mid-December 1989, however, a new high-velocity,
+high-security software patch was distributed to each of the 4ESS switches
+that would enable them to switch over even more quickly, making the System 7
+network that much more secure.
+
+Unfortunately, every one of these 4ESS switches was now in possession
+of a small but deadly flaw.
+
+In order to maintain the network, switches must monitor
+the condition of other switches--whether they are up and running,
+whether they have temporarily shut down, whether they are overloaded
+and in need of assistance, and so forth. The new software helped
+control this bookkeeping function by monitoring the status calls
+from other switches.
+
+It only takes four to six seconds for a troubled 4ESS switch
+to rid itself of all its calls, drop everything temporarily,
+and re-boot its software from scratch. Starting over from scratch
+will generally rid the switch of any software problems that may have
+developed in the course of running the system. Bugs that arise will
+be simply wiped out by this process. It is a clever idea. This process
+of automatically re-booting from scratch is known as the "normal fault
+recovery routine." Since AT&T's software is in fact exceptionally stable,
+systems rarely have to go into "fault recovery" in the first place;
+but AT&T has always boasted of its "real world" reliability, and this
+tactic is a belt-and-suspenders routine.
+
+The 4ESS switch used its new software to monitor its fellow switches
+as they recovered from faults. As other switches came back on line
+after recovery, they would send their "OK" signals to the switch.
+The switch would make a little note to that effect in its "status map,"
+recognizing that the fellow switch was back and ready to go,
+and should be sent some calls and put back to regular work.
+
+Unfortunately, while it was busy bookkeeping with the status map,
+the tiny flaw in the brand-new software came into play.
+The flaw caused the 4ESS switch to interact, subtly but drastically,
+with incoming telephone calls from human users. If--and only if--
+two incoming phone-calls happened to hit the switch within a hundredth
+of a second, then a small patch of data would be garbled by the flaw.
+
+But the switch had been programmed to monitor itself
+constantly for any possible damage to its data.
+When the switch perceived that its data had been somehow garbled,
+then it too would go down, for swift repairs to its software.
+It would signal its fellow switches not to send any more work.
+It would go into the fault-recovery mode for four to six seconds.
+And then the switch would be fine again, and would send out its "OK,
+ready for work" signal.
+
+However, the "OK, ready for work" signal was the VERY THING THAT
+HAD CAUSED THE SWITCH TO GO DOWN IN THE FIRST PLACE. And ALL the
+System 7 switches had the same flaw in their status-map software.
+As soon as they stopped to make the bookkeeping note that their fellow
+switch was "OK," then they too would become vulnerable to the slight
+chance that two phone-calls would hit them within a hundredth of a second.
+
+At approximately 2:25 P.M. EST on Monday, January 15,
+one of AT&T's 4ESS toll switching systems in New York City
+had an actual, legitimate, minor problem. It went into fault
+recovery routines, announced "I'm going down," then announced,
+"I'm back, I'm OK." And this cheery message then blasted
+throughout the network to many of its fellow 4ESS switches.
+
+Many of the switches, at first, completely escaped trouble.
+These lucky switches were not hit by the coincidence of
+two phone calls within a hundredth of a second.
+Their software did not fail--at first. But three switches--
+in Atlanta, St. Louis, and Detroit--were unlucky,
+and were caught with their hands full. And they went down.
+And they came back up, almost immediately. And they too began
+to broadcast the lethal message that they, too, were "OK" again,
+activating the lurking software bug in yet other switches.
+
+As more and more switches did have that bit of bad luck
+and collapsed, the call-traffic became more and more densely
+packed in the remaining switches, which were groaning
+to keep up with the load. And of course, as the calls
+became more densely packed, the switches were MUCH MORE LIKELY
+to be hit twice within a hundredth of a second.
+
+It only took four seconds for a switch to get well.
+There was no PHYSICAL damage of any kind to the switches,
+after all. Physically, they were working perfectly.
+This situation was "only" a software problem.
+
+But the 4ESS switches were leaping up and down every
+four to six seconds, in a virulent spreading wave all over America,
+in utter, manic, mechanical stupidity. They kept KNOCKING
+one another down with their contagious "OK" messages.
+
+It took about ten minutes for the chain reaction to cripple the network.
+Even then, switches would periodically luck-out and manage to resume
+their normal work. Many calls--millions of them--were managing
+to get through. But millions weren't.
+
+The switching stations that used System 6 were not directly affected.
+Thanks to these old-fashioned switches, AT&T's national system avoided
+complete collapse. This fact also made it clear to engineers that
+System 7 was at fault.
+
+Bell Labs engineers, working feverishly in New Jersey, Illinois,
+and Ohio, first tried their entire repertoire of standard network
+remedies on the malfunctioning System 7. None of the remedies worked,
+of course, because nothing like this had ever happened to any
+phone system before.
+
+By cutting out the backup safety network entirely,
+they were able to reduce the frenzy of "OK" messages
+by about half. The system then began to recover, as the
+chain reaction slowed. By 11:30 P.M. on Monday January
+15, sweating engineers on the midnight shift breathed a
+sigh of relief as the last switch cleared-up.
+
+By Tuesday they were pulling all the brand-new 4ESS software
+and replacing it with an earlier version of System 7.
+
+If these had been human operators, rather than
+computers at work, someone would simply have
+eventually stopped screaming. It would have been
+OBVIOUS that the situation was not "OK," and common
+sense would have kicked in. Humans possess common sense--
+at least to some extent. Computers simply don't.
+
+On the other hand, computers can handle hundreds
+of calls per second. Humans simply can't. If every single
+human being in America worked for the phone company,
+we couldn't match the performance of digital switches:
+direct-dialling, three-way calling, speed-calling, call-
+waiting, Caller ID, all the rest of the cornucopia
+of digital bounty. Replacing computers with operators
+is simply not an option any more.
+
+And yet we still, anachronistically, expect humans to
+be running our phone system. It is hard for us
+to understand that we have sacrificed huge amounts
+of initiative and control to senseless yet powerful machines.
+When the phones fail, we want somebody to be responsible.
+We want somebody to blame.
+
+When the Crash of January 15 happened, the American populace
+was simply not prepared to understand that enormous landslides
+in cyberspace, like the Crash itself, can happen,
+and can be nobody's fault in particular. It was easier to believe,
+maybe even in some odd way more reassuring to believe,
+that some evil person, or evil group, had done this to us.
+"Hackers" had done it. With a virus. A trojan horse.
+A software bomb. A dirty plot of some kind. People believed this,
+responsible people. In 1990, they were looking hard for evidence
+to confirm their heartfelt suspicions.
+
+And they would look in a lot of places.
+
+Come 1991, however, the outlines of an apparent new reality
+would begin to emerge from the fog.
+
+On July 1 and 2, 1991, computer-software collapses
+in telephone switching stations disrupted service in
+Washington DC, Pittsburgh, Los Angeles and San Francisco.
+Once again, seemingly minor maintenance problems had
+crippled the digital System 7. About twelve million
+people were affected in the Crash of July 1, 1991.
+
+Said the New York Times Service: "Telephone company executives
+and federal regulators said they were not ruling out the possibility
+of sabotage by computer hackers, but most seemed to think the problems
+stemmed from some unknown defect in the software running the networks."
+
+And sure enough, within the week, a red-faced software company,
+DSC Communications Corporation of Plano, Texas, owned up
+to "glitches" in the "signal transfer point" software that
+DSC had designed for Bell Atlantic and Pacific Bell.
+The immediate cause of the July 1 Crash was a single
+mistyped character: one tiny typographical flaw
+in one single line of the software. One mistyped letter,
+in one single line, had deprived the nation's capital of phone service.
+It was not particularly surprising that this tiny flaw had escaped attention:
+a typical System 7 station requires TEN MILLION lines of code.
+
+On Tuesday, September 17, 1991, came the most spectacular outage yet.
+This case had nothing to do with software failures--at least, not directly.
+Instead, a group of AT&T's switching stations in New York City had simply
+run out of electrical power and shut down cold. Their back-up batteries
+had failed. Automatic warning systems were supposed to warn of the loss
+of battery power, but those automatic systems had failed as well.
+
+This time, Kennedy, La Guardia, and Newark airports
+all had their voice and data communications cut.
+This horrifying event was particularly ironic, as attacks
+on airport computers by hackers had long been a standard
+nightmare scenario, much trumpeted by computer-security
+experts who feared the computer underground. There had even
+been a Hollywood thriller about sinister hackers ruining
+airport computers--DIE HARD II.
+
+Now AT&T itself had crippled airports with computer malfunctions--
+not just one airport, but three at once, some of the busiest in the world.
+
+Air traffic came to a standstill throughout the Greater New York area,
+causing more than 500 flights to be cancelled, in a spreading wave
+all over America and even into Europe. Another 500 or so flights
+were delayed, affecting, all in all, about 85,000 passengers.
+(One of these passengers was the chairman of the Federal
+Communications Commission.)
+
+Stranded passengers in New York and New Jersey were further
+infuriated to discover that they could not even manage to
+make a long distance phone call, to explain their delay
+to loved ones or business associates. Thanks to the crash,
+about four and a half million domestic calls, and half a million
+international calls, failed to get through.
+
+The September 17 NYC Crash, unlike the previous ones,
+involved not a whisper of "hacker" misdeeds. On the contrary,
+by 1991, AT&T itself was suffering much of the vilification
+that had formerly been directed at hackers. Congressmen were grumbling.
+So were state and federal regulators. And so was the press.
+
+For their part, ancient rival MCI took out snide full-page
+newspaper ads in New York, offering their own long-distance
+services for the "next time that AT&T goes down."
+
+"You wouldn't find a classy company like AT&T using such advertising,"
+protested AT&T Chairman Robert Allen, unconvincingly. Once again,
+out came the full-page AT&T apologies in newspapers, apologies for
+"an inexcusable culmination of both human and mechanical failure."
+(This time, however, AT&T offered no discount on later calls.
+Unkind critics suggested that AT&T were worried about setting any precedent
+for refunding the financial losses caused by telephone crashes.)
+
+Industry journals asked publicly if AT&T was "asleep at the switch."
+The telephone network, America's purported marvel of high-tech reliability,
+had gone down three times in 18 months. Fortune magazine listed the
+Crash of September 17 among the "Biggest Business Goofs of 1991,"
+cruelly parodying AT&T's ad campaign in an article entitled
+"AT&T Wants You Back (Safely On the Ground, God Willing)."
+
+Why had those New York switching systems simply run out of power?
+Because no human being had attended to the alarm system.
+Why did the alarm systems blare automatically,
+without any human being noticing? Because the three
+telco technicians who SHOULD have been listening
+were absent from their stations in the power-room,
+on another floor of the building--attending a training class.
+A training class about the alarm systems for the power room!
+
+"Crashing the System" was no longer "unprecedented" by late 1991.
+On the contrary, it no longer even seemed an oddity. By 1991,
+it was clear that all the policemen in the world could no longer
+"protect" the phone system from crashes. By far the worst crashes
+the system had ever had, had been inflicted, by the system,
+upon ITSELF. And this time nobody was making cocksure statements
+that this was an anomaly, something that would never happen again.
+By 1991 the System's defenders had met their nebulous Enemy,
+and the Enemy was--the System.
+
+
+
+PART TWO: THE DIGITAL UNDERGROUND
+
+
+The date was May 9, 1990. The Pope was touring Mexico City.
+Hustlers from the Medellin Cartel were trying to buy
+black-market Stinger missiles in Florida. On the comics page,
+Doonesbury character Andy was dying of AIDS. And then. . .a highly
+unusual item whose novelty and calculated rhetoric won it
+headscratching attention in newspapers all over America.
+
+The US Attorney's office in Phoenix, Arizona, had issued
+a press release announcing a nationwide law enforcement crackdown
+against "illegal computer hacking activities." The sweep was
+officially known as "Operation Sundevil."
+
+Eight paragraphs in the press release gave the bare facts:
+twenty-seven search warrants carried out on May 8, with three arrests,
+and a hundred and fifty agents on the prowl in "twelve" cities across America.
+(Different counts in local press reports yielded "thirteen," "fourteen," and
+"sixteen" cities.) Officials estimated that criminal losses of revenue
+to telephone companies "may run into millions of dollars." Credit for
+the Sundevil investigations was taken by the US Secret Service,
+Assistant US Attorney Tim Holtzen of Phoenix, and the Assistant
+Attorney General of Arizona, Gail Thackeray.
+
+The prepared remarks of Garry M. Jenkins, appearing in a U.S. Department
+of Justice press release, were of particular interest. Mr. Jenkins was the
+Assistant Director of the US Secret Service, and the highest-ranking federal
+official to take any direct public role in the hacker crackdown of 1990.
+
+"Today, the Secret Service is sending a clear message to those computer hackers
+who have decided to violate the laws of this nation in the mistaken belief
+that they can successfully avoid detection by hiding behind the relative
+anonymity of their computer terminals. (. . .) "Underground groups have been
+formed for the purpose of exchanging information relevant to their criminal
+activities. These groups often communicate with each other through message
+systems between computers called `bulletin boards.' "Our experience shows
+that many computer hacker suspects are no longer misguided teenagers,
+mischievously playing games with their computers in their bedrooms.
+Some are now high tech computer operators using computers to engage
+in unlawful conduct."
+
+Who were these "underground groups" and "high-tech operators?"
+Where had they come from? What did they want? Who WERE they?
+Were they "mischievous?" Were they dangerous? How had "misguided teenagers"
+managed to alarm the United States Secret Service? And just how widespread
+was this sort of thing?
+
+Of all the major players in the Hacker Crackdown: the phone companies,
+law enforcement, the civil libertarians, and the "hackers" themselves--
+the "hackers" are by far the most mysterious, by far the hardest to
+understand, by far the WEIRDEST.
+
+Not only are "hackers" novel in their activities, but they come
+in a variety of odd subcultures, with a variety of languages,
+motives and values.
+
+The earliest proto-hackers were probably those unsung mischievous
+telegraph boys who were summarily fired by the Bell Company in 1878.
+
+Legitimate "hackers," those computer enthusiasts who are independent-minded
+but law-abiding, generally trace their spiritual ancestry to elite technical
+universities, especially M.I.T. and Stanford, in the 1960s.
+
+But the genuine roots of the modern hacker UNDERGROUND can probably be traced
+most successfully to a now much-obscured hippie anarchist movement known as
+the Yippies. The Yippies, who took their name from the largely fictional
+"Youth International Party," carried out a loud and lively policy of surrealistic
+subversion and outrageous political mischief. Their basic tenets were flagrant
+sexual promiscuity, open and copious drug use, the political overthrow of any
+powermonger over thirty years of age, and an immediate end to the war
+in Vietnam, by any means necessary, including the psychic levitation
+of the Pentagon.
+
+The two most visible Yippies were Abbie Hoffman and Jerry Rubin.
+Rubin eventually became a Wall Street broker. Hoffman, ardently sought
+by federal authorities, went into hiding for seven years,
+in Mexico, France, and the United States. While on the lam,
+Hoffman continued to write and publish, with help from sympathizers
+in the American anarcho-leftist underground. Mostly, Hoffman survived
+through false ID and odd jobs. Eventually he underwent facial plastic
+surgery and adopted an entirely new identity as one "Barry Freed."
+After surrendering himself to authorities in 1980, Hoffman spent a year
+in prison on a cocaine conviction.
+
+Hoffman's worldview grew much darker as the glory days of the 1960s faded.
+In 1989, he purportedly committed suicide, under odd and, to some, rather
+suspicious circumstances.
+
+Abbie Hoffman is said to have caused the Federal Bureau of Investigation
+to amass the single largest investigation file ever opened on an individual
+American citizen. (If this is true, it is still questionable whether the
+FBI regarded Abbie Hoffman a serious public threat--quite possibly,
+his file was enormous simply because Hoffman left colorful legendry
+wherever he went). He was a gifted publicist, who regarded electronic
+media as both playground and weapon. He actively enjoyed manipulating
+network TV and other gullible, image-hungry media, with various weird lies,
+mindboggling rumors, impersonation scams, and other sinister distortions,
+all absolutely guaranteed to upset cops, Presidential candidates,
+and federal judges. Hoffman's most famous work was a book self-reflexively
+known as STEAL THIS BOOK, which publicized a number of methods by which young,
+penniless hippie agitators might live off the fat of a system supported by
+humorless drones. STEAL THIS BOOK, whose title urged readers to damage
+the very means of distribution which had put it into their hands,
+might be described as a spiritual ancestor of a computer virus.
+
+Hoffman, like many a later conspirator, made extensive use of
+pay-phones for his agitation work--in his case, generally through
+the use of cheap brass washers as coin-slugs.
+
+During the Vietnam War, there was a federal surtax imposed on telephone
+service; Hoffman and his cohorts could, and did, argue that in systematically
+stealing phone service they were engaging in civil disobedience:
+virtuously denying tax funds to an illegal and immoral war.
+
+But this thin veil of decency was soon dropped entirely.
+Ripping-off the System found its own justification in deep alienation
+and a basic outlaw contempt for conventional bourgeois values.
+Ingenious, vaguely politicized varieties of rip-off,
+which might be described as "anarchy by convenience,"
+became very popular in Yippie circles, and because rip-off
+was so useful, it was to survive the Yippie movement itself.
+
+In the early 1970s, it required fairly limited expertise
+and ingenuity to cheat payphones, to divert "free"
+electricity and gas service, or to rob vending machines
+and parking meters for handy pocket change. It also required
+a conspiracy to spread this knowledge, and the gall
+and nerve actually to commit petty theft, but the Yippies
+had these qualifications in plenty. In June 1971, Abbie
+Hoffman and a telephone enthusiast sarcastically known
+as "Al Bell" began publishing a newsletter called Youth
+International Party Line. This newsletter was dedicated
+to collating and spreading Yippie rip-off techniques,
+especially of phones, to the joy of the freewheeling
+underground and the insensate rage of all straight people.
+As a political tactic, phone-service theft ensured
+that Yippie advocates would always have ready access
+to the long-distance telephone as a medium, despite
+the Yippies' chronic lack of organization, discipline,
+money, or even a steady home address.
+
+PARTY LINE was run out of Greenwich Village for a couple of years,
+then "Al Bell" more or less defected from the faltering ranks of Yippiedom,
+changing the newsletter's name to TAP or Technical Assistance Program.
+After the Vietnam War ended, the steam began leaking rapidly out of American
+radical dissent. But by this time, "Bell" and his dozen or so
+core contributors had the bit between their teeth,
+and had begun to derive tremendous gut-level satisfaction
+from the sensation of pure TECHNICAL POWER.
+
+TAP articles, once highly politicized, became pitilessly jargonized
+and technical, in homage or parody to the Bell System's own technical
+documents, which TAP studied closely, gutted, and reproduced without
+permission. The TAP elite revelled in gloating possession
+of the specialized knowledge necessary to beat the system.
+
+"Al Bell" dropped out of the game by the late 70s,
+and "Tom Edison" took over; TAP readers (some 1400 of
+them, all told) now began to show more interest in telex
+switches and the growing phenomenon of computer systems.
+
+In 1983, "Tom Edison" had his computer stolen and his house
+set on fire by an arsonist. This was an eventually mortal blow
+to TAP (though the legendary name was to be resurrected
+in 1990 by a young Kentuckian computer-outlaw named "Predat0r.")
+
+#
+
+Ever since telephones began to make money, there have been
+people willing to rob and defraud phone companies.
+The legions of petty phone thieves vastly outnumber those
+"phone phreaks" who "explore the system" for the sake
+of the intellectual challenge. The New York metropolitan area
+(long in the vanguard of American crime) claims over 150,000
+physical attacks on pay telephones every year! Studied carefully,
+a modern payphone reveals itself as a little fortress, carefully
+designed and redesigned over generations, to resist coin-slugs,
+zaps of electricity, chunks of coin-shaped ice, prybars, magnets,
+lockpicks, blasting caps. Public pay- phones must survive in a world
+of unfriendly, greedy people, and a modern payphone is as exquisitely
+evolved as a cactus.
+Because the phone network pre-dates the computer network,
+the scofflaws known as "phone phreaks" pre-date the scofflaws
+known as "computer hackers." In practice, today, the line
+between "phreaking" and "hacking" is very blurred,
+just as the distinction between telephones and computers
+has blurred. The phone system has been digitized,
+and computers have learned to "talk" over phone-lines.
+What's worse--and this was the point of the Mr. Jenkins
+of the Secret Service--some hackers have learned to steal,
+and some thieves have learned to hack.
+
+Despite the blurring, one can still draw a few useful
+behavioral distinctions between "phreaks" and "hackers."
+Hackers are intensely interested in the "system" per se,
+and enjoy relating to machines. "Phreaks" are more
+social, manipulating the system in a rough-and-ready
+fashion in order to get through to other human beings,
+fast, cheap and under the table.
+
+Phone phreaks love nothing so much as "bridges,"
+illegal conference calls of ten or twelve chatting
+conspirators, seaboard to seaboard, lasting for many hours
+--and running, of course, on somebody else's tab,
+preferably a large corporation's.
+
+As phone-phreak conferences wear on, people drop out
+(or simply leave the phone off the hook, while they
+sashay off to work or school or babysitting),
+and new people are phoned up and invited to join in,
+from some other continent, if possible. Technical trivia,
+boasts, brags, lies, head-trip deceptions, weird rumors,
+and cruel gossip are all freely exchanged.
+
+The lowest rung of phone-phreaking is the theft of telephone access codes.
+Charging a phone call to somebody else's stolen number is, of course,
+a pig-easy way of stealing phone service, requiring practically no
+technical expertise. This practice has been very widespread,
+especially among lonely people without much money who are far from home.
+Code theft has flourished especially in college dorms, military bases,
+and, notoriously, among roadies for rock bands. Of late, code theft
+has spread very rapidly among Third Worlders in the US, who pile up
+enormous unpaid long-distance bills to the Caribbean, South America,
+and Pakistan.
+
+The simplest way to steal phone-codes is simply to look over
+a victim's shoulder as he punches-in his own code-number
+on a public payphone. This technique is known as "shoulder-surfing,"
+and is especially common in airports, bus terminals, and train stations.
+The code is then sold by the thief for a few dollars. The buyer abusing
+the code has no computer expertise, but calls his Mom in New York,
+Kingston or Caracas and runs up a huge bill with impunity. The losses
+from this primitive phreaking activity are far, far greater than the
+monetary losses caused by computer-intruding hackers.
+
+In the mid-to-late 1980s, until the introduction of sterner telco
+security measures, COMPUTERIZED code theft worked like a charm,
+and was virtually omnipresent throughout the digital underground,
+among phreaks and hackers alike. This was accomplished through
+programming one's computer to try random code numbers over the telephone
+until one of them worked. Simple programs to do this were widely available
+in the underground; a computer running all night was likely to come up with
+a dozen or so useful hits. This could be repeated week after week until
+one had a large library of stolen codes.
+
+Nowadays, the computerized dialling of hundreds of numbers
+can be detected within hours and swiftly traced.
+If a stolen code is repeatedly abused, this too can
+be detected within a few hours. But for years in the 1980s,
+the publication of stolen codes was a kind of elementary etiquette
+for fledgling hackers. The simplest way to establish your bona-fides
+as a raider was to steal a code through repeated random dialling
+and offer it to the "community" for use. Codes could be both stolen,
+and used, simply and easily from the safety of one's own bedroom,
+with very little fear of detection or punishment.
+
+Before computers and their phone-line modems entered American homes
+in gigantic numbers, phone phreaks had their own special telecommunications
+hardware gadget, the famous "blue box." This fraud device (now rendered
+increasingly useless by the digital evolution of the phone system) could
+trick switching systems into granting free access to long-distance lines.
+It did this by mimicking the system's own signal, a tone of 2600 hertz.
+
+Steven Jobs and Steve Wozniak, the founders of Apple Computer, Inc.,
+once dabbled in selling blue-boxes in college dorms in California.
+For many, in the early days of phreaking, blue-boxing was scarcely
+perceived as "theft," but rather as a fun (if sneaky) way to use
+excess phone capacity harmlessly. After all, the long-distance
+lines were JUST SITTING THERE. . . . Whom did it hurt, really?
+If you're not DAMAGING the system, and you're not USING UP ANY
+TANGIBLE RESOURCE, and if nobody FINDS OUT what you did,
+then what real harm have you done? What exactly HAVE you "stolen,"
+anyway? If a tree falls in the forest and nobody hears it,
+how much is the noise worth? Even now this remains a rather
+dicey question.
+
+Blue-boxing was no joke to the phone companies, however.
+Indeed, when Ramparts magazine, a radical publication in California,
+printed the wiring schematics necessary to create a mute box in June 1972,
+the magazine was seized by police and Pacific Bell phone-company officials.
+The mute box, a blue-box variant, allowed its user to receive long-distance
+calls free of charge to the caller. This device was closely described in a
+Ramparts article wryly titled "Regulating the Phone Company In Your Home."
+Publication of this article was held to be in violation of Californian
+State Penal Code section 502.7, which outlaws ownership of wire-fraud
+devices and the selling of "plans or instructions for any instrument,
+apparatus, or device intended to avoid telephone toll charges."
+
+Issues of Ramparts were recalled or seized on the newsstands,
+and the resultant loss of income helped put the magazine out of business.
+This was an ominous precedent for free-expression issues, but the telco's
+crushing of a radical-fringe magazine passed without serious challenge
+at the time. Even in the freewheeling California 1970s, it was widely felt
+that there was something sacrosanct about what the phone company knew;
+that the telco had a legal and moral right to protect itself by shutting
+off the flow of such illicit information. Most telco information was so
+"specialized" that it would scarcely be understood by any honest member
+of the public. If not published, it would not be missed. To print such
+material did not seem part of the legitimate role of a free press.
+
+In 1990 there would be a similar telco-inspired attack
+on the electronic phreak/hacking "magazine" Phrack.
+The Phrack legal case became a central issue in the
+Hacker Crackdown, and gave rise to great controversy.
+Phrack would also be shut down, for a time, at least,
+but this time both the telcos and their law-enforcement
+allies would pay a much larger price for their actions.
+The Phrack case will be examined in detail, later.
+
+Phone-phreaking as a social practice is still very
+much alive at this moment. Today, phone-phreaking
+is thriving much more vigorously than the better-known
+and worse-feared practice of "computer hacking."
+New forms of phreaking are spreading rapidly, following
+new vulnerabilities in sophisticated phone services.
+
+Cellular phones are especially vulnerable; their chips
+can be re-programmed to present a false caller ID
+and avoid billing. Doing so also avoids police tapping,
+making cellular-phone abuse a favorite among drug-dealers.
+"Call-sell operations" using pirate cellular phones can,
+and have, been run right out of the backs of cars, which move
+from "cell" to "cell" in the local phone system, retailing
+stolen long-distance service, like some kind of demented
+electronic version of the neighborhood ice-cream truck.
+
+Private branch-exchange phone systems in large corporations
+can be penetrated; phreaks dial-up a local company, enter its
+internal phone-system, hack it, then use the company's own
+PBX system to dial back out over the public network,
+causing the company to be stuck with the resulting
+long-distance bill. This technique is known as "diverting."
+"Diverting" can be very costly, especially because phreaks
+tend to travel in packs and never stop talking.
+Perhaps the worst by-product of this "PBX fraud"
+is that victim companies and telcos have sued one another
+over the financial responsibility for the stolen calls,
+thus enriching not only shabby phreaks but well-paid lawyers.
+
+"Voice-mail systems" can also be abused; phreaks
+can seize their own sections of these sophisticated
+electronic answering machines, and use them for trading
+codes or knowledge of illegal techniques. Voice-mail
+abuse does not hurt the company directly, but finding
+supposedly empty slots in your company's answering
+machine all crammed with phreaks eagerly chattering
+and hey-duding one another in impenetrable jargon can
+cause sensations of almost mystical repulsion and dread.
+
+Worse yet, phreaks have sometimes been known to react
+truculently to attempts to "clean up" the voice-mail system.
+Rather than humbly acquiescing to being thrown out of their playground,
+they may very well call up the company officials at work (or at home)
+and loudly demand free voice-mail addresses of their very own.
+Such bullying is taken very seriously by spooked victims.
+
+Acts of phreak revenge against straight people are rare,
+but voice-mail systems are especially tempting and vulnerable,
+and an infestation of angry phreaks in one's voice-mail system is no joke.
+They can erase legitimate messages; or spy on private messages;
+or harass users with recorded taunts and obscenities.
+They've even been known to seize control of voice-mail security,
+and lock out legitimate users, or even shut down the system entirely.
+
+Cellular phone-calls, cordless phones, and ship-to-shore
+telephony can all be monitored by various forms of radio;
+this kind of "passive monitoring" is spreading explosively today.
+Technically eavesdropping on other people's cordless and cellular
+phone-calls is the fastest-growing area in phreaking today.
+This practice strongly appeals to the lust for power and conveys
+gratifying sensations of technical superiority over the eavesdropping
+victim. Monitoring is rife with all manner of tempting evil mischief.
+Simple prurient snooping is by far the most common activity.
+But credit-card numbers unwarily spoken over the phone can be recorded,
+stolen and used. And tapping people's phone-calls (whether through
+active telephone taps or passive radio monitors) does lend itself
+conveniently to activities like blackmail, industrial espionage,
+and political dirty tricks.
+
+It should be repeated that telecommunications fraud,
+the theft of phone service, causes vastly greater monetary
+losses than the practice of entering into computers by stealth.
+Hackers are mostly young suburban American white males,
+and exist in their hundreds--but "phreaks" come from both sexes
+and from many nationalities, ages and ethnic backgrounds,
+and are flourishing in the thousands.
+
+#
+
+The term "hacker" has had an unfortunate history.
+This book, The Hacker Crackdown, has little to say about
+"hacking" in its finer, original sense. The term can signify
+the free-wheeling intellectual exploration of the highest
+and deepest potential of computer systems. Hacking can
+describe the determination to make access to computers
+and information as free and open as possible. Hacking
+can involve the heartfelt conviction that beauty can
+be found in computers, that the fine aesthetic in a perfect
+program can liberate the mind and spirit. This is "hacking"
+as it was defined in Steven Levy's much-praised history
+of the pioneer computer milieu, Hackers, published in 1984.
+
+Hackers of all kinds are absolutely soaked through with heroic
+anti-bureaucratic sentiment. Hackers long for recognition
+as a praiseworthy cultural archetype, the postmodern electronic
+equivalent of the cowboy and mountain man. Whether they deserve
+such a reputation is something for history to decide. But many hackers--
+including those outlaw hackers who are computer intruders, and whose
+activities are defined as criminal--actually attempt to LIVE UP TO
+this techno-cowboy reputation. And given that electronics and
+telecommunications are still largely unexplored territories,
+there is simply NO TELLING what hackers might uncover.
+
+For some people, this freedom is the very breath of oxygen,
+the inventive spontaneity that makes life worth living
+and that flings open doors to marvellous possibility and
+individual empowerment. But for many people
+--and increasingly so--the hacker is an ominous figure,
+a smart-aleck sociopath ready to burst out of his basement
+wilderness and savage other people's lives for his own
+anarchical convenience.
+
+Any form of power without responsibility, without direct
+and formal checks and balances, is frightening to people--
+and reasonably so. It should be frankly admitted that
+hackers ARE frightening, and that the basis of this fear
+is not irrational.
+
+Fear of hackers goes well beyond the fear of merely criminal activity.
+
+Subversion and manipulation of the phone system
+is an act with disturbing political overtones.
+In America, computers and telephones are potent symbols
+of organized authority and the technocratic business elite.
+
+But there is an element in American culture that
+has always strongly rebelled against these symbols;
+rebelled against all large industrial computers
+and all phone companies. A certain anarchical tinge deep
+in the American soul delights in causing confusion and pain
+to all bureaucracies, including technological ones.
+
+There is sometimes malice and vandalism in this attitude,
+but it is a deep and cherished part of the American national character.
+The outlaw, the rebel, the rugged individual, the pioneer,
+the sturdy Jeffersonian yeoman, the private citizen resisting
+interference in his pursuit of happiness--these are figures that all
+Americans recognize, and that many will strongly applaud and defend.
+
+Many scrupulously law-abiding citizens today do cutting-edge work
+with electronics--work that has already had tremendous social influence
+and will have much more in years to come. In all truth, these talented,
+hardworking, law-abiding, mature, adult people are far more disturbing
+to the peace and order of the current status quo than any scofflaw group
+of romantic teenage punk kids. These law-abiding hackers have the power,
+ability, and willingness to influence other people's lives quite unpredictably.
+They have means, motive, and opportunity to meddle drastically with the
+American social order. When corralled into governments, universities,
+or large multinational companies, and forced to follow rulebooks
+and wear suits and ties, they at least have some conventional halters
+on their freedom of action. But when loosed alone, or in small groups,
+and fired by imagination and the entrepreneurial spirit, they can move
+mountains--causing landslides that will likely crash directly into your
+office and living room.
+
+These people, as a class, instinctively recognize that a public,
+politicized attack on hackers will eventually spread to them--
+that the term "hacker," once demonized, might be used to knock
+their hands off the levers of power and choke them out of existence.
+There are hackers today who fiercely and publicly resist any besmirching
+of the noble title of hacker. Naturally and understandably, they deeply
+resent the attack on their values implicit in using the word "hacker"
+as a synonym for computer-criminal.
+
+This book, sadly but in my opinion unavoidably, rather adds
+to the degradation of the term. It concerns itself mostly with "hacking"
+in its commonest latter-day definition, i.e., intruding into computer
+systems by stealth and without permission. The term "hacking" is used
+routinely today by almost all law enforcement officials with any
+professional interest in computer fraud and abuse. American police
+describe almost any crime committed with, by, through, or against
+a computer as hacking.
+
+Most importantly, "hacker" is what computer-intruders
+choose to call THEMSELVES. Nobody who "hacks" into systems
+willingly describes himself (rarely, herself) as a "computer intruder,"
+"computer trespasser," "cracker," "wormer," "darkside hacker"
+or "high tech street gangster." Several other demeaning terms
+have been invented in the hope that the press and public
+will leave the original sense of the word alone. But few people
+actually use these terms. (I exempt the term "cyberpunk,"
+which a few hackers and law enforcement people actually do use.
+The term "cyberpunk" is drawn from literary criticism and has
+some odd and unlikely resonances, but, like hacker,
+cyberpunk too has become a criminal pejorative today.)
+
+In any case, breaking into computer systems was hardly alien
+to the original hacker tradition. The first tottering systems
+of the 1960s required fairly extensive internal surgery merely
+to function day-by-day. Their users "invaded" the deepest,
+most arcane recesses of their operating software almost
+as a matter of routine. "Computer security" in these early,
+primitive systems was at best an afterthought. What security
+there was, was entirely physical, for it was assumed that
+anyone allowed near this expensive, arcane hardware would be
+a fully qualified professional expert.
+
+In a campus environment, though, this meant that grad students,
+teaching assistants, undergraduates, and eventually,
+all manner of dropouts and hangers-on ended up accessing
+and often running the works.
+
+Universities, even modern universities, are not in
+the business of maintaining security over information.
+On the contrary, universities, as institutions, pre-date
+the "information economy" by many centuries and are not-
+for-profit cultural entities, whose reason for existence
+(purportedly) is to discover truth, codify it through
+techniques of scholarship, and then teach it. Universities
+are meant to PASS THE TORCH OF CIVILIZATION, not just
+download data into student skulls, and the values of the
+academic community are strongly at odds with those of all
+would-be information empires. Teachers at all levels, from
+kindergarten up, have proven to be shameless and persistent
+software and data pirates. Universities do not merely
+"leak information" but vigorously broadcast free thought.
+
+This clash of values has been fraught with controversy.
+Many hackers of the 1960s remember their professional
+apprenticeship as a long guerilla war against the uptight
+mainframe-computer "information priesthood." These computer-hungry
+youngsters had to struggle hard for access to computing power,
+and many of them were not above certain, er, shortcuts.
+But, over the years, this practice freed computing
+from the sterile reserve of lab-coated technocrats and
+was largely responsible for the explosive growth of computing
+in general society--especially PERSONAL computing.
+
+Access to technical power acted like catnip on certain
+of these youngsters. Most of the basic techniques of
+computer intrusion: password cracking, trapdoors, backdoors,
+trojan horses--were invented in college environments in the 1960s,
+in the early days of network computing. Some off-the-cuff
+experience at computer intrusion was to be in the informal
+resume of most "hackers" and many future industry giants.
+Outside of the tiny cult of computer enthusiasts, few people
+thought much about the implications of "breaking into"
+computers. This sort of activity had not yet been publicized,
+much less criminalized.
+
+In the 1960s, definitions of "property" and "privacy"
+had not yet been extended to cyberspace. Computers
+were not yet indispensable to society. There were no vast
+databanks of vulnerable, proprietary information stored
+in computers, which might be accessed, copied without
+permission, erased, altered, or sabotaged. The stakes
+were low in the early days--but they grew every year,
+exponentially, as computers themselves grew.
+
+By the 1990s, commercial and political pressures
+had become overwhelming, and they broke the social
+boundaries of the hacking subculture. Hacking
+had become too important to be left to the hackers.
+Society was now forced to tackle the intangible nature
+of cyberspace-as-property, cyberspace as privately-owned
+unreal-estate. In the new, severe, responsible, high-stakes
+context of the "Information Society" of the 1990s,
+"hacking" was called into question.
+
+What did it mean to break into a computer without
+permission and use its computational power, or look
+around inside its files without hurting anything?
+What were computer-intruding hackers, anyway--how should
+society, and the law, best define their actions?
+Were they just BROWSERS, harmless intellectual explorers?
+Were they VOYEURS, snoops, invaders of privacy? Should
+they be sternly treated as potential AGENTS OF ESPIONAGE,
+or perhaps as INDUSTRIAL SPIES? Or were they best
+defined as TRESPASSERS, a very common teenage
+misdemeanor? Was hacking THEFT OF SERVICE?
+(After all, intruders were getting someone else's
+computer to carry out their orders, without permission
+and without paying). Was hacking FRAUD? Maybe it was
+best described as IMPERSONATION. The commonest mode
+of computer intrusion was (and is) to swipe or snoop
+somebody else's password, and then enter the computer
+in the guise of another person--who is commonly stuck
+with the blame and the bills.
+
+Perhaps a medical metaphor was better--hackers should
+be defined as "sick," as COMPUTER ADDICTS unable
+to control their irresponsible, compulsive behavior.
+
+But these weighty assessments meant little to the
+people who were actually being judged. From inside
+the underground world of hacking itself, all these
+perceptions seem quaint, wrongheaded, stupid, or meaningless.
+The most important self-perception of underground hackers--
+from the 1960s, right through to the present day--is that
+they are an ELITE. The day-to-day struggle in the underground
+is not over sociological definitions--who cares?--but for power,
+knowledge, and status among one's peers.
+
+When you are a hacker, it is your own inner conviction
+of your elite status that enables you to break, or let
+us say "transcend," the rules. It is not that ALL rules
+go by the board. The rules habitually broken by hackers
+are UNIMPORTANT rules--the rules of dopey greedhead telco
+bureaucrats and pig-ignorant government pests.
+
+Hackers have their OWN rules, which separate behavior
+which is cool and elite, from behavior which is rodentlike,
+stupid and losing. These "rules," however, are mostly unwritten
+and enforced by peer pressure and tribal feeling. Like all rules
+that depend on the unspoken conviction that everybody else
+is a good old boy, these rules are ripe for abuse. The mechanisms
+of hacker peer- pressure, "teletrials" and ostracism, are rarely used
+and rarely work. Back-stabbing slander, threats, and electronic
+harassment are also freely employed in down-and-dirty intrahacker feuds,
+but this rarely forces a rival out of the scene entirely. The only real
+solution for the problem of an utterly losing, treacherous and rodentlike
+hacker is to TURN HIM IN TO THE POLICE. Unlike the Mafia or Medellin Cartel,
+the hacker elite cannot simply execute the bigmouths, creeps and troublemakers
+among their ranks, so they turn one another in with astonishing frequency.
+
+There is no tradition of silence or OMERTA in the hacker underworld.
+Hackers can be shy, even reclusive, but when they do talk, hackers
+tend to brag, boast and strut. Almost everything hackers do is INVISIBLE;
+if they don't brag, boast, and strut about it, then NOBODY WILL EVER KNOW.
+If you don't have something to brag, boast, and strut about, then nobody
+in the underground will recognize you and favor you with vital cooperation
+and respect.
+
+The way to win a solid reputation in the underground
+is by telling other hackers things that could only
+have been learned by exceptional cunning and stealth.
+Forbidden knowledge, therefore, is the basic currency
+of the digital underground, like seashells among
+Trobriand Islanders. Hackers hoard this knowledge,
+and dwell upon it obsessively, and refine it,
+and bargain with it, and talk and talk about it.
+
+Many hackers even suffer from a strange obsession to TEACH--
+to spread the ethos and the knowledge of the digital underground.
+They'll do this even when it gains them no particular advantage
+and presents a grave personal risk.
+
+And when that risk catches up with them, they will go right on teaching
+and preaching--to a new audience this time, their interrogators from law
+enforcement. Almost every hacker arrested tells everything he knows--
+all about his friends, his mentors, his disciples--legends, threats,
+horror stories, dire rumors, gossip, hallucinations. This is, of course,
+convenient for law enforcement--except when law enforcement begins
+to believe hacker legendry.
+
+Phone phreaks are unique among criminals in their willingness
+to call up law enforcement officials--in the office, at their homes--
+and give them an extended piece of their mind. It is hard not to
+interpret this as BEGGING FOR ARREST, and in fact it is an act
+of incredible foolhardiness. Police are naturally nettled
+by these acts of chutzpah and will go well out of their way
+to bust these flaunting idiots. But it can also be interpreted
+as a product of a world-view so elitist, so closed and hermetic,
+that electronic police are simply not perceived as "police,"
+but rather as ENEMY PHONE PHREAKS who should be scolded
+into behaving "decently."
+
+Hackers at their most grandiloquent perceive themselves
+as the elite pioneers of a new electronic world.
+Attempts to make them obey the democratically
+established laws of contemporary American society are
+seen as repression and persecution. After all, they argue,
+if Alexander Graham Bell had gone along with the rules
+of the Western Union telegraph company, there would have
+been no telephones. If Jobs and Wozniak had believed
+that IBM was the be-all and end-all, there would have
+been no personal computers. If Benjamin Franklin and
+Thomas Jefferson had tried to "work within the system"
+there would have been no United States.
+
+Not only do hackers privately believe this as an article of faith,
+but they have been known to write ardent manifestos about it.
+Here are some revealing excerpts from an especially vivid hacker manifesto:
+"The Techno-Revolution" by "Dr. Crash," which appeared in electronic
+form in Phrack Volume 1, Issue 6, Phile 3.
+
+
+"To fully explain the true motives behind hacking,
+we must first take a quick look into the past. In the 1960s,
+a group of MIT students built the first modern computer system.
+This wild, rebellious group of young men were the first to bear
+the name `hackers.' The systems that they developed were intended
+to be used to solve world problems and to benefit all of mankind.
+"As we can see, this has not been the case. The computer system
+has been solely in the hands of big businesses and the government.
+The wonderful device meant to enrich life has become a weapon which
+dehumanizes people. To the government and large businesses,
+people are no more than disk space, and the government doesn't
+use computers to arrange aid for the poor, but to control nuclear
+death weapons. The average American can only have access
+to a small microcomputer which is worth only a fraction
+of what they pay for it. The businesses keep the
+true state-of-the-art equipment away from the people
+behind a steel wall of incredibly high prices and bureaucracy.
+It is because of this state of affairs that hacking was born. (. . .)
+"Of course, the government doesn't want the monopoly of technology broken,
+so they have outlawed hacking and arrest anyone who is caught. (. . .)
+The phone company is another example of technology abused and kept
+from people with high prices. (. . .) "Hackers often find that their
+existing equipment, due to the monopoly tactics of computer companies,
+is inefficient for their purposes. Due to the exorbitantly high prices,
+it is impossible to legally purchase the necessary equipment.
+This need has given still another segment of the fight: Credit Carding.
+Carding is a way of obtaining the necessary goods without paying for them.
+It is again due to the companies' stupidity that Carding is so easy,
+and shows that the world's businesses are in the hands of those
+with considerably less technical know-how than we, the hackers. (. . .)
+"Hacking must continue. We must train newcomers to the art of hacking.
+(. . . .) And whatever you do, continue the fight. Whether you know it
+or not, if you are a hacker, you are a revolutionary. Don't worry,
+you're on the right side."
+
+The defense of "carding" is rare. Most hackers regard credit-card
+theft as "poison" to the underground, a sleazy and immoral effort that,
+worse yet, is hard to get away with. Nevertheless, manifestos advocating
+credit-card theft, the deliberate crashing of computer systems,
+and even acts of violent physical destruction such as vandalism
+and arson do exist in the underground. These boasts and threats
+are taken quite seriously by the police. And not every hacker
+is an abstract, Platonic computer-nerd. Some few are quite experienced
+at picking locks, robbing phone-trucks, and breaking and entering buildings.
+
+Hackers vary in their degree of hatred for authority
+and the violence of their rhetoric. But, at a bottom line,
+they are scofflaws. They don't regard the current rules
+of electronic behavior as respectable efforts to preserve
+law and order and protect public safety. They regard these
+laws as immoral efforts by soulless corporations to protect
+their profit margins and to crush dissidents. "Stupid" people,
+including police, businessmen, politicians, and journalists,
+simply have no right to judge the actions of those possessed of genius,
+techno-revolutionary intentions, and technical expertise.
+
+#
+
+Hackers are generally teenagers and college kids not
+engaged in earning a living. They often come from fairly
+well-to-do middle-class backgrounds, and are markedly
+anti-materialistic (except, that is, when it comes to
+computer equipment). Anyone motivated by greed for
+mere money (as opposed to the greed for power,
+knowledge and status) is swiftly written-off as a narrow-
+minded breadhead whose interests can only be corrupt
+and contemptible. Having grown up in the 1970s and
+1980s, the young Bohemians of the digital underground
+regard straight society as awash in plutocratic corruption,
+where everyone from the President down is for sale and
+whoever has the gold makes the rules.
+
+Interestingly, there's a funhouse-mirror image of this attitude
+on the other side of the conflict. The police are also
+one of the most markedly anti-materialistic groups
+in American society, motivated not by mere money
+but by ideals of service, justice, esprit-de-corps,
+and, of course, their own brand of specialized knowledge
+and power. Remarkably, the propaganda war between cops
+and hackers has always involved angry allegations
+that the other side is trying to make a sleazy buck.
+Hackers consistently sneer that anti-phreak prosecutors
+are angling for cushy jobs as telco lawyers and that
+computer-crime police are aiming to cash in later
+as well-paid computer-security consultants in the private sector.
+
+For their part, police publicly conflate all
+hacking crimes with robbing payphones with crowbars.
+Allegations of "monetary losses" from computer intrusion
+are notoriously inflated. The act of illicitly copying
+a document from a computer is morally equated with
+directly robbing a company of, say, half a million dollars.
+The teenage computer intruder in possession of this "proprietary"
+document has certainly not sold it for such a sum, would likely
+have little idea how to sell it at all, and quite probably
+doesn't even understand what he has. He has not made a cent
+in profit from his felony but is still morally equated with
+a thief who has robbed the church poorbox and lit out for Brazil.
+
+Police want to believe that all hackers are thieves.
+It is a tortuous and almost unbearable act for the American
+justice system to put people in jail because they want
+to learn things which are forbidden for them to know.
+In an American context, almost any pretext for punishment
+is better than jailing people to protect certain restricted
+kinds of information. Nevertheless, POLICING INFORMATION
+is part and parcel of the struggle against hackers.
+
+This dilemma is well exemplified by the remarkable
+activities of "Emmanuel Goldstein," editor and publisher
+of a print magazine known as 2600: The Hacker Quarterly.
+Goldstein was an English major at Long Island's State University
+of New York in the '70s, when he became involved with the local
+college radio station. His growing interest in electronics
+caused him to drift into Yippie TAP circles and thus into
+the digital underground, where he became a self-described
+techno-rat. His magazine publishes techniques of computer
+intrusion and telephone "exploration" as well as gloating
+exposes of telco misdeeds and governmental failings.
+
+Goldstein lives quietly and very privately in a large,
+crumbling Victorian mansion in Setauket, New York.
+The seaside house is decorated with telco decals, chunks of
+driftwood, and the basic bric-a-brac of a hippie crash-pad.
+He is unmarried, mildly unkempt, and survives mostly
+on TV dinners and turkey-stuffing eaten straight out
+of the bag. Goldstein is a man of considerable charm
+and fluency, with a brief, disarming smile and the kind
+of pitiless, stubborn, thoroughly recidivist integrity
+that America's electronic police find genuinely alarming.
+
+Goldstein took his nom-de-plume, or "handle," from
+a character in Orwell's 1984, which may be taken,
+correctly, as a symptom of the gravity of his sociopolitical
+worldview. He is not himself a practicing computer
+intruder, though he vigorously abets these actions,
+especially when they are pursued against large
+corporations or governmental agencies. Nor is he a thief,
+for he loudly scorns mere theft of phone service, in favor
+of "exploring and manipulating the system." He is probably
+best described and understood as a DISSIDENT.
+
+Weirdly, Goldstein is living in modern America
+under conditions very similar to those of former
+East European intellectual dissidents. In other words,
+he flagrantly espouses a value-system that is deeply
+and irrevocably opposed to the system of those in power
+and the police. The values in 2600 are generally expressed
+in terms that are ironic, sarcastic, paradoxical, or just
+downright confused. But there's no mistaking their
+radically anti-authoritarian tenor. 2600 holds that
+technical power and specialized knowledge, of any kind
+obtainable, belong by right in the hands of those individuals
+brave and bold enough to discover them--by whatever means necessary.
+Devices, laws, or systems that forbid access, and the free
+spread of knowledge, are provocations that any free
+and self-respecting hacker should relentlessly attack.
+The "privacy" of governments, corporations and other soulless
+technocratic organizations should never be protected
+at the expense of the liberty and free initiative
+of the individual techno-rat.
+
+However, in our contemporary workaday world, both governments
+and corporations are very anxious indeed to police information
+which is secret, proprietary, restricted, confidential,
+copyrighted, patented, hazardous, illegal, unethical,
+embarrassing, or otherwise sensitive. This makes Goldstein
+persona non grata, and his philosophy a threat.
+
+Very little about the conditions of Goldstein's daily
+life would astonish, say, Vaclav Havel. (We may note
+in passing that President Havel once had his word-processor
+confiscated by the Czechoslovak police.) Goldstein lives
+by SAMIZDAT, acting semi-openly as a data-center
+for the underground, while challenging the powers-that-be
+to abide by their own stated rules: freedom of speech
+and the First Amendment.
+
+Goldstein thoroughly looks and acts the part of techno-rat,
+with shoulder-length ringlets and a piratical black
+fisherman's-cap set at a rakish angle. He often shows up
+like Banquo's ghost at meetings of computer professionals,
+where he listens quietly, half-smiling and taking thorough notes.
+
+Computer professionals generally meet publicly,
+and find it very difficult to rid themselves of Goldstein
+and his ilk without extralegal and unconstitutional actions.
+Sympathizers, many of them quite respectable people
+with responsible jobs, admire Goldstein's attitude and
+surreptitiously pass him information. An unknown but
+presumably large proportion of Goldstein's 2,000-plus
+readership are telco security personnel and police,
+who are forced to subscribe to 2600 to stay abreast
+of new developments in hacking. They thus find themselves
+PAYING THIS GUY'S RENT while grinding their teeth in anguish,
+a situation that would have delighted Abbie Hoffman
+(one of Goldstein's few idols).
+
+Goldstein is probably the best-known public representative
+of the hacker underground today, and certainly the best-hated.
+Police regard him as a Fagin, a corrupter of youth, and speak
+of him with untempered loathing. He is quite an accomplished gadfly.
+After the Martin Luther King Day Crash of 1990, Goldstein,
+for instance, adeptly rubbed salt into the wound in the pages of 2600.
+"Yeah, it was fun for the phone phreaks as we watched the network crumble,"
+he admitted cheerfully. "But it was also an ominous sign of what's
+to come. . . . Some AT&T people, aided by well-meaning but ignorant media,
+were spreading the notion that many companies had the same software
+and therefore could face the same problem someday. Wrong. This was
+entirely an AT&T software deficiency. Of course, other companies could
+face entirely DIFFERENT software problems. But then, so too could AT&T."
+
+After a technical discussion of the system's failings,
+the Long Island techno-rat went on to offer thoughtful
+criticism to the gigantic multinational's hundreds of
+professionally qualified engineers. "What we don't know
+is how a major force in communications like AT&T could
+be so sloppy. What happened to backups? Sure,
+computer systems go down all the time, but people
+making phone calls are not the same as people logging
+on to computers. We must make that distinction. It's not
+acceptable for the phone system or any other essential
+service to `go down.' If we continue to trust technology
+without understanding it, we can look forward to many
+variations on this theme.
+
+"AT&T owes it to its customers to be prepared to INSTANTLY
+switch to another network if something strange and unpredictable
+starts occurring. The news here isn't so much the failure
+of a computer program, but the failure of AT&T's entire structure."
+
+The very idea of this. . . . this PERSON. . . . offering
+"advice" about "AT&T's entire structure" is more than
+some people can easily bear. How dare this near-criminal
+dictate what is or isn't "acceptable" behavior from AT&T?
+Especially when he's publishing, in the very same issue,
+detailed schematic diagrams for creating various switching-network
+signalling tones unavailable to the public.
+
+"See what happens when you drop a `silver box' tone or two
+down your local exchange or through different long distance
+service carriers," advises 2600 contributor "Mr. Upsetter"
+in "How To Build a Signal Box." "If you experiment systematically
+and keep good records, you will surely discover something interesting."
+
+This is, of course, the scientific method, generally regarded
+as a praiseworthy activity and one of the flowers of modern civilization.
+One can indeed learn a great deal with this sort of structured
+intellectual activity. Telco employees regard this mode of "exploration"
+as akin to flinging sticks of dynamite into their pond to see what lives
+on the bottom.
+
+2600 has been published consistently since 1984.
+It has also run a bulletin board computer system,
+printed 2600 T-shirts, taken fax calls. . . .
+The Spring 1991 issue has an interesting announcement on page 45:
+"We just discovered an extra set of wires attached to our fax line
+and heading up the pole. (They've since been clipped.)
+Your faxes to us and to anyone else could be monitored."
+In the worldview of 2600, the tiny band of techno-rat brothers
+(rarely, sisters) are a beseiged vanguard of the truly free and honest.
+The rest of the world is a maelstrom of corporate crime and high-level
+governmental corruption, occasionally tempered with well-meaning
+ignorance. To read a few issues in a row is to enter a nightmare
+akin to Solzhenitsyn's, somewhat tempered by the fact that 2600
+is often extremely funny.
+
+Goldstein did not become a target of the Hacker Crackdown,
+though he protested loudly, eloquently, and publicly about it,
+and it added considerably to his fame. It was not that he is not
+regarded as dangerous, because he is so regarded. Goldstein has had
+brushes with the law in the past: in 1985, a 2600 bulletin board
+computer was seized by the FBI, and some software on it was formally
+declared "a burglary tool in the form of a computer program."
+But Goldstein escaped direct repression in 1990, because his
+magazine is printed on paper, and recognized as subject
+to Constitutional freedom of the press protection.
+As was seen in the Ramparts case, this is far from
+an absolute guarantee. Still, as a practical matter,
+shutting down 2600 by court-order would create so much
+legal hassle that it is simply unfeasible, at least
+for the present. Throughout 1990, both Goldstein
+and his magazine were peevishly thriving.
+
+Instead, the Crackdown of 1990 would concern itself
+with the computerized version of forbidden data.
+The crackdown itself, first and foremost, was about
+BULLETIN BOARD SYSTEMS. Bulletin Board Systems, most often
+known by the ugly and un-pluralizable acronym "BBS," are
+the life-blood of the digital underground. Boards were
+also central to law enforcement's tactics and strategy
+in the Hacker Crackdown.
+
+A "bulletin board system" can be formally defined as
+a computer which serves as an information and message-
+passing center for users dialing-up over the phone-lines
+through the use of modems. A "modem," or modulator-
+demodulator, is a device which translates the digital
+impulses of computers into audible analog telephone
+signals, and vice versa. Modems connect computers
+to phones and thus to each other.
+
+Large-scale mainframe computers have been connected since the 1960s,
+but PERSONAL computers, run by individuals out of their homes,
+were first networked in the late 1970s. The "board" created
+by Ward Christensen and Randy Suess in February 1978,
+in Chicago, Illinois, is generally regarded as the first
+personal-computer bulletin board system worthy of the name.
+
+Boards run on many different machines, employing many
+different kinds of software. Early boards were crude and buggy,
+and their managers, known as "system operators" or "sysops,"
+were hard-working technical experts who wrote their own software.
+But like most everything else in the world of electronics,
+boards became faster, cheaper, better-designed, and generally
+far more sophisticated throughout the 1980s. They also moved
+swiftly out of the hands of pioneers and into those of the
+general public. By 1985 there were something in the
+neighborhood of 4,000 boards in America. By 1990 it was
+calculated, vaguely, that there were about 30,000 boards in
+the US, with uncounted thousands overseas.
+
+Computer bulletin boards are unregulated enterprises.
+Running a board is a rough-and-ready, catch-as-catch-can proposition.
+Basically, anybody with a computer, modem, software and a phone-line
+can start a board. With second-hand equipment and public-domain
+free software, the price of a board might be quite small--
+less than it would take to publish a magazine or even a
+decent pamphlet. Entrepreneurs eagerly sell bulletin-board
+software, and will coach nontechnical amateur sysops in its use.
+
+Boards are not "presses." They are not magazines,
+or libraries, or phones, or CB radios, or traditional cork
+bulletin boards down at the local laundry, though they
+have some passing resemblance to those earlier media.
+Boards are a new medium--they may even be a LARGE NUMBER of new media.
+
+Consider these unique characteristics: boards are cheap,
+yet they can have a national, even global reach.
+Boards can be contacted from anywhere in the global
+telephone network, at NO COST to the person running the board--
+the caller pays the phone bill, and if the caller is local,
+the call is free. Boards do not involve an editorial elite
+addressing a mass audience. The "sysop" of a board is not
+an exclusive publisher or writer--he is managing an electronic salon,
+where individuals can address the general public, play the part
+of the general public, and also exchange private mail
+with other individuals. And the "conversation" on boards,
+though fluid, rapid, and highly interactive, is not spoken,
+but written. It is also relatively anonymous, sometimes completely so.
+
+And because boards are cheap and ubiquitous, regulations
+and licensing requirements would likely be practically unenforceable.
+It would almost be easier to "regulate," "inspect," and "license"
+the content of private mail--probably more so, since the mail system
+is operated by the federal government. Boards are run by individuals,
+independently, entirely at their own whim.
+
+For the sysop, the cost of operation is not the primary
+limiting factor. Once the investment in a computer and
+modem has been made, the only steady cost is the charge
+for maintaining a phone line (or several phone lines).
+The primary limits for sysops are time and energy.
+Boards require upkeep. New users are generally "validated"--
+they must be issued individual passwords, and called at
+home by voice-phone, so that their identity can be
+verified. Obnoxious users, who exist in plenty, must be
+chided or purged. Proliferating messages must be deleted
+when they grow old, so that the capacity of the system
+is not overwhelmed. And software programs (if such things
+are kept on the board) must be examined for possible
+computer viruses. If there is a financial charge to use
+the board (increasingly common, especially in larger and
+fancier systems) then accounts must be kept, and users
+must be billed. And if the board crashes--a very common
+occurrence--then repairs must be made.
+
+Boards can be distinguished by the amount of effort
+spent in regulating them. First, we have the completely
+open board, whose sysop is off chugging brews and
+watching re-runs while his users generally degenerate
+over time into peevish anarchy and eventual silence.
+Second comes the supervised board, where the sysop
+breaks in every once in a while to tidy up, calm brawls,
+issue announcements, and rid the community of dolts
+and troublemakers. Third is the heavily supervised
+board, which sternly urges adult and responsible behavior
+and swiftly edits any message considered offensive,
+impertinent, illegal or irrelevant. And last comes
+the completely edited "electronic publication," which
+is presented to a silent audience which is not allowed
+to respond directly in any way.
+
+Boards can also be grouped by their degree of anonymity.
+There is the completely anonymous board, where everyone
+uses pseudonyms--"handles"--and even the sysop is unaware
+of the user's true identity. The sysop himself is likely
+pseudonymous on a board of this type. Second, and rather
+more common, is the board where the sysop knows (or thinks
+he knows) the true names and addresses of all users,
+but the users don't know one another's names and may not know his.
+Third is the board where everyone has to use real names,
+and roleplaying and pseudonymous posturing are forbidden.
+
+Boards can be grouped by their immediacy. "Chat-lines"
+are boards linking several users together over several
+different phone-lines simultaneously, so that people
+exchange messages at the very moment that they type.
+(Many large boards feature "chat" capabilities along
+with other services.) Less immediate boards,
+perhaps with a single phoneline, store messages serially,
+one at a time. And some boards are only open for business
+in daylight hours or on weekends, which greatly slows response.
+A NETWORK of boards, such as "FidoNet," can carry electronic mail
+from board to board, continent to continent, across huge distances--
+but at a relative snail's pace, so that a message can take several
+days to reach its target audience and elicit a reply.
+
+Boards can be grouped by their degree of community.
+Some boards emphasize the exchange of private,
+person-to-person electronic mail. Others emphasize
+public postings and may even purge people who "lurk,"
+merely reading posts but refusing to openly participate.
+Some boards are intimate and neighborly. Others are frosty
+and highly technical. Some are little more than storage
+dumps for software, where users "download" and "upload" programs,
+but interact among themselves little if at all.
+
+Boards can be grouped by their ease of access. Some boards
+are entirely public. Others are private and restricted only
+to personal friends of the sysop. Some boards divide users by status.
+On these boards, some users, especially beginners, strangers or children,
+will be restricted to general topics, and perhaps forbidden to post.
+Favored users, though, are granted the ability to post as they please,
+and to stay "on-line" as long as they like, even to the disadvantage
+of other people trying to call in. High-status users can be given access
+to hidden areas in the board, such as off-color topics, private discussions,
+and/or valuable software. Favored users may even become "remote sysops"
+with the power to take remote control of the board through their own
+home computers. Quite often "remote sysops" end up doing all the work
+and taking formal control of the enterprise, despite the fact that it's
+physically located in someone else's house. Sometimes several "co-sysops"
+share power.
+
+And boards can also be grouped by size. Massive, nationwide
+commercial networks, such as CompuServe, Delphi, GEnie and Prodigy,
+are run on mainframe computers and are generally not considered "boards,"
+though they share many of their characteristics, such as electronic mail,
+discussion topics, libraries of software, and persistent and growing problems
+with civil-liberties issues. Some private boards have as many as
+thirty phone-lines and quite sophisticated hardware. And then
+there are tiny boards.
+
+Boards vary in popularity. Some boards are huge and crowded,
+where users must claw their way in against a constant busy-signal.
+Others are huge and empty--there are few things sadder than a formerly
+flourishing board where no one posts any longer, and the dead conversations
+of vanished users lie about gathering digital dust. Some boards are tiny
+and intimate, their telephone numbers intentionally kept confidential
+so that only a small number can log on.
+
+And some boards are UNDERGROUND.
+
+Boards can be mysterious entities. The activities of
+their users can be hard to differentiate from conspiracy.
+Sometimes they ARE conspiracies. Boards have harbored,
+or have been accused of harboring, all manner of fringe groups,
+and have abetted, or been accused of abetting, every manner
+of frowned-upon, sleazy, radical, and criminal activity.
+There are Satanist boards. Nazi boards. Pornographic boards.
+Pedophile boards. Drug- dealing boards. Anarchist boards.
+Communist boards. Gay and Lesbian boards (these exist in great profusion,
+many of them quite lively with well-established histories).
+Religious cult boards. Evangelical boards. Witchcraft
+boards, hippie boards, punk boards, skateboarder boards.
+Boards for UFO believers. There may well be boards for
+serial killers, airline terrorists and professional assassins.
+There is simply no way to tell. Boards spring up, flourish,
+and disappear in large numbers, in most every corner of
+the developed world. Even apparently innocuous public
+boards can, and sometimes do, harbor secret areas known
+only to a few. And even on the vast, public, commercial services,
+private mail is very private--and quite possibly criminal.
+
+Boards cover most every topic imaginable and some
+that are hard to imagine. They cover a vast spectrum
+of social activity. However, all board users do have
+something in common: their possession of computers
+and phones. Naturally, computers and phones are
+primary topics of conversation on almost every board.
+
+And hackers and phone phreaks, those utter devotees
+of computers and phones, live by boards. They swarm by boards.
+They are bred by boards. By the late 1980s, phone-phreak groups
+and hacker groups, united by boards, had proliferated fantastically.
+
+
+As evidence, here is a list of hacker groups compiled
+by the editors of Phrack on August 8, 1988.
+
+
+The Administration.
+Advanced Telecommunications, Inc.
+ALIAS.
+American Tone Travelers.
+Anarchy Inc.
+Apple Mafia.
+The Association.
+Atlantic Pirates Guild.
+
+Bad Ass Mother Fuckers.
+Bellcore.
+Bell Shock Force.
+Black Bag.
+
+Camorra.
+C&M Productions.
+Catholics Anonymous.
+Chaos Computer Club.
+Chief Executive Officers.
+Circle Of Death.
+Circle Of Deneb.
+Club X.
+Coalition of Hi-Tech
+Pirates.
+Coast-To-Coast.
+Corrupt Computing.
+Cult Of The
+Dead Cow.
+Custom Retaliations.
+
+Damage Inc.
+D&B Communications.
+The Danger Gang.
+Dec Hunters.
+Digital Gang.
+DPAK.
+
+Eastern Alliance.
+The Elite Hackers Guild.
+Elite Phreakers and Hackers Club.
+The Elite Society Of America.
+EPG.
+Executives Of Crime.
+Extasyy Elite.
+
+Fargo 4A.
+Farmers Of Doom.
+The Federation.
+Feds R Us.
+First Class.
+Five O.
+Five Star.
+Force Hackers.
+The 414s.
+
+Hack-A-Trip.
+Hackers Of America.
+High Mountain Hackers.
+High Society.
+The Hitchhikers.
+
+IBM Syndicate.
+The Ice Pirates.
+Imperial Warlords.
+Inner Circle.
+Inner Circle II.
+Insanity Inc.
+International Computer Underground Bandits.
+
+Justice League of America.
+
+Kaos Inc.
+Knights Of Shadow.
+Knights Of The Round Table.
+
+League Of Adepts.
+Legion Of Doom.
+Legion Of Hackers.
+Lords Of Chaos.
+Lunatic Labs, Unlimited.
+
+Master Hackers.
+MAD!
+The Marauders.
+MD/PhD.
+
+Metal Communications, Inc.
+MetalliBashers, Inc.
+MBI.
+
+Metro Communications.
+Midwest Pirates Guild.
+
+NASA Elite.
+The NATO Association.
+Neon Knights.
+
+Nihilist Order.
+Order Of The Rose.
+OSS.
+
+Pacific Pirates Guild.
+Phantom Access Associates.
+
+PHido PHreaks.
+The Phirm.
+Phlash.
+PhoneLine Phantoms.
+Phone Phreakers Of America.
+Phortune 500.
+
+Phreak Hack Delinquents.
+Phreak Hack Destroyers.
+
+Phreakers, Hackers, And Laundromat Employees Gang (PHALSE Gang).
+Phreaks Against Geeks.
+Phreaks Against Phreaks Against Geeks.
+Phreaks and Hackers of America.
+Phreaks Anonymous World Wide.
+Project Genesis.
+The Punk Mafia.
+
+The Racketeers.
+Red Dawn Text Files.
+Roscoe Gang.
+
+
+SABRE.
+Secret Circle of Pirates.
+Secret Service.
+707 Club.
+Shadow Brotherhood.
+Sharp Inc.
+65C02 Elite.
+
+Spectral Force.
+Star League.
+Stowaways.
+Strata-Crackers.
+
+
+Team Hackers '86.
+Team Hackers '87.
+
+TeleComputist Newsletter Staff.
+Tribunal Of Knowledge.
+
+Triple Entente.
+Turn Over And Die Syndrome (TOADS).
+
+300 Club.
+1200 Club.
+2300 Club.
+2600 Club.
+2601 Club.
+
+2AF.
+
+The United Soft WareZ Force.
+United Technical Underground.
+
+Ware Brigade.
+The Warelords.
+WASP.
+
+Contemplating this list is an impressive, almost humbling business.
+As a cultural artifact, the thing approaches poetry.
+
+Underground groups--subcultures--can be distinguished
+from independent cultures by their habit of referring
+constantly to the parent society. Undergrounds by their
+nature constantly must maintain a membrane of differentiation.
+Funny/distinctive clothes and hair, specialized jargon, specialized
+ghettoized areas in cities, different hours of rising, working,
+sleeping. . . . The digital underground, which specializes in information,
+relies very heavily on language to distinguish itself. As can be seen
+from this list, they make heavy use of parody and mockery.
+It's revealing to see who they choose to mock.
+
+First, large corporations. We have the Phortune 500,
+The Chief Executive Officers, Bellcore, IBM Syndicate,
+SABRE (a computerized reservation service maintained
+by airlines). The common use of "Inc." is telling--
+none of these groups are actual corporations,
+but take clear delight in mimicking them.
+
+Second, governments and police. NASA Elite, NATO Association.
+"Feds R Us" and "Secret Service" are fine bits of fleering boldness.
+OSS--the Office of Strategic Services was the forerunner of the CIA.
+
+Third, criminals. Using stigmatizing pejoratives as a perverse
+badge of honor is a time-honored tactic for subcultures:
+punks, gangs, delinquents, mafias, pirates, bandits, racketeers.
+
+Specialized orthography, especially the use of "ph" for "f"
+and "z" for the plural "s," are instant recognition symbols.
+So is the use of the numeral "0" for the letter "O"
+--computer-software orthography generally features a
+slash through the zero, making the distinction obvious.
+
+Some terms are poetically descriptive of computer intrusion:
+the Stowaways, the Hitchhikers, the PhoneLine Phantoms, Coast-to-Coast.
+Others are simple bravado and vainglorious puffery.
+(Note the insistent use of the terms "elite" and "master.")
+Some terms are blasphemous, some obscene, others merely cryptic--
+anything to puzzle, offend, confuse, and keep the straights at bay.
+
+Many hacker groups further re-encrypt their names
+by the use of acronyms: United Technical Underground
+becomes UTU, Farmers of Doom become FoD, the United SoftWareZ
+Force becomes, at its own insistence, "TuSwF," and woe to the
+ignorant rodent who capitalizes the wrong letters.
+
+It should be further recognized that the members of these groups
+are themselves pseudonymous. If you did, in fact, run across
+the "PhoneLine Phantoms," you would find them to consist of
+"Carrier Culprit," "The Executioner," "Black Majik,"
+"Egyptian Lover," "Solid State," and "Mr Icom."
+"Carrier Culprit" will likely be referred to by his friends
+as "CC," as in, "I got these dialups from CC of PLP."
+
+It's quite possible that this entire list refers to as
+few as a thousand people. It is not a complete list
+of underground groups--there has never been such a list,
+and there never will be. Groups rise, flourish, decline,
+share membership, maintain a cloud of wannabes and
+casual hangers-on. People pass in and out, are ostracized,
+get bored, are busted by police, or are cornered by telco
+security and presented with huge bills. Many "underground
+groups" are software pirates, "warez d00dz," who might break
+copy protection and pirate programs, but likely wouldn't dare
+to intrude on a computer-system.
+
+It is hard to estimate the true population of the digital
+underground. There is constant turnover. Most hackers
+start young, come and go, then drop out at age 22--
+the age of college graduation. And a large majority
+of "hackers" access pirate boards, adopt a handle,
+swipe software and perhaps abuse a phone-code or two,
+while never actually joining the elite.
+
+Some professional informants, who make it their business
+to retail knowledge of the underground to paymasters in private
+corporate security, have estimated the hacker population
+at as high as fifty thousand. This is likely highly inflated,
+unless one counts every single teenage software pirate
+and petty phone-booth thief. My best guess is about 5,000 people.
+Of these, I would guess that as few as a hundred are truly "elite"
+--active computer intruders, skilled enough to penetrate
+sophisticated systems and truly to worry corporate security
+and law enforcement.
+
+Another interesting speculation is whether this group
+is growing or not. Young teenage hackers are often
+convinced that hackers exist in vast swarms and will soon
+dominate the cybernetic universe. Older and wiser
+veterans, perhaps as wizened as 24 or 25 years old,
+are convinced that the glory days are long gone, that the cops
+have the underground's number now, and that kids these days
+are dirt-stupid and just want to play Nintendo.
+
+My own assessment is that computer intrusion, as a non-profit act
+of intellectual exploration and mastery, is in slow decline,
+at least in the United States; but that electronic fraud,
+especially telecommunication crime, is growing by leaps and bounds.
+
+One might find a useful parallel to the digital underground
+in the drug underground. There was a time, now much-obscured
+by historical revisionism, when Bohemians freely shared joints
+at concerts, and hip, small-scale marijuana dealers might
+turn people on just for the sake of enjoying a long stoned conversation
+about the Doors and Allen Ginsberg. Now drugs are increasingly verboten,
+except in a high-stakes, highly-criminal world of highly addictive drugs.
+Over years of disenchantment and police harassment, a vaguely ideological,
+free-wheeling drug underground has relinquished the business of drug-dealing
+to a far more savage criminal hard-core. This is not a pleasant prospect
+to contemplate, but the analogy is fairly compelling.
+
+What does an underground board look like? What distinguishes
+it from a standard board? It isn't necessarily the conversation--
+hackers often talk about common board topics, such as hardware, software,
+sex, science fiction, current events, politics, movies, personal gossip.
+Underground boards can best be distinguished by their files, or "philes,"
+pre-composed texts which teach the techniques and ethos of the underground.
+These are prized reservoirs of forbidden knowledge. Some are anonymous,
+but most proudly bear the handle of the "hacker" who has created them,
+and his group affiliation, if he has one.
+
+Here is a partial table-of-contents of philes from an underground board,
+somewhere in the heart of middle America, circa 1991. The descriptions
+are mostly self-explanatory.
+
+
+BANKAMER.ZIP 5406 06-11-91 Hacking Bank America
+CHHACK.ZIP 4481 06-11-91 Chilton Hacking
+CITIBANK.ZIP 4118 06-11-91 Hacking Citibank
+CREDIMTC.ZIP 3241 06-11-91 Hacking Mtc Credit Company
+DIGEST.ZIP 5159 06-11-91 Hackers Digest
+HACK.ZIP 14031 06-11-91 How To Hack
+HACKBAS.ZIP 5073 06-11-91 Basics Of Hacking
+HACKDICT.ZIP 42774 06-11-91 Hackers Dictionary
+HACKER.ZIP 57938 06-11-91 Hacker Info
+HACKERME.ZIP 3148 06-11-91 Hackers Manual
+HACKHAND.ZIP 4814 06-11-91 Hackers Handbook
+HACKTHES.ZIP 48290 06-11-91 Hackers Thesis
+HACKVMS.ZIP 4696 06-11-91 Hacking Vms Systems
+MCDON.ZIP 3830 06-11-91 Hacking Macdonalds (Home Of The Archs)
+P500UNIX.ZIP 15525 06-11-91 Phortune 500 Guide To Unix
+RADHACK.ZIP 8411 06-11-91 Radio Hacking
+TAOTRASH.DOC 4096 12-25-89 Suggestions For Trashing
+TECHHACK.ZIP 5063 06-11-91 Technical Hacking
+
+
+The files above are do-it-yourself manuals about computer intrusion.
+The above is only a small section of a much larger library of hacking
+and phreaking techniques and history. We now move into a different
+and perhaps surprising area.
+
++------------+
+ |Anarchy|
++------------+
+
+ANARC.ZIP 3641 06-11-91 Anarchy Files
+ANARCHST.ZIP 63703 06-11-91 Anarchist Book
+ANARCHY.ZIP 2076 06-11-91 Anarchy At Home
+ANARCHY3.ZIP 6982 06-11-91 Anarchy No 3
+ANARCTOY.ZIP 2361 06-11-91 Anarchy Toys
+ANTIMODM.ZIP 2877 06-11-91 Anti-modem Weapons
+ATOM.ZIP 4494 06-11-91 How To Make An Atom Bomb
+BARBITUA.ZIP 3982 06-11-91 Barbiturate Formula
+BLCKPWDR.ZIP 2810 06-11-91 Black Powder Formulas
+BOMB.ZIP 3765 06-11-91 How To Make Bombs
+BOOM.ZIP 2036 06-11-91 Things That Go Boom
+CHLORINE.ZIP 1926 06-11-91 Chlorine Bomb
+COOKBOOK.ZIP 1500 06-11-91 Anarchy Cook Book
+DESTROY.ZIP 3947 06-11-91 Destroy Stuff
+DUSTBOMB.ZIP 2576 06-11-91 Dust Bomb
+ELECTERR.ZIP 3230 06-11-91 Electronic Terror
+EXPLOS1.ZIP 2598 06-11-91 Explosives 1
+EXPLOSIV.ZIP 18051 06-11-91 More Explosives
+EZSTEAL.ZIP 4521 06-11-91 Ez-stealing
+FLAME.ZIP 2240 06-11-91 Flame Thrower
+FLASHLT.ZIP 2533 06-11-91 Flashlight Bomb
+FMBUG.ZIP 2906 06-11-91 How To Make An Fm Bug
+OMEEXPL.ZIP 2139 06-11-91 Home Explosives
+HOW2BRK.ZIP 3332 06-11-91 How To Break In
+LETTER.ZIP 2990 06-11-91 Letter Bomb
+LOCK.ZIP 2199 06-11-91 How To Pick Locks
+MRSHIN.ZIP 3991 06-11-91 Briefcase Locks
+NAPALM.ZIP 3563 06-11-91 Napalm At Home
+NITRO.ZIP 3158 06-11-91 Fun With Nitro
+PARAMIL.ZIP 2962 06-11-91 Paramilitary Info
+PICKING.ZIP 3398 06-11-91 Picking Locks
+PIPEBOMB.ZIP 2137 06-11-91 Pipe Bomb
+POTASS.ZIP 3987 06-11-91 Formulas With Potassium
+PRANK.TXT 11074 08-03-90 More Pranks To Pull On Idiots!
+REVENGE.ZIP 4447 06-11-91 Revenge Tactics
+ROCKET.ZIP 2590 06-11-91 Rockets For Fun
+SMUGGLE.ZIP 3385 06-11-91 How To Smuggle
+
+HOLY COW! The damned thing is full of stuff about bombs!
+
+What are we to make of this?
+
+First, it should be acknowledged that spreading
+knowledge about demolitions to teenagers is a highly and
+deliberately antisocial act. It is not, however, illegal.
+
+Second, it should be recognized that most of these
+philes were in fact WRITTEN by teenagers. Most adult
+American males who can remember their teenage years
+will recognize that the notion of building a flamethrower
+in your garage is an incredibly neat-o idea. ACTUALLY,
+building a flamethrower in your garage, however, is
+fraught with discouraging difficulty. Stuffing gunpowder
+into a booby-trapped flashlight, so as to blow the arm off
+your high-school vice-principal, can be a thing of dark
+beauty to contemplate. Actually committing assault by
+explosives will earn you the sustained attention of the
+federal Bureau of Alcohol, Tobacco and Firearms.
+
+Some people, however, will actually try these plans.
+A determinedly murderous American teenager can probably
+buy or steal a handgun far more easily than he can brew
+fake "napalm" in the kitchen sink. Nevertheless,
+if temptation is spread before people, a certain number
+will succumb, and a small minority will actually attempt
+these stunts. A large minority of that small minority
+will either fail or, quite likely, maim themselves,
+since these "philes" have not been checked for accuracy,
+are not the product of professional experience,
+and are often highly fanciful. But the gloating menace
+of these philes is not to be entirely dismissed.
+
+Hackers may not be "serious" about bombing; if they were,
+we would hear far more about exploding flashlights, homemade bazookas,
+and gym teachers poisoned by chlorine and potassium.
+However, hackers are VERY serious about forbidden knowledge.
+They are possessed not merely by curiosity, but by
+a positive LUST TO KNOW. The desire to know what
+others don't is scarcely new. But the INTENSITY
+of this desire, as manifested by these young technophilic
+denizens of the Information Age, may in fact BE new,
+and may represent some basic shift in social values--
+a harbinger of what the world may come to, as society
+lays more and more value on the possession,
+assimilation and retailing of INFORMATION
+as a basic commodity of daily life.
+
+There have always been young men with obsessive interests
+in these topics. Never before, however, have they been able
+to network so extensively and easily, and to propagandize
+their interests with impunity to random passers-by.
+High-school teachers will recognize that there's always
+one in a crowd, but when the one in a crowd escapes control
+by jumping into the phone-lines, and becomes a hundred such kids
+all together on a board, then trouble is brewing visibly.
+The urge of authority to DO SOMETHING, even something drastic,
+is hard to resist. And in 1990, authority did something.
+In fact authority did a great deal.
+
+#
+
+The process by which boards create hackers goes something
+like this. A youngster becomes interested in computers--
+usually, computer games. He hears from friends that
+"bulletin boards" exist where games can be obtained for free.
+(Many computer games are "freeware," not copyrighted--
+invented simply for the love of it and given away to the public;
+some of these games are quite good.) He bugs his parents for a modem,
+or quite often, uses his parents' modem.
+
+The world of boards suddenly opens up. Computer games
+can be quite expensive, real budget-breakers for a kid,
+but pirated games, stripped of copy protection, are cheap or free.
+They are also illegal, but it is very rare, almost unheard of,
+for a small-scale software pirate to be prosecuted.
+Once "cracked" of its copy protection, the program,
+being digital data, becomes infinitely reproducible.
+Even the instructions to the game, any manuals that accompany it,
+can be reproduced as text files, or photocopied from legitimate sets.
+Other users on boards can give many useful hints in game-playing tactics.
+And a youngster with an infinite supply of free computer games can
+certainly cut quite a swath among his modem-less friends.
+
+And boards are pseudonymous. No one need know that you're
+fourteen years old--with a little practice at subterfuge,
+you can talk to adults about adult things, and be accepted
+and taken seriously! You can even pretend to be a girl,
+or an old man, or anybody you can imagine. If you find this
+kind of deception gratifying, there is ample opportunity
+to hone your ability on boards.
+
+But local boards can grow stale. And almost every board maintains
+a list of phone-numbers to other boards, some in distant, tempting,
+exotic locales. Who knows what they're up to, in Oregon or Alaska
+or Florida or California? It's very easy to find out--just order
+the modem to call through its software--nothing to this, just typing
+on a keyboard, the same thing you would do for most any computer game.
+The machine reacts swiftly and in a few seconds you are talking to
+a bunch of interesting people on another seaboard.
+
+And yet the BILLS for this trivial action can be staggering!
+Just by going tippety-tap with your fingers, you may have
+saddled your parents with four hundred bucks in long-distance charges,
+and gotten chewed out but good. That hardly seems fair.
+
+How horrifying to have made friends in another state
+and to be deprived of their company--and their software--
+just because telephone companies demand absurd amounts of money!
+How painful, to be restricted to boards in one's own AREA CODE--
+what the heck is an "area code" anyway, and what makes it so special?
+A few grumbles, complaints, and innocent questions of this sort
+will often elicit a sympathetic reply from another board user--
+someone with some stolen codes to hand. You dither a while,
+knowing this isn't quite right, then you make up your mind
+to try them anyhow--AND THEY WORK! Suddenly you're doing something
+even your parents can't do. Six months ago you were just some kid--now,
+you're the Crimson Flash of Area Code 512! You're bad--you're nationwide!
+
+Maybe you'll stop at a few abused codes. Maybe you'll decide that
+boards aren't all that interesting after all, that it's wrong,
+not worth the risk --but maybe you won't. The next step
+is to pick up your own repeat-dialling program--
+to learn to generate your own stolen codes.
+(This was dead easy five years ago, much harder
+to get away with nowadays, but not yet impossible.)
+And these dialling programs are not complex or intimidating--
+some are as small as twenty lines of software.
+
+Now, you too can share codes. You can trade codes to learn
+other techniques. If you're smart enough to catch on,
+and obsessive enough to want to bother, and ruthless enough
+to start seriously bending rules, then you'll get better, fast.
+You start to develop a rep. You move up to a heavier class
+of board--a board with a bad attitude, the kind of board
+that naive dopes like your classmates and your former self
+have never even heard of! You pick up the jargon of phreaking
+and hacking from the board. You read a few of those anarchy philes--
+and man, you never realized you could be a real OUTLAW without
+ever leaving your bedroom.
+
+You still play other computer games, but now you have a new
+and bigger game. This one will bring you a different kind of status
+than destroying even eight zillion lousy space invaders.
+
+Hacking is perceived by hackers as a "game." This is
+not an entirely unreasonable or sociopathic perception.
+You can win or lose at hacking, succeed or fail,
+but it never feels "real." It's not simply that
+imaginative youngsters sometimes have a hard time
+telling "make-believe" from "real life." Cyberspace
+is NOT REAL! "Real" things are physical objects
+like trees and shoes and cars. Hacking takes place
+on a screen. Words aren't physical, numbers
+(even telephone numbers and credit card numbers)
+aren't physical. Sticks and stones may break my bones,
+but data will never hurt me. Computers SIMULATE reality,
+like computer games that simulate tank battles or dogfights
+or spaceships. Simulations are just make-believe,
+and the stuff in computers is NOT REAL.
+
+Consider this: if "hacking" is supposed to be so serious and
+real-life and dangerous, then how come NINE-YEAR-OLD KIDS have
+computers and modems? You wouldn't give a nine year old his own car,
+or his own rifle, or his own chainsaw--those things are "real."
+
+People underground are perfectly aware that the "game"
+is frowned upon by the powers that be. Word gets around
+about busts in the underground. Publicizing busts is one
+of the primary functions of pirate boards, but they also
+promulgate an attitude about them, and their own idiosyncratic
+ideas of justice. The users of underground boards won't complain
+if some guy is busted for crashing systems, spreading viruses,
+or stealing money by wire-fraud. They may shake their heads
+with a sneaky grin, but they won't openly defend these practices.
+But when a kid is charged with some theoretical amount of theft:
+$233,846.14, for instance, because he sneaked into a computer
+and copied something, and kept it in his house on a floppy disk--
+this is regarded as a sign of near-insanity from prosecutors,
+a sign that they've drastically mistaken the immaterial game
+of computing for their real and boring everyday world
+of fatcat corporate money.
+
+It's as if big companies and their suck-up lawyers
+think that computing belongs to them, and they can
+retail it with price stickers, as if it were boxes
+of laundry soap! But pricing "information" is like
+trying to price air or price dreams. Well, anybody
+on a pirate board knows that computing can be,
+and ought to be, FREE. Pirate boards are little
+independent worlds in cyberspace, and they don't belong
+to anybody but the underground. Underground boards
+aren't "brought to you by Procter & Gamble."
+
+To log on to an underground board can mean to
+experience liberation, to enter a world where,
+for once, money isn't everything and adults
+don't have all the answers.
+
+Let's sample another vivid hacker manifesto. Here are
+some excerpts from "The Conscience of a Hacker," by "The Mentor,"
+from Phrack Volume One, Issue 7, Phile 3.
+
+"I made a discovery today. I found a computer.
+Wait a second, this is cool. It does what I want it to.
+If it makes a mistake, it's because I screwed it up.
+Not because it doesn't like me. (. . .)
+"And then it happened. . .a door opened to a world. . .
+rushing through the phone line like heroin through an
+addict's veins, an electronic pulse is sent out,
+a refuge from day-to-day incompetencies is sought. . .
+a board is found. `This is it. . .this is where I belong. . .'
+"I know everyone here. . .even if I've never met them,
+never talked to them, may never hear from them again. . .
+I know you all. . . (. . .)
+
+"This is our world now. . .the world of the electron
+and the switch, the beauty of the baud. We make use of a
+service already existing without paying for what could be
+dirt-cheap if it wasn't run by profiteering gluttons, and you
+call us criminals. We explore. . .and you call us criminals.
+We seek after knowledge. . .and you call us criminals.
+We exist without skin color, without nationality,
+without religious bias. . .and you call us criminals.
+You build atomic bombs, you wage wars, you murder,
+cheat and lie to us and try to make us believe that
+it's for our own good, yet we're the criminals.
+
+"Yes, I am a criminal. My crime is that of curiosity.
+My crime is that of judging people by what they say and think,
+not what they look like. My crime is that of outsmarting you,
+something that you will never forgive me for."
+
+#
+
+There have been underground boards almost as long
+as there have been boards. One of the first was 8BBS,
+which became a stronghold of the West Coast phone-phreak elite.
+After going on-line in March 1980, 8BBS sponsored "Susan Thunder,"
+and "Tuc," and, most notoriously, "the Condor." "The Condor"
+bore the singular distinction of becoming the most vilified
+American phreak and hacker ever. Angry underground associates,
+fed up with Condor's peevish behavior, turned him in to police,
+along with a heaping double-helping of outrageous hacker legendry.
+As a result, Condor was kept in solitary confinement for seven months,
+for fear that he might start World War Three by triggering missile silos
+from the prison payphone. (Having served his time, Condor is now
+walking around loose; WWIII has thus far conspicuously failed to occur.)
+
+The sysop of 8BBS was an ardent free-speech enthusiast
+who simply felt that ANY attempt to restrict the expression
+of his users was unconstitutional and immoral.
+Swarms of the technically curious entered 8BBS
+and emerged as phreaks and hackers, until, in 1982,
+a friendly 8BBS alumnus passed the sysop a new modem
+which had been purchased by credit-card fraud.
+Police took this opportunity to seize the entire board
+and remove what they considered an attractive nuisance.
+
+Plovernet was a powerful East Coast pirate board
+that operated in both New York and Florida.
+Owned and operated by teenage hacker "Quasi Moto,"
+Plovernet attracted five hundred eager users in 1983.
+"Emmanuel Goldstein" was one-time co-sysop of Plovernet,
+along with "Lex Luthor," founder of the "Legion of Doom" group.
+Plovernet bore the signal honor of being the original home
+of the "Legion of Doom," about which the reader will be hearing
+a great deal, soon.
+
+"Pirate-80," or "P-80," run by a sysop known as "Scan-Man,"
+got into the game very early in Charleston, and continued
+steadily for years. P-80 flourished so flagrantly that
+even its most hardened users became nervous, and some
+slanderously speculated that "Scan Man" must have ties
+to corporate security, a charge he vigorously denied.
+
+"414 Private" was the home board for the first GROUP
+to attract conspicuous trouble, the teenage "414 Gang,"
+whose intrusions into Sloan-Kettering Cancer Center and
+Los Alamos military computers were to be a nine-days-wonder in 1982.
+
+At about this time, the first software piracy boards
+began to open up, trading cracked games for the Atari 800
+and the Commodore C64. Naturally these boards were
+heavily frequented by teenagers. And with the 1983
+release of the hacker-thriller movie War Games,
+the scene exploded. It seemed that every kid
+in America had demanded and gotten a modem for Christmas.
+Most of these dabbler wannabes put their modems in the attic
+after a few weeks, and most of the remainder minded their
+P's and Q's and stayed well out of hot water. But some
+stubborn and talented diehards had this hacker kid in
+War Games figured for a happening dude. They simply
+could not rest until they had contacted the underground--
+or, failing that, created their own.
+
+In the mid-80s, underground boards sprang up like digital fungi.
+ShadowSpawn Elite. Sherwood Forest I, II, and III.
+Digital Logic Data Service in Florida, sysoped by no less
+a man than "Digital Logic" himself; Lex Luthor of the
+Legion of Doom was prominent on this board, since it
+was in his area code. Lex's own board, "Legion of Doom,"
+started in 1984. The Neon Knights ran a network of Apple-
+hacker boards: Neon Knights North, South, East and West.
+Free World II was run by "Major Havoc." Lunatic Labs
+is still in operation as of this writing. Dr. Ripco
+in Chicago, an anything-goes anarchist board with an
+extensive and raucous history, was seized by Secret Service
+agents in 1990 on Sundevil day, but up again almost immediately,
+with new machines and scarcely diminished vigor.
+
+The St. Louis scene was not to rank with major centers
+of American hacking such as New York and L.A. But St.
+Louis did rejoice in possession of "Knight Lightning"
+and "Taran King," two of the foremost JOURNALISTS native
+to the underground. Missouri boards like Metal Shop,
+Metal Shop Private, Metal Shop Brewery, may not have
+been the heaviest boards around in terms of illicit
+expertise. But they became boards where hackers could
+exchange social gossip and try to figure out what the
+heck was going on nationally--and internationally.
+Gossip from Metal Shop was put into the form of news files,
+then assembled into a general electronic publication,
+Phrack, a portmanteau title coined from "phreak" and "hack."
+The Phrack editors were as obsessively curious about other
+hackers as hackers were about machines.
+
+Phrack, being free of charge and lively reading, began
+to circulate throughout the underground. As Taran King
+and Knight Lightning left high school for college,
+Phrack began to appear on mainframe machines linked to BITNET,
+and, through BITNET to the "Internet," that loose but
+extremely potent not-for-profit network where academic,
+governmental and corporate machines trade data through
+the UNIX TCP/IP protocol. (The "Internet Worm" of
+November 2-3,1988, created by Cornell grad student Robert Morris,
+was to be the largest and best-publicized computer-intrusion scandal
+to date. Morris claimed that his ingenious "worm" program was meant
+to harmlessly explore the Internet, but due to bad programming,
+the Worm replicated out of control and crashed some six thousand
+Internet computers. Smaller-scale and less ambitious Internet hacking
+was a standard for the underground elite.)
+
+Most any underground board not hopelessly lame and out-of-it
+would feature a complete run of Phrack--and, possibly,
+the lesser-known standards of the underground:
+the Legion of Doom Technical Journal, the obscene
+and raucous Cult of the Dead Cow files, P/HUN magazine,
+Pirate, the Syndicate Reports, and perhaps the highly
+anarcho-political Activist Times Incorporated.
+
+Possession of Phrack on one's board was prima facie
+evidence of a bad attitude. Phrack was seemingly everywhere,
+aiding, abetting, and spreading the underground ethos.
+And this did not escape the attention of corporate security
+or the police.
+
+We now come to the touchy subject of police and boards.
+Police, do, in fact, own boards. In 1989, there were
+police-sponsored boards in California, Colorado, Florida,
+Georgia, Idaho, Michigan, Missouri, Texas, and Virginia:
+boards such as "Crime Bytes," "Crimestoppers," "All Points"
+and "Bullet-N-Board." Police officers, as private computer
+enthusiasts, ran their own boards in Arizona, California,
+Colorado, Connecticut, Florida, Missouri, Maryland,
+New Mexico, North Carolina, Ohio, Tennessee and Texas.
+Police boards have often proved helpful in community relations.
+Sometimes crimes are reported on police boards.
+
+Sometimes crimes are COMMITTED on police boards.
+This has sometimes happened by accident, as naive hackers
+blunder onto police boards and blithely begin offering telephone codes.
+Far more often, however, it occurs through the now almost-traditional
+use of "sting boards." The first police sting-boards were established
+in 1985: "Underground Tunnel" in Austin, Texas, whose sysop
+Sgt. Robert Ansley called himself "Pluto"--"The Phone Company"
+in Phoenix, Arizona, run by Ken MacLeod of the Maricopa County
+Sheriff's office--and Sgt. Dan Pasquale's board in Fremont, California.
+Sysops posed as hackers, and swiftly garnered coteries of ardent users,
+who posted codes and loaded pirate software with abandon,
+and came to a sticky end.
+
+Sting boards, like other boards, are cheap to operate,
+very cheap by the standards of undercover police operations.
+Once accepted by the local underground, sysops will likely be
+invited into other pirate boards, where they can compile more dossiers.
+And when the sting is announced and the worst offenders arrested,
+the publicity is generally gratifying. The resultant paranoia
+in the underground--perhaps more justly described as a "deterrence effect"--
+tends to quell local lawbreaking for quite a while.
+
+Obviously police do not have to beat the underbrush for hackers.
+On the contrary, they can go trolling for them. Those caught
+can be grilled. Some become useful informants. They can lead
+the way to pirate boards all across the country.
+
+And boards all across the country showed the sticky
+fingerprints of Phrack, and of that loudest and most
+flagrant of all underground groups, the "Legion of Doom."
+
+The term "Legion of Doom" came from comic books. The Legion of Doom,
+a conspiracy of costumed super- villains headed by the chrome-domed
+criminal ultra- mastermind Lex Luthor, gave Superman a lot of four-color
+graphic trouble for a number of decades. Of course, Superman,
+that exemplar of Truth, Justice, and the American Way,
+always won in the long run. This didn't matter to the hacker Doomsters--
+"Legion of Doom" was not some thunderous and evil Satanic reference,
+it was not meant to be taken seriously. "Legion of Doom" came
+from funny-books and was supposed to be funny.
+
+"Legion of Doom" did have a good mouthfilling ring to it, though.
+It sounded really cool. Other groups, such as the "Farmers of Doom,"
+closely allied to LoD, recognized this grandiloquent quality,
+and made fun of it. There was even a hacker group called
+"Justice League of America," named after Superman's club
+of true-blue crimefighting superheros.
+
+But they didn't last; the Legion did.
+
+The original Legion of Doom, hanging out on Quasi Moto's Plovernet board,
+were phone phreaks. They weren't much into computers. "Lex Luthor" himself
+(who was under eighteen when he formed the Legion) was a COSMOS expert,
+COSMOS being the "Central System for Mainframe Operations,"
+a telco internal computer network. Lex would eventually become
+quite a dab hand at breaking into IBM mainframes, but although
+everyone liked Lex and admired his attitude, he was not considered
+a truly accomplished computer intruder. Nor was he the "mastermind"
+of the Legion of Doom--LoD were never big on formal leadership.
+As a regular on Plovernet and sysop of his "Legion of Doom BBS,"
+Lex was the Legion's cheerleader and recruiting officer.
+
+Legion of Doom began on the ruins of an earlier phreak group,
+The Knights of Shadow. Later, LoD was to subsume the personnel
+of the hacker group "Tribunal of Knowledge." People came and went
+constantly in LoD; groups split up or formed offshoots.
+
+Early on, the LoD phreaks befriended a few computer-intrusion
+enthusiasts, who became the associated "Legion of Hackers."
+Then the two groups conflated into the "Legion of Doom/Hackers,"
+or LoD/H. When the original "hacker" wing, Messrs. "Compu-Phreak"
+and "Phucked Agent 04," found other matters to occupy their time,
+the extra "/H" slowly atrophied out of the name; but by this time
+the phreak wing, Messrs. Lex Luthor, "Blue Archer," "Gary Seven,"
+"Kerrang Khan," "Master of Impact," "Silver Spy," "The Marauder,"
+and "The Videosmith," had picked up a plethora of intrusion
+expertise and had become a force to be reckoned with.
+
+LoD members seemed to have an instinctive understanding
+that the way to real power in the underground lay through
+covert publicity. LoD were flagrant. Not only was it one
+of the earliest groups, but the members took pains to widely
+distribute their illicit knowledge. Some LoD members,
+like "The Mentor," were close to evangelical about it.
+Legion of Doom Technical Journal began to show up on boards
+throughout the underground.
+
+LoD Technical Journal was named in cruel parody
+of the ancient and honored AT&T Technical Journal.
+The material in these two publications was quite similar--
+much of it, adopted from public journals and discussions
+in the telco community. And yet, the predatory attitude
+of LoD made even its most innocuous data seem deeply sinister;
+an outrage; a clear and present danger.
+
+To see why this should be, let's consider the following
+(invented) paragraphs, as a kind of thought experiment.
+
+(A) "W. Fred Brown, AT&T Vice President for
+Advanced Technical Development, testified May 8
+at a Washington hearing of the National Telecommunications
+and Information Administration (NTIA), regarding
+Bellcore's GARDEN project. GARDEN (Generalized
+Automatic Remote Distributed Electronic Network) is a
+telephone-switch programming tool that makes it possible
+to develop new telecom services, including hold-on-hold
+and customized message transfers, from any keypad terminal,
+within seconds. The GARDEN prototype combines centrex
+lines with a minicomputer using UNIX operating system software."
+
+(B) "Crimson Flash 512 of the Centrex Mobsters reports:
+D00dz, you wouldn't believe this GARDEN bullshit Bellcore's
+just come up with! Now you don't even need a lousy Commodore
+to reprogram a switch--just log on to GARDEN as a technician,
+and you can reprogram switches right off the keypad in any
+public phone booth! You can give yourself hold-on-hold
+and customized message transfers, and best of all,
+the thing is run off (notoriously insecure) centrex lines
+using--get this--standard UNIX software! Ha ha ha ha!"
+
+Message (A), couched in typical techno-bureaucratese,
+appears tedious and almost unreadable. (A) scarcely seems
+threatening or menacing. Message (B), on the other hand,
+is a dreadful thing, prima facie evidence of a dire conspiracy,
+definitely not the kind of thing you want your teenager reading.
+
+The INFORMATION, however, is identical. It is PUBLIC
+information, presented before the federal government in
+an open hearing. It is not "secret." It is not "proprietary."
+It is not even "confidential." On the contrary, the
+development of advanced software systems is a matter
+of great public pride to Bellcore.
+
+However, when Bellcore publicly announces a project of this kind,
+it expects a certain attitude from the public--something along
+the lines of GOSH WOW, YOU GUYS ARE GREAT, KEEP THAT UP, WHATEVER IT IS--
+certainly not cruel mimickry, one-upmanship and outrageous speculations
+about possible security holes.
+
+Now put yourself in the place of a policeman confronted by
+an outraged parent, or telco official, with a copy of Version (B).
+This well-meaning citizen, to his horror, has discovered
+a local bulletin-board carrying outrageous stuff like (B),
+which his son is examining with a deep and unhealthy interest.
+If (B) were printed in a book or magazine, you, as an American
+law enforcement officer, would know that it would take
+a hell of a lot of trouble to do anything about it;
+but it doesn't take technical genius to recognize that
+if there's a computer in your area harboring stuff like (B),
+there's going to be trouble.
+
+In fact, if you ask around, any computer-literate cop
+will tell you straight out that boards with stuff like (B)
+are the SOURCE of trouble. And the WORST source of trouble
+on boards are the ringleaders inventing and spreading stuff like (B).
+If it weren't for these jokers, there wouldn't BE any trouble.
+
+And Legion of Doom were on boards like nobody else.
+Plovernet. The Legion of Doom Board. The Farmers of Doom Board.
+Metal Shop. OSUNY. Blottoland. Private Sector. Atlantis.
+Digital Logic. Hell Phrozen Over.
+
+LoD members also ran their own boards. "Silver Spy" started
+his own board, "Catch-22," considered one of the heaviest around.
+So did "Mentor," with his "Phoenix Project." When they didn't run boards
+themselves, they showed up on other people's boards, to brag, boast,
+and strut. And where they themselves didn't go, their philes went,
+carrying evil knowledge and an even more evil attitude.
+
+As early as 1986, the police were under the vague impression
+that EVERYONE in the underground was Legion of Doom.
+LoD was never that large--considerably smaller than either
+"Metal Communications" or "The Administration," for instance--
+but LoD got tremendous press. Especially in Phrack,
+which at times read like an LoD fan magazine; and Phrack
+was everywhere, especially in the offices of telco security.
+You couldn't GET busted as a phone phreak, a hacker,
+or even a lousy codes kid or warez dood, without the cops
+asking if you were LoD.
+
+This was a difficult charge to deny, as LoD never
+distributed membership badges or laminated ID cards.
+If they had, they would likely have died out quickly,
+for turnover in their membership was considerable.
+LoD was less a high-tech street-gang than an ongoing
+state-of-mind. LoD was the Gang That Refused to Die.
+By 1990, LoD had RULED for ten years, and it seemed WEIRD
+to police that they were continually busting people who were
+only sixteen years old. All these teenage small-timers
+were pleading the tiresome hacker litany of "just curious,
+no criminal intent." Somewhere at the center of this
+conspiracy there had to be some serious adult masterminds,
+not this seemingly endless supply of myopic suburban
+white kids with high SATs and funny haircuts.
+
+There was no question that most any American hacker
+arrested would "know" LoD. They knew the handles
+of contributors to LoD Tech Journal, and were likely
+to have learned their craft through LoD boards and LoD activism.
+But they'd never met anyone from LoD. Even some of the
+rotating cadre who were actually and formally "in LoD"
+knew one another only by board-mail and pseudonyms.
+This was a highly unconventional profile for a criminal conspiracy.
+Computer networking, and the rapid evolution of the digital underground,
+made the situation very diffuse and confusing.
+
+Furthermore, a big reputation in the digital underground
+did not coincide with one's willingness to commit "crimes."
+Instead, reputation was based on cleverness and technical mastery.
+As a result, it often seemed that the HEAVIER the hackers were,
+the LESS likely they were to have committed any kind of common,
+easily prosecutable crime. There were some hackers who could really steal.
+And there were hackers who could really hack. But the two groups didn't seem
+to overlap much, if at all. For instance, most people in the underground
+looked up to "Emmanuel Goldstein" of 2600 as a hacker demigod.
+But Goldstein's publishing activities were entirely legal--
+Goldstein just printed dodgy stuff and talked about politics,
+he didn't even hack. When you came right down to it,
+Goldstein spent half his time complaining that computer security
+WASN'T STRONG ENOUGH and ought to be drastically improved
+across the board!
+
+Truly heavy-duty hackers, those with serious technical skills
+who had earned the respect of the underground, never stole money
+or abused credit cards. Sometimes they might abuse phone-codes--
+but often, they seemed to get all the free phone-time they wanted
+without leaving a trace of any kind.
+
+The best hackers, the most powerful and technically accomplished,
+were not professional fraudsters. They raided computers habitually,
+but wouldn't alter anything, or damage anything. They didn't even steal
+computer equipment--most had day-jobs messing with hardware,
+and could get all the cheap secondhand equipment they wanted.
+The hottest hackers, unlike the teenage wannabes, weren't snobs
+about fancy or expensive hardware. Their machines tended to be
+raw second-hand digital hot-rods full of custom add-ons that
+they'd cobbled together out of chickenwire, memory chips and spit.
+Some were adults, computer software writers and consultants by trade,
+and making quite good livings at it. Some of them ACTUALLY WORKED
+FOR THE PHONE COMPANY--and for those, the "hackers" actually found
+under the skirts of Ma Bell, there would be little mercy in 1990.
+
+It has long been an article of faith in the
+underground that the "best" hackers never get caught.
+They're far too smart, supposedly. They never get caught
+because they never boast, brag, or strut. These demigods
+may read underground boards (with a condescending smile),
+but they never say anything there. The "best" hackers,
+according to legend, are adult computer professionals,
+such as mainframe system administrators, who already know
+the ins and outs of their particular brand of security.
+Even the "best" hacker can't break in to just any computer at random:
+the knowledge of security holes is too specialized, varying widely
+with different software and hardware. But if people are employed to run,
+say, a UNIX mainframe or a VAX/VMS machine, then they tend to learn
+security from the inside out. Armed with this knowledge,
+they can look into most anybody else's UNIX or VMS
+without much trouble or risk, if they want to.
+And, according to hacker legend, of course they want to,
+so of course they do. They just don't make a big deal
+of what they've done. So nobody ever finds out.
+
+It is also an article of faith in the underground that
+professional telco people "phreak" like crazed weasels.
+OF COURSE they spy on Madonna's phone calls--I mean,
+WOULDN'T YOU? Of course they give themselves free long-
+distance--why the hell should THEY pay, they're running
+the whole shebang!
+
+It has, as a third matter, long been an article of faith
+that any hacker caught can escape serious punishment if
+he confesses HOW HE DID IT. Hackers seem to believe
+that governmental agencies and large corporations are
+blundering about in cyberspace like eyeless jellyfish
+or cave salamanders. They feel that these large
+but pathetically stupid organizations will proffer up
+genuine gratitude, and perhaps even a security post
+and a big salary, to the hot-shot intruder who will deign
+to reveal to them the supreme genius of his modus operandi.
+
+In the case of longtime LoD member "Control-C,"
+this actually happened, more or less. Control-C had led
+Michigan Bell a merry chase, and when captured in 1987,
+he turned out to be a bright and apparently physically
+harmless young fanatic, fascinated by phones. There was
+no chance in hell that Control-C would actually repay the
+enormous and largely theoretical sums in long-distance
+service that he had accumulated from Michigan Bell.
+He could always be indicted for fraud or computer-intrusion,
+but there seemed little real point in this--he hadn't
+physically damaged any computer. He'd just plead guilty,
+and he'd likely get the usual slap-on-the-wrist,
+and in the meantime it would be a big hassle for Michigan Bell
+just to bring up the case. But if kept on the payroll,
+he might at least keep his fellow hackers at bay.
+
+There were uses for him. For instance, a contrite
+Control-C was featured on Michigan Bell internal posters,
+sternly warning employees to shred their trash.
+He'd always gotten most of his best inside info from
+"trashing"--raiding telco dumpsters, for useful data
+indiscreetly thrown away. He signed these posters, too.
+Control-C had become something like a Michigan Bell mascot.
+And in fact, Control-C DID keep other hackers at bay.
+Little hackers were quite scared of Control-C and his
+heavy-duty Legion of Doom friends. And big hackers WERE
+his friends and didn't want to screw up his cushy situation.
+
+No matter what one might say of LoD, they did stick together.
+When "Wasp," an apparently genuinely malicious New York hacker,
+began crashing Bellcore machines, Control-C received swift volunteer
+help from "the Mentor" and the Georgia LoD wing made up of
+"The Prophet," "Urvile," and "Leftist." Using Mentor's Phoenix
+Project board to coordinate, the Doomsters helped telco security
+to trap Wasp, by luring him into a machine with a tap
+and line-trace installed. Wasp lost. LoD won! And my, did they brag.
+
+Urvile, Prophet and Leftist were well-qualified for this activity,
+probably more so even than the quite accomplished Control-C.
+The Georgia boys knew all about phone switching-stations.
+Though relative johnny-come-latelies in the Legion of Doom,
+they were considered some of LoD's heaviest guys,
+into the hairiest systems around. They had the good fortune
+to live in or near Atlanta, home of the sleepy and apparently
+tolerant BellSouth RBOC.
+
+As RBOC security went, BellSouth were "cake." US West (of Arizona,
+the Rockies and the Pacific Northwest) were tough and aggressive,
+probably the heaviest RBOC around. Pacific Bell, California's PacBell,
+were sleek, high-tech, and longtime veterans of the LA phone-phreak wars.
+NYNEX had the misfortune to run the New York City area, and were warily
+prepared for most anything. Even Michigan Bell, a division of the
+Ameritech RBOC, at least had the elementary sense to hire their own hacker
+as a useful scarecrow. But BellSouth, even though their corporate P.R.
+proclaimed them to have "Everything You Expect From a Leader," were pathetic.
+
+When rumor about LoD's mastery of Georgia's switching network got around
+to BellSouth through Bellcore and telco security scuttlebutt,
+they at first refused to believe it. If you paid serious attention
+to every rumor out and about these hacker kids, you would hear all kinds
+of wacko saucer-nut nonsense: that the National Security Agency
+monitored all American phone calls, that the CIA and DEA tracked
+traffic on bulletin-boards with word-analysis programs,
+that the Condor could start World War III from a payphone.
+
+If there were hackers into BellSouth switching-stations, then how come
+nothing had happened? Nothing had been hurt. BellSouth's machines
+weren't crashing. BellSouth wasn't suffering especially badly from fraud.
+BellSouth's customers weren't complaining. BellSouth was headquartered
+in Atlanta, ambitious metropolis of the new high-tech Sunbelt;
+and BellSouth was upgrading its network by leaps and bounds,
+digitizing the works left right and center. They could hardly be
+considered sluggish or naive. BellSouth's technical expertise
+was second to none, thank you kindly. But then came the Florida business.
+
+On June 13, 1989, callers to the Palm Beach County Probation Department,
+in Delray Beach, Florida, found themselves involved in a remarkable
+discussion with a phone-sex worker named "Tina" in New York State.
+Somehow, ANY call to this probation office near Miami was instantly
+and magically transported across state lines, at no extra charge to the user,
+to a pornographic phone-sex hotline hundreds of miles away!
+
+This practical joke may seem utterly hilarious at first hearing,
+and indeed there was a good deal of chuckling about it in
+phone phreak circles, including the Autumn 1989 issue of 2600.
+But for Southern Bell (the division of the BellSouth RBOC
+supplying local service for Florida, Georgia, North Carolina
+and South Carolina), this was a smoking gun. For the first time ever,
+a computer intruder had broken into a BellSouth central office
+switching station and re-programmed it!
+
+Or so BellSouth thought in June 1989. Actually, LoD members had been
+frolicking harmlessly in BellSouth switches since September 1987.
+The stunt of June 13--call-forwarding a number through manipulation
+of a switching station--was child's play for hackers as accomplished
+as the Georgia wing of LoD. Switching calls interstate sounded like
+a big deal, but it took only four lines of code to accomplish this.
+An easy, yet more discreet, stunt, would be to call-forward another
+number to your own house. If you were careful and considerate,
+and changed the software back later, then not a soul would know.
+Except you. And whoever you had bragged to about it.
+
+As for BellSouth, what they didn't know wouldn't hurt them.
+
+Except now somebody had blown the whole thing wide open, and BellSouth knew.
+
+A now alerted and considerably paranoid BellSouth began searching switches
+right and left for signs of impropriety, in that hot summer of 1989.
+No fewer than forty-two BellSouth employees were put on 12-hour shifts,
+twenty-four hours a day, for two solid months, poring over records
+and monitoring computers for any sign of phony access. These forty-two
+overworked experts were known as BellSouth's "Intrusion Task Force."
+
+What the investigators found astounded them. Proprietary telco databases
+had been manipulated: phone numbers had been created out of thin air,
+with no users' names and no addresses. And perhaps worst of all,
+no charges and no records of use. The new digital ReMOB (Remote Observation)
+diagnostic feature had been extensively tampered with--hackers had learned to
+reprogram ReMOB software, so that they could listen in on any switch-routed
+call at their leisure! They were using telco property to SPY!
+
+The electrifying news went out throughout law enforcement in 1989.
+It had never really occurred to anyone at BellSouth that their prized
+and brand-new digital switching-stations could be RE-PROGRAMMED.
+People seemed utterly amazed that anyone could have the nerve.
+Of course these switching stations were "computers," and everybody
+knew hackers liked to "break into computers:" but telephone people's
+computers were DIFFERENT from normal people's computers.
+
+The exact reason WHY these computers were "different" was
+rather ill-defined. It certainly wasn't the extent of their security.
+The security on these BellSouth computers was lousy; the AIMSX computers,
+for instance, didn't even have passwords. But there was no question that
+BellSouth strongly FELT that their computers were very different indeed.
+And if there were some criminals out there who had not gotten that message,
+BellSouth was determined to see that message taught.
+
+After all, a 5ESS switching station was no mere bookkeeping system for
+some local chain of florists. Public service depended on these stations.
+Public SAFETY depended on these stations.
+
+And hackers, lurking in there call-forwarding or ReMobbing, could spy
+on anybody in the local area! They could spy on telco officials!
+They could spy on police stations! They could spy on local offices
+of the Secret Service. . . .
+
+In 1989, electronic cops and hacker-trackers began using scrambler-phones
+and secured lines. It only made sense. There was no telling who was into
+those systems. Whoever they were, they sounded scary. This was some
+new level of antisocial daring. Could be West German hackers, in the pay
+of the KGB. That too had seemed a weird and farfetched notion,
+until Clifford Stoll had poked and prodded a sluggish Washington
+law-enforcement bureaucracy into investigating a computer intrusion
+that turned out to be exactly that--HACKERS, IN THE PAY OF THE KGB!
+Stoll, the systems manager for an Internet lab in Berkeley California,
+had ended up on the front page of the New Nork Times, proclaimed a national
+hero in the first true story of international computer espionage.
+Stoll's counterspy efforts, which he related in a bestselling book,
+The Cuckoo's Egg, in 1989, had established the credibility of `hacking'
+as a possible threat to national security. The United States Secret Service
+doesn't mess around when it suspects a possible action by a foreign
+intelligence apparat.
+
+The Secret Service scrambler-phones and secured lines put
+a tremendous kink in law enforcement's ability to operate freely;
+to get the word out, cooperate, prevent misunderstandings.
+Nevertheless, 1989 scarcely seemed the time for half-measures.
+If the police and Secret Service themselves were not operationally secure,
+then how could they reasonably demand measures of security from
+private enterprise? At least, the inconvenience made people aware
+of the seriousness of the threat.
+
+If there was a final spur needed to get the police off the dime,
+it came in the realization that the emergency 911 system was vulnerable.
+The 911 system has its own specialized software, but it is run on the same
+digital switching systems as the rest of the telephone network.
+911 is not physically different from normal telephony. But it is
+certainly culturally different, because this is the area of
+telephonic cyberspace reserved for the police and emergency services.
+
+Your average policeman may not know much about hackers or phone-phreaks.
+Computer people are weird; even computer COPS are rather weird;
+the stuff they do is hard to figure out. But a threat to the 911 system
+is anything but an abstract threat. If the 911 system goes, people can die.
+
+Imagine being in a car-wreck, staggering to a phone-booth,
+punching 911 and hearing "Tina" pick up the phone-sex line
+somewhere in New York! The situation's no longer comical, somehow.
+
+And was it possible? No question. Hackers had attacked 911
+systems before. Phreaks can max-out 911 systems just by siccing
+a bunch of computer-modems on them in tandem, dialling them over
+and over until they clog. That's very crude and low-tech,
+but it's still a serious business.
+
+The time had come for action. It was time to take stern measures
+with the underground. It was time to start picking up the dropped threads,
+the loose edges, the bits of braggadocio here and there; it was time to get
+on the stick and start putting serious casework together. Hackers weren't
+"invisible." They THOUGHT they were invisible; but the truth was,
+they had just been tolerated too long.
+
+Under sustained police attention in the summer of '89, the digital
+underground began to unravel as never before.
+
+The first big break in the case came very early on: July 1989,
+the following month. The perpetrator of the "Tina" switch was caught,
+and confessed. His name was "Fry Guy," a 16-year-old in Indiana.
+Fry Guy had been a very wicked young man.
+
+Fry Guy had earned his handle from a stunt involving French fries.
+Fry Guy had filched the log-in of a local MacDonald's manager
+and had logged-on to the MacDonald's mainframe on the Sprint
+Telenet system. Posing as the manager, Fry Guy had altered
+MacDonald's records, and given some teenage hamburger-flipping
+friends of his, generous raises. He had not been caught.
+
+Emboldened by success, Fry Guy moved on to credit-card abuse.
+Fry Guy was quite an accomplished talker; with a gift for
+"social engineering." If you can do "social engineering"
+--fast-talk, fake-outs, impersonation, conning, scamming--
+then card abuse comes easy. (Getting away with it in
+the long run is another question).
+
+Fry Guy had run across "Urvile" of the Legion of Doom
+on the ALTOS Chat board in Bonn, Germany. ALTOS Chat
+was a sophisticated board, accessible through globe-spanning
+computer networks like BITnet, Tymnet, and Telenet.
+ALTOS was much frequented by members of Germany's
+Chaos Computer Club. Two Chaos hackers who hung out on ALTOS,
+"Jaeger" and "Pengo," had been the central villains of
+Clifford Stoll's Cuckoo's Egg case: consorting in East Berlin
+with a spymaster from the KGB, and breaking into American
+computers for hire, through the Internet.
+
+When LoD members learned the story of Jaeger's depredations
+from Stoll's book, they were rather less than impressed,
+technically speaking. On LoD's own favorite board of the moment,
+"Black Ice," LoD members bragged that they themselves could have done
+all the Chaos break-ins in a week flat! Nevertheless, LoD were grudgingly
+impressed by the Chaos rep, the sheer hairy-eyed daring of hash-smoking
+anarchist hackers who had rubbed shoulders with the fearsome big-boys
+of international Communist espionage. LoD members sometimes traded
+bits of knowledge with friendly German hackers on ALTOS--phone numbers
+for vulnerable VAX/VMS computers in Georgia, for instance.
+Dutch and British phone phreaks, and the Australian clique of
+"Phoenix," "Nom," and "Electron," were ALTOS regulars, too.
+In underground circles, to hang out on ALTOS was considered
+the sign of an elite dude, a sophisticated hacker of the
+international digital jet-set.
+
+Fry Guy quickly learned how to raid information from credit-card
+consumer-reporting agencies. He had over a hundred stolen credit-card
+numbers in his notebooks, and upwards of a thousand swiped long-distance
+access codes. He knew how to get onto Altos, and how to talk the talk of
+the underground convincingly. He now wheedled knowledge of switching-station
+tricks from Urvile on the ALTOS system.
+
+Combining these two forms of knowledge enabled Fry Guy to bootstrap
+his way up to a new form of wire-fraud. First, he'd snitched credit card
+numbers from credit-company computers. The data he copied included names,
+addresses and phone numbers of the random card-holders.
+
+Then Fry Guy, impersonating a card-holder, called up Western Union
+and asked for a cash advance on "his" credit card. Western Union,
+as a security guarantee, would call the customer back, at home,
+to verify the transaction.
+
+But, just as he had switched the Florida probation office to "Tina"
+in New York, Fry Guy switched the card-holder's number to a local pay-phone.
+There he would lurk in wait, muddying his trail by routing and re-routing
+the call, through switches as far away as Canada. When the call came through,
+he would boldly "social-engineer," or con, the Western Union people, pretending
+to be the legitimate card-holder. Since he'd answered the proper phone number,
+the deception was not very hard. Western Union's money was then shipped to
+a confederate of Fry Guy's in his home town in Indiana.
+
+Fry Guy and his cohort, using LoD techniques, stole six thousand dollars
+from Western Union between December 1988 and July 1989. They also dabbled
+in ordering delivery of stolen goods through card-fraud. Fry Guy
+was intoxicated with success. The sixteen-year-old fantasized wildly
+to hacker rivals, boasting that he'd used rip-off money to hire himself
+a big limousine, and had driven out-of-state with a groupie from
+his favorite heavy-metal band, Motley Crue.
+
+Armed with knowledge, power, and a gratifying stream of free money,
+Fry Guy now took it upon himself to call local representatives
+of Indiana Bell security, to brag, boast, strut, and utter
+tormenting warnings that his powerful friends in the notorious
+Legion of Doom could crash the national telephone network.
+Fry Guy even named a date for the scheme: the Fourth of July,
+a national holiday.
+
+This egregious example of the begging-for-arrest syndrome was shortly
+followed by Fry Guy's arrest. After the Indiana telephone company figured
+out who he was, the Secret Service had DNRs--Dialed Number Recorders--
+installed on his home phone lines. These devices are not taps, and can't
+record the substance of phone calls, but they do record the phone numbers
+of all calls going in and out. Tracing these numbers showed Fry Guy's
+long-distance code fraud, his extensive ties to pirate bulletin boards,
+and numerous personal calls to his LoD friends in Atlanta. By July 11,
+1989, Prophet, Urvile and Leftist also had Secret Service DNR
+"pen registers" installed on their own lines.
+
+The Secret Service showed up in force at Fry Guy's house on July 22, 1989,
+to the horror of his unsuspecting parents. The raiders were led by
+a special agent from the Secret Service's Indianapolis office.
+However, the raiders were accompanied and advised by Timothy M. Foley
+of the Secret Service's Chicago office (a gentleman about whom
+we will soon be hearing a great deal).
+
+Following federal computer-crime techniques that had been standard
+since the early 1980s, the Secret Service searched the house thoroughly,
+and seized all of Fry Guy's electronic equipment and notebooks.
+All Fry Guy's equipment went out the door in the custody of the
+Secret Service, which put a swift end to his depredations.
+
+The USSS interrogated Fry Guy at length. His case was put in the charge
+of Deborah Daniels, the federal US Attorney for the Southern District
+of Indiana. Fry Guy was charged with eleven counts of computer fraud,
+unauthorized computer access, and wire fraud. The evidence was thorough
+and irrefutable. For his part, Fry Guy blamed his corruption on the
+Legion of Doom and offered to testify against them.
+
+Fry Guy insisted that the Legion intended to crash the phone system
+on a national holiday. And when AT&T crashed on Martin Luther King Day,
+1990, this lent a credence to his claim that genuinely alarmed telco
+security and the Secret Service.
+
+Fry Guy eventually pled guilty on May 31, 1990. On September 14,
+he was sentenced to forty-four months' probation and four hundred hours'
+community service. He could have had it much worse; but it made sense
+to prosecutors to take it easy on this teenage minor, while zeroing
+in on the notorious kingpins of the Legion of Doom.
+
+But the case against LoD had nagging flaws. Despite the best effort
+of investigators, it was impossible to prove that the Legion had crashed
+the phone system on January 15, because they, in fact, hadn't done so.
+The investigations of 1989 did show that certain members of
+the Legion of Doom had achieved unprecedented power over the telco
+switching stations, and that they were in active conspiracy
+to obtain more power yet. Investigators were privately convinced
+that the Legion of Doom intended to do awful things with this knowledge,
+but mere evil intent was not enough to put them in jail.
+
+And although the Atlanta Three--Prophet, Leftist, and especially Urvile--
+had taught Fry Guy plenty, they were not themselves credit-card fraudsters.
+The only thing they'd "stolen" was long-distance service--and since they'd
+done much of that through phone-switch manipulation, there was no easy way
+to judge how much they'd "stolen," or whether this practice was even "theft"
+of any easily recognizable kind.
+
+Fry Guy's theft of long-distance codes had cost the phone companies plenty.
+The theft of long-distance service may be a fairly theoretical "loss,"
+but it costs genuine money and genuine time to delete all those stolen codes,
+and to re-issue new codes to the innocent owners of those corrupted codes.
+The owners of the codes themselves are victimized, and lose time and money
+and peace of mind in the hassle. And then there were the credit-card victims
+to deal with, too, and Western Union. When it came to rip-off, Fry Guy was
+far more of a thief than LoD. It was only when it came to actual computer
+expertise that Fry Guy was small potatoes.
+
+The Atlanta Legion thought most "rules" of cyberspace were for rodents
+and losers, but they DID have rules. THEY NEVER CRASHED ANYTHING,
+AND THEY NEVER TOOK MONEY. These were rough rules-of-thumb, and
+rather dubious principles when it comes to the ethical subtleties
+of cyberspace, but they enabled the Atlanta Three to operate with
+a relatively clear conscience (though never with peace of mind).
+
+If you didn't hack for money, if you weren't robbing people of actual funds
+--money in the bank, that is-- then nobody REALLY got hurt, in LoD's opinion.
+"Theft of service" was a bogus issue, and "intellectual property" was
+a bad joke. But LoD had only elitist contempt for rip-off artists,
+"leechers," thieves. They considered themselves clean. In their opinion,
+if you didn't smash-up or crash any systems --(well, not on purpose, anyhow--
+accidents can happen, just ask Robert Morris) then it was very unfair
+to call you a "vandal" or a "cracker." When you were hanging out on-line
+with your "pals" in telco security, you could face them down from the higher
+plane of hacker morality. And you could mock the police from the supercilious
+heights of your hacker's quest for pure knowledge.
+
+But from the point of view of law enforcement and telco security, however,
+Fry Guy was not really dangerous. The Atlanta Three WERE dangerous.
+It wasn't the crimes they were committing, but the DANGER,
+the potential hazard, the sheer TECHNICAL POWER LoD had accumulated,
+that had made the situation untenable. Fry Guy was not LoD.
+He'd never laid eyes on anyone in LoD; his only contacts with them
+had been electronic. Core members of the Legion of Doom tended to meet
+physically for conventions every year or so, to get drunk, give each other
+the hacker high-sign, send out for pizza and ravage hotel suites.
+Fry Guy had never done any of this. Deborah Daniels assessed Fry Guy
+accurately as "an LoD wannabe."
+
+Nevertheless Fry Guy's crimes would be directly attributed to LoD
+in much future police propaganda. LoD would be described as
+"a closely knit group" involved in "numerous illegal activities"
+including "stealing and modifying individual credit histories,"
+and "fraudulently obtaining money and property." Fry Guy did this,
+but the Atlanta Three didn't; they simply weren't into theft,
+but rather intrusion. This caused a strange kink in
+the prosecution's strategy. LoD were accused of
+"disseminating information about attacking computers
+to other computer hackers in an effort to shift the focus
+of law enforcement to those other hackers and away from the Legion of Doom."
+
+This last accusation (taken directly from a press release by the Chicago
+Computer Fraud and Abuse Task Force) sounds particularly far-fetched.
+One might conclude at this point that investigators would have been
+well-advised to go ahead and "shift their focus" from the "Legion of Doom."
+Maybe they SHOULD concentrate on "those other hackers"--the ones who were
+actually stealing money and physical objects.
+
+But the Hacker Crackdown of 1990 was not a simple policing action.
+It wasn't meant just to walk the beat in cyberspace--it was a CRACKDOWN,
+a deliberate attempt to nail the core of the operation, to send a dire
+and potent message that would settle the hash of the digital underground
+for good.
+
+By this reasoning, Fry Guy wasn't much more than the electronic equivalent
+of a cheap streetcorner dope dealer. As long as the masterminds of LoD were
+still flagrantly operating, pushing their mountains of illicit knowledge
+right and left, and whipping up enthusiasm for blatant lawbreaking,
+then there would be an INFINITE SUPPLY of Fry Guys.
+
+Because LoD were flagrant, they had left trails everywhere,
+to be picked up by law enforcement in New York, Indiana,
+Florida, Texas, Arizona, Missouri, even Australia.
+But 1990's war on the Legion of Doom was led out of Illinois,
+by the Chicago Computer Fraud and Abuse Task Force.
+
+#
+
+The Computer Fraud and Abuse Task Force, led by federal prosecutor
+William J. Cook, had started in 1987 and had swiftly become one
+of the most aggressive local "dedicated computer-crime units."
+Chicago was a natural home for such a group. The world's first
+computer bulletin-board system had been invented in Illinois.
+The state of Illinois had some of the nation's first and sternest
+computer crime laws. Illinois State Police were markedly alert
+to the possibilities of white-collar crime and electronic fraud.
+
+And William J. Cook in particular was a rising star in
+electronic crime-busting. He and his fellow federal prosecutors
+at the U.S. Attorney's office in Chicago had a tight relation
+with the Secret Service, especially go-getting Chicago-based agent
+Timothy Foley. While Cook and his Department of Justice colleagues
+plotted strategy, Foley was their man on the street.
+
+Throughout the 1980s, the federal government had given prosecutors
+an armory of new, untried legal tools against computer crime.
+Cook and his colleagues were pioneers in the use of these new statutes
+in the real-life cut-and-thrust of the federal courtroom.
+
+On October 2, 1986, the US Senate had passed the
+"Computer Fraud and Abuse Act" unanimously, but there
+were pitifully few convictions under this statute.
+Cook's group took their name from this statute,
+since they were determined to transform this powerful but
+rather theoretical Act of Congress into a real-life engine
+of legal destruction against computer fraudsters and scofflaws.
+
+It was not a question of merely discovering crimes,
+investigating them, and then trying and punishing their
+perpetrators. The Chicago unit, like most everyone else
+in the business, already KNEW who the bad guys were:
+the Legion of Doom and the writers and editors of Phrack.
+The task at hand was to find some legal means of putting
+these characters away.
+
+This approach might seem a bit dubious, to someone not
+acquainted with the gritty realities of prosecutorial work.
+But prosecutors don't put people in jail for crimes
+they have committed; they put people in jail for crimes
+they have committed THAT CAN BE PROVED IN COURT.
+Chicago federal police put Al Capone in prison
+for income-tax fraud. Chicago is a big town,
+with a rough-and-ready bare-knuckle tradition
+on both sides of the law.
+
+Fry Guy had broken the case wide open and alerted telco security
+to the scope of the problem. But Fry Guy's crimes would not
+put the Atlanta Three behind bars--much less the wacko underground
+journalists of Phrack. So on July 22, 1989, the same day that
+Fry Guy was raided in Indiana, the Secret Service descended upon
+the Atlanta Three.
+
+This was likely inevitable. By the summer of 1989, law enforcement
+were closing in on the Atlanta Three from at least six directions at once.
+First, there were the leads from Fry Guy, which had led to the DNR registers
+being installed on the lines of the Atlanta Three. The DNR evidence alone
+would have finished them off, sooner or later.
+
+But second, the Atlanta lads were already well-known to Control-C
+and his telco security sponsors. LoD's contacts with telco security
+had made them overconfident and even more boastful than usual;
+they felt that they had powerful friends in high places,
+and that they were being openly tolerated by telco security.
+But BellSouth's Intrusion Task Force were hot on the trail of LoD
+and sparing no effort or expense.
+
+The Atlanta Three had also been identified by name and listed
+on the extensive anti-hacker files maintained, and retailed for pay,
+by private security operative John Maxfield of Detroit.
+Maxfield, who had extensive ties to telco security
+and many informants in the underground, was a bete noire
+of the Phrack crowd, and the dislike was mutual.
+
+
+The Atlanta Three themselves had written articles for Phrack.
+This boastful act could not possibly escape telco and law enforcement
+attention.
+
+"Knightmare," a high-school age hacker from Arizona,
+was a close friend and disciple of Atlanta LoD,
+but he had been nabbed by the formidable Arizona
+Organized Crime and Racketeering Unit. Knightmare was
+on some of LoD's favorite boards--"Black Ice" in particular--
+and was privy to their secrets. And to have Gail Thackeray,
+the Assistant Attorney General of Arizona, on one's trail
+was a dreadful peril for any hacker.
+
+And perhaps worst of all, Prophet had committed a major blunder
+by passing an illicitly copied BellSouth computer-file to Knight Lightning,
+who had published it in Phrack. This, as we will see, was an act of dire
+consequence for almost everyone concerned.
+
+On July 22, 1989, the Secret Service showed up at the Leftist's house,
+where he lived with his parents. A massive squad of some twenty officers
+surrounded the building: Secret Service, federal marshals, local police,
+possibly BellSouth telco security; it was hard to tell in the crush.
+Leftist's dad, at work in his basement office, first noticed
+a muscular stranger in plain clothes crashing through the
+back yard with a drawn pistol. As more strangers poured
+into the house, Leftist's dad naturally assumed there was
+an armed robbery in progress.
+
+Like most hacker parents, Leftist's mom and dad had only the vaguest
+notions of what their son had been up to all this time. Leftist had
+a day-job repairing computer hardware. His obsession with computers
+seemed a bit odd, but harmless enough, and likely to produce a well-
+paying career. The sudden, overwhelming raid left Leftist's
+parents traumatized.
+
+The Leftist himself had been out after work with his co-workers,
+surrounding a couple of pitchers of margaritas. As he came trucking
+on tequila-numbed feet up the pavement, toting a bag full of floppy-disks,
+he noticed a large number of unmarked cars parked in his driveway.
+All the cars sported tiny microwave antennas.
+
+The Secret Service had knocked the front door off its hinges,
+almost flattening his mom.
+
+Inside, Leftist was greeted by Special Agent James Cool
+of the US Secret Service, Atlanta office. Leftist was flabbergasted.
+He'd never met a Secret Service agent before. He could not imagine
+that he'd ever done anything worthy of federal attention.
+He'd always figured that if his activities became intolerable,
+one of his contacts in telco security would give him a private
+phone-call and tell him to knock it off.
+
+But now Leftist was pat-searched for weapons by grim professionals,
+and his bag of floppies was quickly seized. He and his parents were
+all shepherded into separate rooms and grilled at length as a score
+of officers scoured their home for anything electronic.
+
+Leftist was horrified as his treasured IBM AT personal computer
+with its forty-meg hard disk, and his recently purchased 80386 IBM-clone
+with a whopping hundred-meg hard disk, both went swiftly out the door
+in Secret Service custody. They also seized all his disks, all his notebooks,
+and a tremendous booty in dogeared telco documents that Leftist had snitched
+out of trash dumpsters.
+
+Leftist figured the whole thing for a big misunderstanding.
+He'd never been into MILITARY computers. He wasn't a SPY or a COMMUNIST.
+He was just a good ol' Georgia hacker, and now he just wanted all these
+people out of the house. But it seemed they wouldn't go until he made
+some kind of statement.
+
+And so, he levelled with them.
+
+And that, Leftist said later from his federal prison camp in Talladega,
+Alabama, was a big mistake. The Atlanta area was unique,
+in that it had three members of the Legion of Doom who actually
+occupied more or less the same physical locality. Unlike the rest
+of LoD, who tended to associate by phone and computer,
+Atlanta LoD actually WERE "tightly knit." It was no real
+surprise that the Secret Service agents apprehending Urvile
+at the computer-labs at Georgia Tech, would discover Prophet
+with him as well.
+
+Urvile, a 21-year-old Georgia Tech student in polymer chemistry,
+posed quite a puzzling case for law enforcement. Urvile--also known
+as "Necron 99," as well as other handles, for he tended to change his
+cover-alias about once a month--was both an accomplished hacker
+and a fanatic simulation-gamer.
+
+Simulation games are an unusual hobby; but then hackers are unusual people,
+and their favorite pastimes tend to be somewhat out of the ordinary.
+The best-known American simulation game is probably "Dungeons & Dragons,"
+a multi-player parlor entertainment played with paper, maps, pencils,
+statistical tables and a variety of oddly-shaped dice. Players pretend
+to be heroic characters exploring a wholly-invented fantasy world.
+The fantasy worlds of simulation gaming are commonly pseudo-medieval,
+involving swords and sorcery--spell-casting wizards, knights in armor,
+unicorns and dragons, demons and goblins.
+
+Urvile and his fellow gamers preferred their fantasies highly technological.
+They made use of a game known as "G.U.R.P.S.," the "Generic Universal Role
+Playing System," published by a company called Steve Jackson Games (SJG).
+
+"G.U.R.P.S." served as a framework for creating a wide variety of artificial
+fantasy worlds. Steve Jackson Games published a smorgasboard of books,
+full of detailed information and gaming hints, which were used to flesh-out
+many different fantastic backgrounds for the basic GURPS framework.
+Urvile made extensive use of two SJG books called GURPS High-Tech
+and GURPS Special Ops.
+
+In the artificial fantasy-world of GURPS Special Ops,
+players entered a modern fantasy of intrigue and international espionage.
+On beginning the game, players started small and powerless,
+perhaps as minor-league CIA agents or penny-ante arms dealers.
+But as players persisted through a series of game sessions
+(game sessions generally lasted for hours, over long,
+elaborate campaigns that might be pursued for months on end)
+then they would achieve new skills, new knowledge, new power.
+They would acquire and hone new abilities, such as marksmanship,
+karate, wiretapping, or Watergate burglary. They could also win
+various kinds of imaginary booty, like Berettas, or martini shakers,
+or fast cars with ejection seats and machine-guns under the headlights.
+
+As might be imagined from the complexity of these games,
+Urvile's gaming notes were very detailed and extensive.
+Urvile was a "dungeon-master," inventing scenarios
+for his fellow gamers, giant simulated adventure-puzzles
+for his friends to unravel. Urvile's game notes covered
+dozens of pages with all sorts of exotic lunacy, all about
+ninja raids on Libya and break-ins on encrypted Red Chinese supercomputers.
+His notes were written on scrap-paper and kept in loose-leaf binders.
+
+The handiest scrap paper around Urvile's college digs were the many pounds of
+BellSouth printouts and documents that he had snitched out of telco dumpsters.
+His notes were written on the back of misappropriated telco property.
+Worse yet, the gaming notes were chaotically interspersed with Urvile's
+hand-scrawled records involving ACTUAL COMPUTER INTRUSIONS that he
+had committed.
+
+Not only was it next to impossible to tell Urvile's fantasy game-notes
+from cyberspace "reality," but Urvile himself barely made this distinction.
+It's no exaggeration to say that to Urvile it was ALL a game. Urvile was
+very bright, highly imaginative, and quite careless of other people's notions
+of propriety. His connection to "reality" was not something to which he paid
+a great deal of attention.
+
+Hacking was a game for Urvile. It was an amusement he was carrying out,
+it was something he was doing for fun. And Urvile was an obsessive young man.
+He could no more stop hacking than he could stop in the middle of
+a jigsaw puzzle, or stop in the middle of reading a Stephen Donaldson
+fantasy trilogy. (The name "Urvile" came from a best-selling Donaldson novel.)
+
+Urvile's airy, bulletproof attitude seriously annoyed his interrogators.
+First of all, he didn't consider that he'd done anything wrong.
+There was scarcely a shred of honest remorse in him. On the contrary,
+he seemed privately convinced that his police interrogators were operating
+in a demented fantasy-world all their own. Urvile was too polite
+and well-behaved to say this straight-out, but his reactions were askew
+and disquieting.
+
+For instance, there was the business about LoD's ability
+to monitor phone-calls to the police and Secret Service.
+Urvile agreed that this was quite possible, and posed
+no big problem for LoD. In fact, he and his friends
+had kicked the idea around on the "Black Ice" board,
+much as they had discussed many other nifty notions,
+such as building personal flame-throwers and jury-rigging
+fistfulls of blasting-caps. They had hundreds of dial-up numbers
+for government agencies that they'd gotten through scanning Atlanta phones,
+or had pulled from raided VAX/VMS mainframe computers.
+
+Basically, they'd never gotten around to listening in on the cops
+because the idea wasn't interesting enough to bother with.
+Besides, if they'd been monitoring Secret Service phone calls,
+obviously they'd never have been caught in the first place. Right?
+
+The Secret Service was less than satisfied with this rapier-like hacker logic.
+
+Then there was the issue of crashing the phone system. No problem,
+Urvile admitted sunnily. Atlanta LoD could have shut down phone service
+all over Atlanta any time they liked. EVEN THE 911 SERVICE?
+Nothing special about that, Urvile explained patiently.
+Bring the switch to its knees, with say the UNIX "makedir" bug,
+and 911 goes down too as a matter of course. The 911 system
+wasn't very interesting, frankly. It might be tremendously
+interesting to cops (for odd reasons of their own), but as
+technical challenges went, the 911 service was yawnsville.
+
+So of course the Atlanta Three could crash service.
+They probably could have crashed service all over
+BellSouth territory, if they'd worked at it for a while.
+But Atlanta LoD weren't crashers. Only losers and rodents
+were crashers. LoD were ELITE.
+
+Urvile was privately convinced that sheer technical
+expertise could win him free of any kind of problem.
+As far as he was concerned, elite status in the digital
+underground had placed him permanently beyond the intellectual
+grasp of cops and straights. Urvile had a lot to learn.
+
+Of the three LoD stalwarts, Prophet was in the most direct trouble.
+Prophet was a UNIX programming expert who burrowed in and out
+of the Internet as a matter of course. He'd started his hacking
+career at around age 14, meddling with a UNIX mainframe system
+at the University of North Carolina.
+
+Prophet himself had written the handy Legion of Doom
+file "UNIX Use and Security From the Ground Up."
+UNIX (pronounced "you-nicks") is a powerful,
+flexible computer operating-system, for multi-user,
+multi-tasking computers. In 1969, when UNIX was created
+in Bell Labs, such computers were exclusive to large
+corporations and universities, but today UNIX is run
+on thousands of powerful home machines. UNIX was
+particularly well-suited to telecommunications programming,
+and had become a standard in the field. Naturally, UNIX
+also became a standard for the elite hacker and phone phreak.
+Lately, Prophet had not been so active as Leftist and Urvile,
+but Prophet was a recidivist. In 1986, when he was eighteen,
+Prophet had been convicted of "unauthorized access
+to a computer network" in North Carolina. He'd been
+discovered breaking into the Southern Bell Data Network,
+a UNIX-based internal telco network supposedly closed to the public.
+He'd gotten a typical hacker sentence: six months suspended,
+120 hours community service, and three years' probation.
+
+After that humiliating bust, Prophet had gotten rid of most of his
+tonnage of illicit phreak and hacker data, and had tried to go straight.
+He was, after all, still on probation. But by the autumn of 1988,
+the temptations of cyberspace had proved too much for young Prophet,
+and he was shoulder-to-shoulder with Urvile and Leftist into some
+of the hairiest systems around.
+
+In early September 1988, he'd broken into BellSouth's centralized
+automation system, AIMSX or "Advanced Information Management System."
+AIMSX was an internal business network for BellSouth, where telco
+employees stored electronic mail, databases, memos, and calendars,
+and did text processing. Since AIMSX did not have public dial-ups,
+it was considered utterly invisible to the public, and was not well-secured
+--it didn't even require passwords. Prophet abused an account known
+as "waa1," the personal account of an unsuspecting telco employee.
+Disguised as the owner of waa1, Prophet made about ten visits to AIMSX.
+
+Prophet did not damage or delete anything in the system.
+His presence in AIMSX was harmless and almost invisible.
+But he could not rest content with that.
+
+One particular piece of processed text on AIMSX was a telco document
+known as "Bell South Standard Practice 660-225-104SV Control Office
+Administration of Enhanced 911 Services for Special Services
+and Major Account Centers dated March 1988."
+
+Prophet had not been looking for this document. It was merely one
+among hundreds of similar documents with impenetrable titles.
+However, having blundered over it in the course of his illicit
+wanderings through AIMSX, he decided to take it with him as a trophy.
+It might prove very useful in some future boasting, bragging,
+and strutting session. So, some time in September 1988,
+Prophet ordered the AIMSX mainframe computer to copy this document
+(henceforth called simply called "the E911 Document") and to transfer
+this copy to his home computer.
+
+No one noticed that Prophet had done this. He had "stolen"
+the E911 Document in some sense, but notions of property
+in cyberspace can be tricky. BellSouth noticed nothing wrong,
+because BellSouth still had their original copy. They had not
+been "robbed" of the document itself. Many people were supposed
+to copy this document--specifically, people who worked for the
+nineteen BellSouth "special services and major account centers,"
+scattered throughout the Southeastern United States. That was
+what it was for, why it was present on a computer network
+in the first place: so that it could be copied and read--
+by telco employees. But now the data had been copied
+by someone who wasn't supposed to look at it.
+
+Prophet now had his trophy. But he further decided to store
+yet another copy of the E911 Document on another person's computer.
+This unwitting person was a computer enthusiast named Richard Andrews
+who lived near Joliet, Illinois. Richard Andrews was a UNIX programmer
+by trade, and ran a powerful UNIX board called "Jolnet," in the basement
+of his house.
+
+Prophet, using the handle "Robert Johnson," had obtained an account
+on Richard Andrews' computer. And there he stashed the E911 Document,
+by storing it in his own private section of Andrews' computer.
+
+Why did Prophet do this? If Prophet had eliminated the E911 Document
+from his own computer, and kept it hundreds of miles away, on another machine, under an
+alias, then he might have been fairly safe from discovery and prosecution--
+although his sneaky action had certainly put the unsuspecting Richard Andrews
+at risk.
+
+But, like most hackers, Prophet was a pack-rat for illicit data.
+When it came to the crunch, he could not bear to part from his trophy.
+When Prophet's place in Decatur, Georgia was raided in July 1989,
+there was the E911 Document, a smoking gun. And there was Prophet
+in the hands of the Secret Service, doing his best to "explain."
+
+Our story now takes us away from the Atlanta Three and their raids
+of the Summer of 1989. We must leave Atlanta Three "cooperating fully"
+with their numerous investigators. And all three of them did cooperate,
+as their Sentencing Memorandum from the US District Court of the
+Northern Division of Georgia explained--just before all three of them
+were sentenced to various federal prisons in November 1990.
+
+We must now catch up on the other aspects of the war on the Legion of Doom.
+The war on the Legion was a war on a network--in fact, a network of three
+networks, which intertwined and interrelated in a complex fashion.
+The Legion itself, with Atlanta LoD, and their hanger-on Fry Guy,
+were the first network. The second network was Phrack magazine,
+with its editors and contributors.
+
+The third network involved the electronic circle around a hacker
+known as "Terminus."
+
+The war against these hacker networks was carried out by
+a law enforcement network. Atlanta LoD and Fry Guy
+were pursued by USSS agents and federal prosecutors in Atlanta,
+Indiana, and Chicago. "Terminus" found himself pursued by USSS
+and federal prosecutors from Baltimore and Chicago. And the war
+against Phrack was almost entirely a Chicago operation.
+
+The investigation of Terminus involved a great deal of energy,
+mostly from the Chicago Task Force, but it was to be the least-known
+and least-publicized of the Crackdown operations. Terminus, who lived
+in Maryland, was a UNIX programmer and consultant, fairly well-known
+(under his given name) in the UNIX community, as an acknowledged expert
+on AT&T minicomputers. Terminus idolized AT&T, especially Bellcore,
+and longed for public recognition as a UNIX expert; his highest ambition
+was to work for Bell Labs.
+
+But Terminus had odd friends and a spotted history.
+Terminus had once been the subject of an admiring interview
+in Phrack (Volume II, Issue 14, Phile 2--dated May 1987).
+In this article, Phrack co-editor Taran King described
+"Terminus" as an electronics engineer, 5'9", brown-haired,
+born in 1959--at 28 years old, quite mature for a hacker.
+
+Terminus had once been sysop of a phreak/hack underground board
+called "MetroNet," which ran on an Apple II. Later he'd replaced
+"MetroNet" with an underground board called "MegaNet,"
+specializing in IBMs. In his younger days, Terminus had written
+one of the very first and most elegant code-scanning programs
+for the IBM-PC. This program had been widely distributed
+in the underground. Uncounted legions of PC-owning phreaks and
+hackers had used Terminus's scanner program to rip-off telco codes.
+This feat had not escaped the attention of telco security;
+it hardly could, since Terminus's earlier handle, "Terminal Technician,"
+was proudly written right on the program.
+
+When he became a full-time computer professional
+(specializing in telecommunications programming),
+he adopted the handle Terminus, meant to indicate that he
+had "reached the final point of being a proficient hacker."
+He'd moved up to the UNIX-based "Netsys" board on an AT&T computer,
+with four phone lines and an impressive 240 megs of storage.
+"Netsys" carried complete issues of Phrack, and Terminus was
+quite friendly with its publishers, Taran King and Knight Lightning.
+
+In the early 1980s, Terminus had been a regular on Plovernet,
+Pirate-80, Sherwood Forest and Shadowland, all well-known pirate boards,
+all heavily frequented by the Legion of Doom. As it happened, Terminus
+was never officially "in LoD," because he'd never been given the official
+LoD high-sign and back-slap by Legion maven Lex Luthor. Terminus had
+never physically met anyone from LoD. But that scarcely mattered much--
+the Atlanta Three themselves had never been officially vetted by Lex, either.
+
+As far as law enforcement was concerned, the issues were clear.
+Terminus was a full-time, adult computer professional
+with particular skills at AT&T software and hardware--
+but Terminus reeked of the Legion of Doom and the underground.
+
+On February 1, 1990--half a month after the Martin Luther King Day Crash--
+USSS agents Tim Foley from Chicago, and Jack Lewis from the Baltimore office,
+accompanied by AT&T security officer Jerry Dalton, travelled to Middle Town,
+Maryland. There they grilled Terminus in his home (to the stark terror of
+his wife and small children), and, in their customary fashion, hauled his
+computers out the door.
+
+The Netsys machine proved to contain a plethora of arcane UNIX software--
+proprietary source code formally owned by AT&T. Software such as:
+UNIX System Five Release 3.2; UNIX SV Release 3.1; UUCP communications
+software; KORN SHELL; RFS; IWB; WWB; DWB; the C++ programming language;
+PMON; TOOL CHEST; QUEST; DACT, and S FIND.
+
+In the long-established piratical tradition of the underground,
+Terminus had been trading this illicitly-copied software with
+a small circle of fellow UNIX programmers. Very unwisely,
+he had stored seven years of his electronic mail on his Netsys machine,
+which documented all the friendly arrangements he had made with
+his various colleagues.
+
+Terminus had not crashed the AT&T phone system on January 15.
+He was, however, blithely running a not-for-profit AT&T
+software-piracy ring. This was not an activity AT&T found amusing.
+AT&T security officer Jerry Dalton valued this "stolen" property
+at over three hundred thousand dollars.
+
+AT&T's entry into the tussle of free enterprise had been complicated
+by the new, vague groundrules of the information economy.
+Until the break-up of Ma Bell, AT&T was forbidden to sell
+computer hardware or software. Ma Bell was the phone company;
+Ma Bell was not allowed to use the enormous revenue from
+telephone utilities, in order to finance any entry into
+the computer market.
+
+AT&T nevertheless invented the UNIX operating system.
+And somehow AT&T managed to make UNIX a minor source of income.
+Weirdly, UNIX was not sold as computer software,
+but actually retailed under an obscure regulatory
+exemption allowing sales of surplus equipment and scrap.
+Any bolder attempt to promote or retail UNIX would have
+aroused angry legal opposition from computer companies.
+Instead, UNIX was licensed to universities, at modest rates,
+where the acids of academic freedom ate away steadily at AT&T's
+proprietary rights.
+
+Come the breakup, AT&T recognized that UNIX was a potential gold-mine.
+By now, large chunks of UNIX code had been created that were not AT&T's,
+and were being sold by others. An entire rival UNIX-based operating system
+had arisen in Berkeley, California (one of the world's great founts of
+ideological hackerdom). Today, "hackers" commonly consider "Berkeley UNIX"
+to be technically superior to AT&T's "System V UNIX," but AT&T has not
+allowed mere technical elegance to intrude on the real-world business
+of marketing proprietary software. AT&T has made its own code deliberately
+incompatible with other folks' UNIX, and has written code that it can prove
+is copyrightable, even if that code happens to be somewhat awkward--"kludgey."
+AT&T UNIX user licenses are serious business agreements, replete with very
+clear copyright statements and non-disclosure clauses.
+
+AT&T has not exactly kept the UNIX cat in the bag,
+but it kept a grip on its scruff with some success.
+By the rampant, explosive standards of software piracy,
+AT&T UNIX source code is heavily copyrighted, well-guarded,
+well-licensed. UNIX was traditionally run only on
+mainframe machines, owned by large groups of suit-and-tie
+professionals, rather than on bedroom machines where
+people can get up to easy mischief.
+
+And AT&T UNIX source code is serious high-level programming.
+The number of skilled UNIX programmers with any actual motive
+to swipe UNIX source code is small. It's tiny, compared to
+the tens of thousands prepared to rip-off, say, entertaining
+PC games like "Leisure Suit Larry."
+
+But by 1989, the warez-d00d underground, in the persons of Terminus
+and his friends, was gnawing at AT&T UNIX. And the property in question
+was not sold for twenty bucks over the counter at the local branch of
+Babbage's or Egghead's; this was massive, sophisticated, multi-line,
+multi-author corporate code worth tens of thousands of dollars.
+
+It must be recognized at this point that Terminus's purported ring of UNIX
+software pirates had not actually made any money from their suspected crimes.
+The $300,000 dollar figure bandied about for the contents of Terminus's
+computer did not mean that Terminus was in actual illicit possession
+of three hundred thousand of AT&T's dollars. Terminus was shipping
+software back and forth, privately, person to person, for free.
+He was not making a commercial business of piracy. He hadn't
+asked for money; he didn't take money. He lived quite modestly.
+
+AT&T employees--as well as freelance UNIX consultants, like Terminus--
+commonly worked with "proprietary" AT&T software, both in the office
+and at home on their private machines. AT&T rarely sent security officers
+out to comb the hard disks of its consultants. Cheap freelance UNIX
+contractors were quite useful to AT&T; they didn't have health insurance
+or retirement programs, much less union membership in the Communication
+Workers of America. They were humble digital drudges, wandering with mop
+and bucket through the Great Technological Temple of AT&T; but when the
+Secret Service arrived at their homes, it seemed they were eating with
+company silverware and sleeping on company sheets! Outrageously, they
+behaved as if the things they worked with every day belonged to them!
+
+And these were no mere hacker teenagers with their hands full
+of trash-paper and their noses pressed to the corporate windowpane.
+These guys were UNIX wizards, not only carrying AT&T data in their
+machines and their heads, but eagerly networking about it,
+over machines that were far more powerful than anything previously
+imagined in private hands. How do you keep people disposable,
+yet assure their awestruck respect for your property? It was a dilemma.
+
+Much UNIX code was public-domain, available for free. Much "proprietary"
+UNIX code had been extensively re-written, perhaps altered so much that it
+became an entirely new product--or perhaps not. Intellectual property rights
+for software developers were, and are, extraordinarily complex and confused.
+And software "piracy," like the private copying of videos, is one of the most
+widely practiced "crimes" in the world today.
+
+The USSS were not experts in UNIX or familiar with the customs of its use.
+The United States Secret Service, considered as a body, did not have one single
+person in it who could program in a UNIX environment--no, not even one.
+The Secret Service WERE making extensive use of expert help, but the "experts"
+they had chosen were AT&T and Bellcore security officials, the very victims of
+the purported crimes under investigation, the very people whose interest in
+AT&T's "proprietary" software was most pronounced.
+
+On February 6, 1990, Terminus was arrested by Agent Lewis.
+Eventually, Terminus would be sent to prison for his illicit
+use of a piece of AT&T software.
+
+The issue of pirated AT&T software would bubble along in the background
+during the war on the Legion of Doom. Some half-dozen of Terminus's on-line
+acquaintances, including people in Illinois, Texas and California,
+were grilled by the Secret Service in connection with the illicit
+copying of software. Except for Terminus, however, none were charged
+with a crime. None of them shared his peculiar prominence in the
+hacker underground.
+
+But that did not mean that these people would, or could,
+stay out of trouble. The transferral of illicit data in
+cyberspace is hazy and ill-defined business, with paradoxical
+dangers for everyone concerned: hackers, signal carriers,
+board owners, cops, prosecutors, even random passers-by.
+Sometimes, well-meant attempts to avert trouble
+or punish wrongdoing bring more trouble than
+would simple ignorance, indifference or impropriety.
+
+Terminus's "Netsys" board was not a common-or-garden
+bulletin board system, though it had most of the usual
+functions of a board. Netsys was not a stand-alone machine,
+but part of the globe-spanning "UUCP" cooperative network.
+The UUCP network uses a set of Unix software programs called
+"Unix-to-Unix Copy," which allows Unix systems to throw data to
+one another at high speed through the public telephone network.
+UUCP is a radically decentralized, not-for-profit network of UNIX computers.
+There are tens of thousands of these UNIX machines. Some are small,
+but many are powerful and also link to other networks. UUCP has
+certain arcane links to major networks such as JANET, EasyNet, BITNET,
+JUNET, VNET, DASnet, PeaceNet and FidoNet, as well as the gigantic Internet.
+(The so-called "Internet" is not actually a network itself, but rather an
+"internetwork" connections standard that allows several globe-spanning
+computer networks to communicate with one another. Readers fascinated
+by the weird and intricate tangles of modern computer networks may enjoy
+John S. Quarterman's authoritative 719-page explication, The Matrix,
+Digital Press, 1990.)
+
+A skilled user of Terminus' UNIX machine could send and receive
+electronic mail from almost any major computer network in the world.
+Netsys was not called a "board" per se, but rather a "node."
+"Nodes" were larger, faster, and more sophisticated than mere "boards,"
+and for hackers, to hang out on internationally-connected "nodes"
+was quite the step up from merely hanging out on local "boards."
+
+Terminus's Netsys node in Maryland had a number of direct
+links to other, similar UUCP nodes, run by people who shared his
+interests and at least something of his free-wheeling attitude.
+One of these nodes was Jolnet, owned by Richard Andrews, who,
+like Terminus, was an independent UNIX consultant.
+Jolnet also ran UNIX, and could be contacted at high speed
+by mainframe machines from all over the world. Jolnet was
+quite a sophisticated piece of work, technically speaking,
+but it was still run by an individual, as a private,
+not-for-profit hobby. Jolnet was mostly used by other
+UNIX programmers--for mail, storage, and access to networks.
+Jolnet supplied access network access to about two hundred people,
+as well as a local junior college.
+
+Among its various features and services, Jolnet also carried
+Phrack magazine.
+
+For reasons of his own, Richard Andrews had become suspicious
+of a new user called "Robert Johnson." Richard Andrews
+took it upon himself to have a look at what "Robert Johnson"
+was storing in Jolnet. And Andrews found the E911 Document.
+
+"Robert Johnson" was the Prophet from the Legion of Doom,
+and the E911 Document was illicitly copied data from Prophet's
+raid on the BellSouth computers.
+
+The E911 Document, a particularly illicit piece of digital property,
+was about to resume its long, complex, and disastrous career.
+
+It struck Andrews as fishy that someone not a telephone employee
+should have a document referring to the "Enhanced 911 System."
+Besides, the document itself bore an obvious warning.
+
+"WARNING: NOT FOR USE OR DISCLOSURE OUTSIDE BELLSOUTH
+OR ANY OF ITS SUBSIDIARIES EXCEPT UNDER WRITTEN AGREEMENT."
+
+These standard nondisclosure tags are often appended to all sorts
+of corporate material. Telcos as a species are particularly notorious
+for stamping most everything in sight as "not for use or disclosure."
+Still, this particular piece of data was about the 911 System.
+That sounded bad to Rich Andrews.
+
+Andrews was not prepared to ignore this sort of trouble.
+He thought it would be wise to pass the document along
+to a friend and acquaintance on the UNIX network, for consultation.
+So, around September 1988, Andrews sent yet another copy of the
+E911 Document electronically to an AT&T employee, one Charles Boykin,
+who ran a UNIX-based node called "attctc" in Dallas, Texas.
+
+"Attctc" was the property of AT&T, and was run from AT&T's
+Customer Technology Center in Dallas, hence the name "attctc."
+"Attctc" was better-known as "Killer," the name of the machine
+that the system was running on. "Killer" was a hefty, powerful,
+AT&T 3B2 500 model, a multi-user, multi-tasking UNIX platform
+with 32 meg of memory and a mind-boggling 3.2 Gigabytes of storage.
+When Killer had first arrived in Texas, in 1985, the 3B2 had been
+one of AT&T's great white hopes for going head-to-head with IBM
+for the corporate computer-hardware market. "Killer" had been shipped
+to the Customer Technology Center in the Dallas Infomart, essentially
+a high-technology mall, and there it sat, a demonstration model.
+
+Charles Boykin, a veteran AT&T hardware and digital communications expert,
+was a local technical backup man for the AT&T 3B2 system. As a display model
+in the Infomart mall, "Killer" had little to do, and it seemed a shame
+to waste the system's capacity. So Boykin ingeniously wrote some UNIX
+bulletin-board software for "Killer," and plugged the machine in to the
+local phone network. "Killer's" debut in late 1985 made it the first
+publicly available UNIX site in the state of Texas. Anyone who wanted to
+play was welcome.
+
+The machine immediately attracted an electronic community.
+It joined the UUCP network, and offered network links
+to over eighty other computer sites, all of which became dependent
+on Killer for their links to the greater world of cyberspace.
+And it wasn't just for the big guys; personal computer users
+also stored freeware programs for the Amiga, the Apple,
+the IBM and the Macintosh on Killer's vast 3,200 meg archives.
+At one time, Killer had the largest library of public-domain
+Macintosh software in Texas.
+
+Eventually, Killer attracted about 1,500 users,
+all busily communicating, uploading and downloading,
+getting mail, gossipping, and linking to arcane
+and distant networks.
+
+Boykin received no pay for running Killer. He considered
+it good publicity for the AT&T 3B2 system (whose sales were
+somewhat less than stellar), but he also simply enjoyed
+the vibrant community his skill had created. He gave away
+the bulletin-board UNIX software he had written, free of charge.
+
+In the UNIX programming community, Charlie Boykin had the
+reputation of a warm, open-hearted, level-headed kind of guy.
+In 1989, a group of Texan UNIX professionals voted Boykin
+"System Administrator of the Year." He was considered
+a fellow you could trust for good advice.
+
+In September 1988, without warning, the E911 Document
+came plunging into Boykin's life, forwarded by Richard Andrews.
+Boykin immediately recognized that the Document was hot property.
+He was not a voice-communications man, and knew little about
+the ins and outs of the Baby Bells, but he certainly knew what
+the 911 System was, and he was angry to see confidential data
+about it in the hands of a nogoodnik. This was clearly a
+matter for telco security. So, on September 21, 1988, Boykin
+made yet ANOTHER copy of the E911 Document and passed this
+one along to a professional acquaintance of his, one Jerome Dalton,
+from AT&T Corporate Information Security. Jerry Dalton was the
+very fellow who would later raid Terminus's house.
+
+From AT&T's security division, the E911 Document went to Bellcore.
+
+Bellcore (or BELL COmmunications REsearch) had once been the central
+laboratory of the Bell System. Bell Labs employees had invented
+the UNIX operating system. Now Bellcore was a quasi-independent,
+jointly owned company that acted as the research arm for all seven
+of the Baby Bell RBOCs. Bellcore was in a good position to co-ordinate
+security technology and consultation for the RBOCs, and the gentleman in
+charge of this effort was Henry M. Kluepfel, a veteran of the Bell System
+who had worked there for twenty-four years.
+
+On October 13, 1988, Dalton passed the E911 Document to Henry Kluepfel.
+Kluepfel, a veteran expert witness in telecommunications fraud and
+computer-fraud cases, had certainly seen worse trouble than this.
+He recognized the document for what it was: a trophy from a hacker break-in.
+
+However, whatever harm had been done in the intrusion was presumably old news.
+At this point there seemed little to be done. Kluepfel made a careful note
+of the circumstances and shelved the problem for the time being.
+
+Whole months passed.
+
+February 1989 arrived. The Atlanta Three were living it up
+in Bell South's switches, and had not yet met their comeuppance.
+The Legion was thriving. So was Phrack magazine.
+A good six months had passed since Prophet's AIMSX break-in.
+Prophet, as hackers will, grew weary of sitting on his laurels.
+"Knight Lightning" and "Taran King," the editors of Phrack,
+were always begging Prophet for material they could publish.
+Prophet decided that the heat must be off by this time,
+and that he could safely brag, boast, and strut.
+
+So he sent a copy of the E911 Document--yet another one--
+from Rich Andrews' Jolnet machine to Knight Lightning's
+BITnet account at the University of Missouri.
+Let's review the fate of the document so far.
+
+0. The original E911 Document. This in the AIMSX system
+on a mainframe computer in Atlanta, available to hundreds of people,
+but all of them, presumably, BellSouth employees. An unknown number
+of them may have their own copies of this document, but they are all
+professionals and all trusted by the phone company.
+
+1. Prophet's illicit copy, at home on his own computer in Decatur, Georgia.
+
+2. Prophet's back-up copy, stored on Rich Andrew's Jolnet machine
+ in the basement of Rich Andrews' house near Joliet Illinois.
+
+3. Charles Boykin's copy on "Killer" in Dallas, Texas,
+ sent by Rich Andrews from Joliet.
+
+4. Jerry Dalton's copy at AT&T Corporate Information Security in New Jersey,
+ sent from Charles Boykin in Dallas.
+
+5. Henry Kluepfel's copy at Bellcore security headquarters in New Jersey,
+ sent by Dalton.
+6. Knight Lightning's copy, sent by Prophet from Rich Andrews' machine,
+ and now in Columbia, Missouri.
+
+We can see that the "security" situation of this proprietary document,
+once dug out of AIMSX, swiftly became bizarre. Without any money
+changing hands, without any particular special effort, this data
+had been reproduced at least six times and had spread itself all over
+the continent. By far the worst, however, was yet to come.
+
+In February 1989, Prophet and Knight Lightning bargained electronically
+over the fate of this trophy. Prophet wanted to boast, but, at the same time,
+scarcely wanted to be caught.
+
+For his part, Knight Lightning was eager to publish as much of the document
+as he could manage. Knight Lightning was a fledgling political-science major
+with a particular interest in freedom-of-information issues. He would gladly
+publish most anything that would reflect glory on the prowess of the
+underground and embarrass the telcos. However, Knight Lightning himself
+had contacts in telco security, and sometimes consulted them on material
+he'd received that might be too dicey for publication.
+
+Prophet and Knight Lightning decided to edit the E911 Document
+so as to delete most of its identifying traits. First of all,
+its large "NOT FOR USE OR DISCLOSURE" warning had to go.
+Then there were other matters. For instance, it listed
+the office telephone numbers of several BellSouth 911
+specialists in Florida. If these phone numbers were
+published in Phrack, the BellSouth employees involved
+would very likely be hassled by phone phreaks,
+which would anger BellSouth no end, and pose a
+definite operational hazard for both Prophet and Phrack.
+
+So Knight Lightning cut the Document almost in half,
+removing the phone numbers and some of the touchier
+and more specific information. He passed it back
+electronically to Prophet; Prophet was still nervous,
+so Knight Lightning cut a bit more. They finally agreed
+that it was ready to go, and that it would be published
+in Phrack under the pseudonym, "The Eavesdropper."
+
+And this was done on February 25, 1989.
+
+The twenty-fourth issue of Phrack featured a chatty interview
+with co-ed phone-phreak "Chanda Leir," three articles on BITNET
+and its links to other computer networks, an article on 800 and 900
+numbers by "Unknown User," "VaxCat's" article on telco basics
+(slyly entitled "Lifting Ma Bell's Veil of Secrecy,)" and
+the usual "Phrack World News."
+
+The News section, with painful irony, featured an extended account
+of the sentencing of "Shadowhawk," an eighteen-year-old Chicago hacker
+who had just been put in federal prison by William J. Cook himself.
+
+And then there were the two articles by "The Eavesdropper."
+The first was the edited E911 Document, now titled
+"Control Office Administration Of Enhanced 911 Services
+for Special Services and Major Account Centers."
+Eavesdropper's second article was a glossary of terms
+explaining the blizzard of telco acronyms and buzzwords
+in the E911 Document.
+
+The hapless document was now distributed, in the usual Phrack routine,
+to a good one hundred and fifty sites. Not a hundred and fifty PEOPLE,
+mind you--a hundred and fifty SITES, some of these sites linked to UNIX
+nodes or bulletin board systems, which themselves had readerships of tens,
+dozens, even hundreds of people.
+
+This was February 1989. Nothing happened immediately.
+Summer came, and the Atlanta crew were raided by the Secret Service.
+Fry Guy was apprehended. Still nothing whatever happened to Phrack.
+Six more issues of Phrack came out, 30 in all, more or less on
+a monthly schedule. Knight Lightning and co-editor Taran King
+went untouched.
+
+Phrack tended to duck and cover whenever the heat came down.
+During the summer busts of 1987--(hacker busts tended to cluster in summer,
+perhaps because hackers were easier to find at home than in college)--
+Phrack had ceased publication for several months, and laid low.
+Several LoD hangers-on had been arrested, but nothing had happened
+to the Phrack crew, the premiere gossips of the underground.
+In 1988, Phrack had been taken over by a new editor,
+"Crimson Death," a raucous youngster with a taste for anarchy files.
+1989, however, looked like a bounty year for the underground.
+Knight Lightning and his co-editor Taran King took up the reins again,
+and Phrack flourished throughout 1989. Atlanta LoD went down hard in
+the summer of 1989, but Phrack rolled merrily on. Prophet's E911 Document
+seemed unlikely to cause Phrack any trouble. By January 1990,
+it had been available in Phrack for almost a year. Kluepfel and Dalton,
+officers of Bellcore and AT&T security, had possessed the document
+for sixteen months--in fact, they'd had it even before Knight Lightning
+himself, and had done nothing in particular to stop its distribution.
+They hadn't even told Rich Andrews or Charles Boykin to erase the copies
+from their UNIX nodes, Jolnet and Killer.
+
+But then came the monster Martin Luther King Day Crash of January 15, 1990.
+
+A flat three days later, on January 18, four agents showed up
+at Knight Lightning's fraternity house. One was Timothy Foley,
+the second Barbara Golden, both of them Secret Service agents
+from the Chicago office. Also along was a University of Missouri
+security officer, and Reed Newlin, a security man from Southwestern Bell,
+the RBOC having jurisdiction over Missouri.
+
+Foley accused Knight Lightning of causing the nationwide crash
+of the phone system.
+
+Knight Lightning was aghast at this allegation. On the face of it,
+the suspicion was not entirely implausible--though Knight Lightning
+knew that he himself hadn't done it. Plenty of hot-dog hackers
+had bragged that they could crash the phone system, however.
+"Shadowhawk," for instance, the Chicago hacker whom William Cook
+had recently put in jail, had several times boasted on boards
+that he could "shut down AT&T's public switched network."
+
+And now this event, or something that looked just like it,
+had actually taken place. The Crash had lit a fire under
+the Chicago Task Force. And the former fence-sitters at
+Bellcore and AT&T were now ready to roll. The consensus
+among telco security--already horrified by the skill of
+the BellSouth intruders --was that the digital underground
+was out of hand. LoD and Phrack must go. And in publishing
+Prophet's E911 Document, Phrack had provided law enforcement
+with what appeared to be a powerful legal weapon.
+
+Foley confronted Knight Lightning about the E911 Document.
+
+Knight Lightning was cowed. He immediately began "cooperating fully"
+in the usual tradition of the digital underground.
+
+He gave Foley a complete run of Phrack, printed out in a set
+of three-ring binders. He handed over his electronic mailing list
+of Phrack subscribers. Knight Lightning was grilled for four hours
+by Foley and his cohorts. Knight Lightning admitted that Prophet
+had passed him the E911 Document, and he admitted that he had known
+it was stolen booty from a hacker raid on a telephone company.
+Knight Lightning signed a statement to this effect, and agreed,
+in writing, to cooperate with investigators.
+
+Next day--January 19, 1990, a Friday --the Secret Service returned
+with a search warrant, and thoroughly searched Knight Lightning's
+upstairs room in the fraternity house. They took all his floppy disks,
+though, interestingly, they left Knight Lightning in possession
+of both his computer and his modem. (The computer had no hard disk,
+and in Foley's judgement was not a store of evidence.) But this was a
+very minor bright spot among Knight Lightning's rapidly multiplying troubles.
+By this time, Knight Lightning was in plenty of hot water, not only with
+federal police, prosecutors, telco investigators, and university security,
+but with the elders of his own campus fraternity, who were outraged
+to think that they had been unwittingly harboring a federal computer-criminal.
+
+On Monday, Knight Lightning was summoned to Chicago, where he was
+further grilled by Foley and USSS veteran agent Barbara Golden, this time
+with an attorney present. And on Tuesday, he was formally indicted
+by a federal grand jury.
+
+The trial of Knight Lightning, which occurred on July 24-27, 1990,
+was the crucial show-trial of the Hacker Crackdown. We will examine
+the trial at some length in Part Four of this book.
+
+In the meantime, we must continue our dogged pursuit of the E911 Document.
+
+It must have been clear by January 1990 that the E911 Document,
+in the form Phrack had published it back in February 1989,
+had gone off at the speed of light in at least a hundred
+and fifty different directions. To attempt to put this
+electronic genie back in the bottle was flatly impossible.
+
+And yet, the E911 Document was STILL stolen property,
+formally and legally speaking. Any electronic transference
+of this document, by anyone unauthorized to have it,
+could be interpreted as an act of wire fraud. Interstate
+transfer of stolen property, including electronic property,
+was a federal crime.
+
+The Chicago Computer Fraud and Abuse Task Force had been assured
+that the E911 Document was worth a hefty sum of money. In fact,
+they had a precise estimate of its worth from BellSouth security personnel:
+$79,449. A sum of this scale seemed to warrant vigorous prosecution.
+Even if the damage could not be undone, at least this large sum
+offered a good legal pretext for stern punishment of the thieves.
+It seemed likely to impress judges and juries. And it could be used
+in court to mop up the Legion of Doom.
+
+The Atlanta crowd was already in the bag, by the time
+the Chicago Task Force had gotten around to Phrack.
+But the Legion was a hydra-headed thing. In late 89,
+a brand-new Legion of Doom board, "Phoenix Project,"
+had gone up in Austin, Texas. Phoenix Project was sysoped
+by no less a man than the Mentor himself, ably assisted by
+University of Texas student and hardened Doomster "Erik Bloodaxe."
+
+As we have seen from his Phrack manifesto, the Mentor was a hacker
+zealot who regarded computer intrusion as something close to a moral duty.
+Phoenix Project was an ambitious effort, intended to revive the digital
+underground to what Mentor considered the full flower of the early 80s.
+The Phoenix board would also boldly bring elite hackers face-to-face
+with the telco "opposition." On "Phoenix," America's cleverest hackers
+would supposedly shame the telco squareheads out of their stick-in-the-mud
+attitudes, and perhaps convince them that the Legion of Doom elite were really
+an all-right crew. The premiere of "Phoenix Project" was heavily trumpeted
+by Phrack,and "Phoenix Project" carried a complete run of Phrack issues,
+including the E911 Document as Phrack had published it.
+
+Phoenix Project was only one of many--possibly hundreds--of nodes and boards
+all over America that were in guilty possession of the E911 Document.
+But Phoenix was an outright, unashamed Legion of Doom board.
+Under Mentor's guidance, it was flaunting itself in the face
+of telco security personnel. Worse yet, it was actively trying
+to WIN THEM OVER as sympathizers for the digital underground elite.
+"Phoenix" had no cards or codes on it. Its hacker elite considered
+Phoenix at least technically legal. But Phoenix was a corrupting influence,
+where hacker anarchy was eating away like digital acid at the underbelly
+of corporate propriety.
+
+The Chicago Computer Fraud and Abuse Task Force now prepared
+to descend upon Austin, Texas.
+
+Oddly, not one but TWO trails of the Task Force's investigation led
+toward Austin. The city of Austin, like Atlanta, had made itself
+a bulwark of the Sunbelt's Information Age, with a strong university
+research presence, and a number of cutting-edge electronics companies,
+including Motorola, Dell, CompuAdd, IBM, Sematech and MCC.
+
+Where computing machinery went, hackers generally followed.
+Austin boasted not only "Phoenix Project," currently LoD's
+most flagrant underground board, but a number of UNIX nodes.
+
+One of these nodes was "Elephant," run by a UNIX consultant
+named Robert Izenberg. Izenberg, in search of a relaxed Southern
+lifestyle and a lowered cost-of-living, had recently migrated
+to Austin from New Jersey. In New Jersey, Izenberg had worked
+for an independent contracting company, programming UNIX code for
+AT&T itself. "Terminus" had been a frequent user on Izenberg's
+privately owned Elephant node.
+
+Having interviewed Terminus and examined the records on Netsys,
+the Chicago Task Force were now convinced that they had discovered
+an underground gang of UNIX software pirates, who were demonstrably
+guilty of interstate trafficking in illicitly copied AT&T source code.
+Izenberg was swept into the dragnet around Terminus, the self-proclaimed
+ultimate UNIX hacker.
+
+Izenberg, in Austin, had settled down into a UNIX job
+with a Texan branch of IBM. Izenberg was no longer
+working as a contractor for AT&T, but he had friends
+in New Jersey, and he still logged on to AT&T UNIX
+computers back in New Jersey, more or less whenever
+it pleased him. Izenberg's activities appeared highly
+suspicious to the Task Force. Izenberg might well be
+breaking into AT&T computers, swiping AT&T software,
+and passing it to Terminus and other possible confederates,
+through the UNIX node network. And this data was worth,
+not merely $79,499, but hundreds of thousands of dollars!
+
+On February 21, 1990, Robert Izenberg arrived home
+from work at IBM to find that all the computers
+had mysteriously vanished from his Austin apartment.
+Naturally he assumed that he had been robbed.
+His "Elephant" node, his other machines, his notebooks,
+his disks, his tapes, all gone! However, nothing much
+else seemed disturbed--the place had not been ransacked.
+The puzzle becaming much stranger some five minutes later.
+Austin U. S. Secret Service Agent Al Soliz, accompanied by
+University of Texas campus-security officer Larry Coutorie
+and the ubiquitous Tim Foley, made their appearance at Izenberg's door.
+They were in plain clothes: slacks, polo shirts. They came in,
+and Tim Foley accused Izenberg of belonging to the Legion of Doom.
+
+Izenberg told them that he had never heard of the "Legion of Doom."
+And what about a certain stolen E911 Document, that posed a direct
+threat to the police emergency lines? Izenberg claimed that he'd
+never heard of that, either.
+
+His interrogators found this difficult to believe.
+Didn't he know Terminus?
+
+Who?
+
+They gave him Terminus's real name. Oh yes, said Izenberg.
+He knew THAT guy all right--he was leading discussions
+on the Internet about AT&T computers, especially the AT&T 3B2.
+
+AT&T had thrust this machine into the marketplace,
+but, like many of AT&T's ambitious attempts to enter
+the computing arena, the 3B2 project had something less
+than a glittering success. Izenberg himself had been
+a contractor for the division of AT&T that supported the 3B2.
+The entire division had been shut down.
+
+Nowadays, the cheapest and quickest way to get help with this
+fractious piece of machinery was to join one of Terminus's
+discussion groups on the Internet, where friendly and knowledgeable
+hackers would help you for free. Naturally the remarks within this
+group were less than flattering about the Death Star. . .was
+THAT the problem?
+
+Foley told Izenberg that Terminus had been acquiring hot software
+through his, Izenberg's, machine.
+
+Izenberg shrugged this off. A good eight megabytes of data flowed
+through his UUCP site every day. UUCP nodes spewed data like fire hoses.
+Elephant had been directly linked to Netsys--not surprising, since Terminus
+was a 3B2 expert and Izenberg had been a 3B2 contractor.
+Izenberg was also linked to "attctc" and the University of Texas.
+Terminus was a well-known UNIX expert, and might have been up to
+all manner of hijinks on Elephant. Nothing Izenberg could do about that.
+That was physically impossible. Needle in a haystack.
+
+In a four-hour grilling, Foley urged Izenberg to come clean
+and admit that he was in conspiracy with Terminus,
+and a member of the Legion of Doom.
+
+Izenberg denied this. He was no weirdo teenage hacker--
+he was thirty-two years old, and didn't even have a "handle."
+Izenberg was a former TV technician and electronics specialist
+who had drifted into UNIX consulting as a full-grown adult.
+Izenberg had never met Terminus, physically. He'd once bought
+a cheap high-speed modem from him, though.
+
+Foley told him that this modem (a Telenet T2500 which ran at 19.2 kilobaud,
+and which had just gone out Izenberg's door in Secret Service custody)
+was likely hot property. Izenberg was taken aback to hear this; but then
+again, most of Izenberg's equipment, like that of most freelance professionals
+in the industry, was discounted, passed hand-to-hand through various kinds
+of barter and gray-market. There was no proof that the modem was stolen,
+and even if it were, Izenberg hardly saw how that gave them the right
+to take every electronic item in his house.
+
+Still, if the United States Secret Service figured they needed
+his computer for national security reasons--or whatever--
+then Izenberg would not kick. He figured he would somehow
+make the sacrifice of his twenty thousand dollars' worth
+of professional equipment, in the spirit of full cooperation
+and good citizenship.
+
+Robert Izenberg was not arrested. Izenberg was not charged with any crime.
+His UUCP node--full of some 140 megabytes of the files, mail, and data
+of himself and his dozen or so entirely innocent users--went out the door
+as "evidence." Along with the disks and tapes, Izenberg had lost about
+800 megabytes of data.
+
+Six months would pass before Izenberg decided to phone the Secret Service
+and ask how the case was going. That was the first time that Robert Izenberg
+would ever hear the name of William Cook. As of January 1992, a full
+two years after the seizure, Izenberg, still not charged with any crime,
+would be struggling through the morass of the courts, in hope of recovering
+his thousands of dollars' worth of seized equipment.
+
+In the meantime, the Izenberg case received absolutely no press coverage.
+The Secret Service had walked into an Austin home, removed a UNIX bulletin-
+board system, and met with no operational difficulties whatsoever.
+
+Except that word of a crackdown had percolated through the Legion of Doom.
+"The Mentor" voluntarily shut down "The Phoenix Project." It seemed a pity,
+especially as telco security employees had, in fact, shown up on Phoenix,
+just as he had hoped--along with the usual motley crowd of LoD heavies,
+hangers-on, phreaks, hackers and wannabes. There was "Sandy" Sandquist from
+US SPRINT security, and some guy named Henry Kluepfel, from Bellcore itself!
+Kluepfel had been trading friendly banter with hackers on Phoenix since
+January 30th (two weeks after the Martin Luther King Day Crash).
+The presence of such a stellar telco official seemed quite the coup
+for Phoenix Project.
+
+Still, Mentor could judge the climate. Atlanta in ruins,
+Phrack in deep trouble, something weird going on with UNIX nodes--
+discretion was advisable. Phoenix Project went off-line.
+
+Kluepfel, of course, had been monitoring this LoD bulletin
+board for his own purposes--and those of the Chicago unit.
+As far back as June 1987, Kluepfel had logged on to a Texas
+underground board called "Phreak Klass 2600." There he'd
+discovered an Chicago youngster named "Shadowhawk,"
+strutting and boasting about rifling AT&T computer files,
+and bragging of his ambitions to riddle AT&T's Bellcore
+computers with trojan horse programs. Kluepfel had passed
+the news to Cook in Chicago, Shadowhawk's computers
+had gone out the door in Secret Service custody,
+and Shadowhawk himself had gone to jail.
+
+Now it was Phoenix Project's turn. Phoenix Project postured
+about "legality" and "merely intellectual interest," but it reeked
+of the underground. It had Phrack on it. It had the E911 Document.
+It had a lot of dicey talk about breaking into systems, including some
+bold and reckless stuff about a supposed "decryption service" that Mentor
+and friends were planning to run, to help crack encrypted passwords off
+of hacked systems.
+
+Mentor was an adult. There was a bulletin board at his place of work,
+as well. Kleupfel logged onto this board, too, and discovered it to be
+called "Illuminati." It was run by some company called Steve Jackson Games.
+
+On March 1, 1990, the Austin crackdown went into high gear.
+
+On the morning of March 1--a Thursday--21-year-old University of Texas
+student "Erik Bloodaxe," co-sysop of Phoenix Project and an avowed member
+of the Legion of Doom, was wakened by a police revolver levelled at his head.
+
+Bloodaxe watched, jittery, as Secret Service agents
+appropriated his 300 baud terminal and, rifling his files,
+discovered his treasured source-code for Robert Morris's
+notorious Internet Worm. But Bloodaxe, a wily operator,
+had suspected that something of the like might be coming.
+All his best equipment had been hidden away elsewhere.
+The raiders took everything electronic, however,
+including his telephone. They were stymied by his
+hefty arcade-style Pac-Man game, and left it in place,
+as it was simply too heavy to move.
+
+Bloodaxe was not arrested. He was not charged with any crime.
+A good two years later, the police still had what they had
+taken from him, however.
+
+The Mentor was less wary. The dawn raid rousted him and his wife
+from bed in their underwear, and six Secret Service agents,
+accompanied by an Austin policeman and Henry Kluepfel himself,
+made a rich haul. Off went the works, into the agents' white
+Chevrolet minivan: an IBM PC-AT clone with 4 meg of RAM and
+a 120-meg hard disk; a Hewlett-Packard LaserJet II printer;
+a completely legitimate and highly expensive SCO-Xenix 286
+operating system; Pagemaker disks and documentation;
+and the Microsoft Word word-processing program. Mentor's wife
+had her incomplete academic thesis stored on the hard-disk;
+that went, too, and so did the couple's telephone. As of two years later,
+all this property remained in police custody.
+
+Mentor remained under guard in his apartment as agents prepared
+to raid Steve Jackson Games. The fact that this was a business
+headquarters and not a private residence did not deter the agents.
+It was still very early; no one was at work yet. The agents prepared
+to break down the door, but Mentor, eavesdropping on the Secret Service
+walkie-talkie traffic, begged them not to do it, and offered his key
+to the building.
+
+The exact details of the next events are unclear. The agents
+would not let anyone else into the building. Their search warrant,
+when produced, was unsigned. Apparently they breakfasted from the local
+"Whataburger," as the litter from hamburgers was later found inside.
+They also extensively sampled a bag of jellybeans kept by an SJG employee.
+Someone tore a "Dukakis for President" sticker from the wall.
+
+SJG employees, diligently showing up for the day's work, were met
+at the door and briefly questioned by U.S. Secret Service agents.
+The employees watched in astonishment as agents wielding crowbars
+and screwdrivers emerged with captive machines. They attacked
+outdoor storage units with boltcutters. The agents wore
+blue nylon windbreakers with "SECRET SERVICE" stencilled
+across the back, with running-shoes and jeans.
+
+Jackson's company lost three computers, several hard-disks,
+hundred of floppy disks, two monitors, three modems,
+a laser printer, various powercords, cables, and adapters
+(and, oddly, a small bag of screws, bolts and nuts).
+The seizure of Illuminati BBS deprived SJG of all the programs,
+text files, and private e-mail on the board. The loss of two other
+SJG computers was a severe blow as well, since it caused the loss
+of electronically stored contracts, financial projections,
+address directories, mailing lists, personnel files,
+business correspondence, and, not least, the drafts
+of forthcoming games and gaming books.
+
+No one at Steve Jackson Games was arrested. No one was accused
+of any crime. No charges were filed. Everything appropriated
+was officially kept as "evidence" of crimes never specified.
+
+After the Phrack show-trial, the Steve Jackson Games scandal
+was the most bizarre and aggravating incident of the Hacker
+Crackdown of 1990. This raid by the Chicago Task Force
+on a science-fiction gaming publisher was to rouse a
+swarming host of civil liberties issues, and gave rise
+to an enduring controversy that was still re-complicating itself,
+and growing in the scope of its implications, a full two years later.
+
+The pursuit of the E911 Document stopped with the Steve Jackson Games raid.
+As we have seen, there were hundreds, perhaps thousands of computer users
+in America with the E911 Document in their possession. Theoretically,
+Chicago had a perfect legal right to raid any of these people,
+and could have legally seized the machines of anybody who subscribed to Phrack.
+However, there was no copy of the E911 Document on Jackson's Illuminati board.
+And there the Chicago raiders stopped dead; they have not raided anyone since.
+
+It might be assumed that Rich Andrews and Charlie Boykin, who had brought
+the E911 Document to the attention of telco security, might be spared
+any official suspicion. But as we have seen, the willingness to
+"cooperate fully" offers little, if any, assurance against federal
+anti-hacker prosecution.
+
+Richard Andrews found himself in deep trouble, thanks to the E911 Document.
+Andrews lived in Illinois, the native stomping grounds of the Chicago
+Task Force. On February 3 and 6, both his home and his place of work
+were raided by USSS. His machines went out the door, too, and he was
+grilled at length (though not arrested). Andrews proved to be in
+purportedly guilty possession of: UNIX SVR 3.2; UNIX SVR 3.1; UUCP;
+PMON; WWB; IWB; DWB; NROFF; KORN SHELL '88; C++; and QUEST,
+among other items. Andrews had received this proprietary code--
+which AT&T officially valued at well over $250,000--through the
+UNIX network, much of it supplied to him as a personal favor by Terminus.
+Perhaps worse yet, Andrews admitted to returning the favor, by passing
+Terminus a copy of AT&T proprietary STARLAN source code.
+
+Even Charles Boykin, himself an AT&T employee, entered some very hot water.
+By 1990, he'd almost forgotten about the E911 problem he'd reported in
+September 88; in fact, since that date, he'd passed two more security alerts
+to Jerry Dalton, concerning matters that Boykin considered far worse than
+the E911 Document.
+
+But by 1990, year of the crackdown, AT&T Corporate Information Security
+was fed up with "Killer." This machine offered no direct income to AT&T,
+and was providing aid and comfort to a cloud of suspicious yokels
+from outside the company, some of them actively malicious toward AT&T,
+its property, and its corporate interests. Whatever goodwill and publicity
+had been won among Killer's 1,500 devoted users was considered no longer
+worth the security risk. On February 20, 1990, Jerry Dalton arrived in
+Dallas and simply unplugged the phone jacks, to the puzzled alarm
+of Killer's many Texan users. Killer went permanently off-line,
+with the loss of vast archives of programs and huge quantities
+of electronic mail; it was never restored to service. AT&T showed
+no particular regard for the "property" of these 1,500 people.
+Whatever "property" the users had been storing on AT&T's computer
+simply vanished completely.
+
+Boykin, who had himself reported the E911 problem,
+now found himself under a cloud of suspicion. In a weird
+private-security replay of the Secret Service seizures,
+Boykin's own home was visited by AT&T Security and his
+own machines were carried out the door.
+
+However, there were marked special features in the Boykin case.
+Boykin's disks and his personal computers were swiftly examined
+by his corporate employers and returned politely in just two days--
+(unlike Secret Service seizures, which commonly take months or years).
+Boykin was not charged with any crime or wrongdoing, and he kept his job
+with AT&T (though he did retire from AT&T in September 1991,
+at the age of 52).
+
+It's interesting to note that the US Secret Service somehow failed
+to seize Boykin's "Killer" node and carry AT&T's own computer out the door.
+Nor did they raid Boykin's home. They seemed perfectly willing to take the
+word of AT&T Security that AT&T's employee, and AT&T's "Killer" node,
+were free of hacker contraband and on the up-and-up.
+
+It's digital water-under-the-bridge at this point, as Killer's
+3,200 megabytes of Texan electronic community were erased in 1990,
+and "Killer" itself was shipped out of the state.
+
+But the experiences of Andrews and Boykin, and the users of their systems,
+remained side issues. They did not begin to assume the social, political,
+and legal importance that gathered, slowly but inexorably, around the issue
+of the raid on Steve Jackson Games.
+
+#
+
+We must now turn our attention to Steve Jackson Games itself,
+and explain what SJG was, what it really did, and how it had
+managed to attract this particularly odd and virulent kind of trouble.
+The reader may recall that this is not the first but the second time
+that the company has appeared in this narrative; a Steve Jackson game
+called GURPS was a favorite pastime of Atlanta hacker Urvile,
+and Urvile's science-fictional gaming notes had been mixed up
+promiscuously with notes about his actual computer intrusions.
+
+First, Steve Jackson Games, Inc., was NOT a publisher of "computer games."
+SJG published "simulation games," parlor games that were played on paper,
+with pencils, and dice, and printed guidebooks full of rules and
+statistics tables. There were no computers involved in the games themselves.
+When you bought a Steve Jackson Game, you did not receive any software disks.
+What you got was a plastic bag with some cardboard game tokens,
+maybe a few maps or a deck of cards. Most of their products were books.
+
+However, computers WERE deeply involved in the Steve Jackson Games business.
+Like almost all modern publishers, Steve Jackson and his fifteen employees
+used computers to write text, to keep accounts, and to run the business
+generally. They also used a computer to run their official bulletin board
+system for Steve Jackson Games, a board called Illuminati. On Illuminati,
+simulation gamers who happened to own computers and modems could associate,
+trade mail, debate the theory and practice of gaming, and keep up with the
+company's news and its product announcements.
+
+Illuminati was a modestly popular board, run on a small computer
+with limited storage, only one phone-line, and no ties to large-scale
+computer networks. It did, however, have hundreds of users,
+many of them dedicated gamers willing to call from out-of-state.
+
+Illuminati was NOT an "underground" board. It did not feature hints
+on computer intrusion, or "anarchy files," or illicitly posted
+credit card numbers, or long-distance access codes.
+Some of Illuminati's users, however, were members of the Legion of Doom.
+And so was one of Steve Jackson's senior employees--the Mentor.
+The Mentor wrote for Phrack, and also ran an underground board,
+Phoenix Project--but the Mentor was not a computer professional.
+The Mentor was the managing editor of Steve Jackson Games and
+a professional game designer by trade. These LoD members did not
+use Illuminati to help their HACKING activities. They used it to
+help their GAME-PLAYING activities--and they were even more dedicated
+to simulation gaming than they were to hacking.
+
+"Illuminati" got its name from a card-game that Steve Jackson himself,
+the company's founder and sole owner, had invented. This multi-player
+card-game was one of Mr Jackson's best-known, most successful,
+most technically innovative products. "Illuminati" was a game
+of paranoiac conspiracy in which various antisocial cults warred
+covertly to dominate the world. "Illuminati" was hilarious,
+and great fun to play, involving flying saucers, the CIA, the KGB,
+the phone companies, the Ku Klux Klan, the South American Nazis,
+the cocaine cartels, the Boy Scouts, and dozens of other splinter groups
+from the twisted depths of Mr. Jackson's professionally fervid imagination.
+For the uninitiated, any public discussion of the "Illuminati" card-game
+sounded, by turns, utterly menacing or completely insane.
+
+And then there was SJG's "Car Wars," in which souped-up armored hot-rods
+with rocket-launchers and heavy machine-guns did battle on the American
+highways of the future. The lively Car Wars discussion on the Illuminati
+board featured many meticulous, painstaking discussions of the effects
+of grenades, land-mines, flamethrowers and napalm. It sounded like
+hacker anarchy files run amuck.
+
+Mr Jackson and his co-workers earned their daily bread by supplying people
+with make-believe adventures and weird ideas. The more far-out, the better.
+
+Simulation gaming is an unusual pastime, but gamers have not
+generally had to beg the permission of the Secret Service to exist.
+Wargames and role-playing adventures are an old and honored pastime,
+much favored by professional military strategists. Once little-known,
+these games are now played by hundreds of thousands of enthusiasts
+throughout North America, Europe and Japan. Gaming-books, once restricted
+to hobby outlets, now commonly appear in chain-stores like B. Dalton's
+and Waldenbooks, and sell vigorously.
+
+Steve Jackson Games, Inc., of Austin, Texas, was a games company
+of the middle rank. In 1989, SJG grossed about a million dollars.
+Jackson himself had a good reputation in his industry as a talented
+and innovative designer of rather unconventional games, but his company
+was something less than a titan of the field--certainly not like the
+multimillion-dollar TSR Inc., or Britain's gigantic "Games Workshop."
+SJG's Austin headquarters was a modest two-story brick office-suite,
+cluttered with phones, photocopiers, fax machines and computers.
+It bustled with semi-organized activity and was littered with
+glossy promotional brochures and dog-eared science-fiction novels.
+Attached to the offices was a large tin-roofed warehouse piled twenty feet
+high with cardboard boxes of games and books. Despite the weird imaginings
+that went on within it, the SJG headquarters was quite a quotidian,
+everyday sort of place. It looked like what it was: a publishers' digs.
+
+Both "Car Wars" and "Illuminati" were well-known, popular games.
+But the mainstay of the Jackson organization was their Generic Universal
+Role-Playing System, "G.U.R.P.S." The GURPS system was considered solid
+and well-designed, an asset for players. But perhaps the most popular
+feature of the GURPS system was that it allowed gaming-masters to design
+scenarios that closely resembled well-known books, movies, and other works
+of fantasy. Jackson had licensed and adapted works from many science fiction
+and fantasy authors. There was GURPS Conan, GURPS Riverworld,
+GURPS Horseclans, GURPS Witch World, names eminently familiar
+to science-fiction readers. And there was GURPS Special Ops,
+from the world of espionage fantasy and unconventional warfare.
+
+And then there was GURPS Cyberpunk.
+
+"Cyberpunk" was a term given to certain science fiction writers
+who had entered the genre in the 1980s. "Cyberpunk," as the label implies,
+had two general distinguishing features. First, its writers had a compelling
+interest in information technology, an interest closely akin
+to science fiction's earlier fascination with space travel.
+And second, these writers were "punks," with all the
+distinguishing features that that implies: Bohemian artiness,
+youth run wild, an air of deliberate rebellion, funny clothes and hair,
+odd politics, a fondness for abrasive rock and roll; in a word, trouble.
+
+The "cyberpunk" SF writers were a small group of mostly college-educated
+white middle-class litterateurs, scattered through the US and Canada.
+Only one, Rudy Rucker, a professor of computer science in Silicon Valley,
+could rank with even the humblest computer hacker. But, except for
+Professor Rucker, the "cyberpunk" authors were not programmers
+or hardware experts; they considered themselves artists
+(as, indeed, did Professor Rucker). However, these writers
+all owned computers, and took an intense and public interest
+in the social ramifications of the information industry.
+
+The cyberpunks had a strong following among the global generation
+that had grown up in a world of computers, multinational networks,
+and cable television. Their outlook was considered somewhat morbid,
+cynical, and dark, but then again, so was the outlook of their
+generational peers. As that generation matured and increased
+in strength and influence, so did the cyberpunks.
+As science-fiction writers went, they were doing
+fairly well for themselves. By the late 1980s,
+their work had attracted attention from gaming companies,
+including Steve Jackson Games, which was planning a cyberpunk
+simulation for the flourishing GURPS gaming-system.
+
+The time seemed ripe for such a product, which had already been proven
+in the marketplace. The first games- company out of the gate,
+with a product boldly called "Cyberpunk" in defiance of possible
+infringement-of-copyright suits, had been an upstart group called
+R. Talsorian. Talsorian's Cyberpunk was a fairly decent game,
+but the mechanics of the simulation system left a lot to be desired.
+Commercially, however, the game did very well.
+
+The next cyberpunk game had been the even more successful Shadowrun
+by FASA Corporation. The mechanics of this game were fine, but the
+scenario was rendered moronic by sappy fantasy elements like elves,
+trolls, wizards, and dragons--all highly ideologically-incorrect,
+according to the hard-edged, high-tech standards of cyberpunk science fiction.
+
+Other game designers were champing at the bit. Prominent among them
+was the Mentor, a gentleman who, like most of his friends in the
+Legion of Doom, was quite the cyberpunk devotee. Mentor reasoned
+that the time had come for a REAL cyberpunk gaming-book--one that the
+princes of computer-mischief in the Legion of Doom could play without
+laughing themselves sick. This book, GURPS Cyberpunk, would reek
+of culturally on-line authenticity.
+
+Mentor was particularly well-qualified for this task.
+Naturally, he knew far more about computer-intrusion
+and digital skullduggery than any previously published
+cyberpunk author. Not only that, but he was good at his work.
+A vivid imagination, combined with an instinctive feeling
+for the working of systems and, especially, the loopholes
+within them, are excellent qualities for a professional game designer.
+
+By March 1st, GURPS Cyberpunk was almost complete, ready to print and ship.
+Steve Jackson expected vigorous sales for this item, which, he hoped,
+would keep the company financially afloat for several months.
+GURPS Cyberpunk, like the other GURPS "modules," was not a "game"
+like a Monopoly set, but a BOOK: a bound paperback book the size
+of a glossy magazine, with a slick color cover, and pages full of text,
+illustrations, tables and footnotes. It was advertised as a game,
+and was used as an aid to game-playing, but it was a book,
+with an ISBN number, published in Texas, copyrighted,
+and sold in bookstores.
+
+And now, that book, stored on a computer, had gone out the door
+in the custody of the Secret Service.
+
+The day after the raid, Steve Jackson visited the local Secret Service
+headquarters with a lawyer in tow. There he confronted Tim Foley
+(still in Austin at that time) and demanded his book back. But there
+was trouble. GURPS Cyberpunk, alleged a Secret Service agent to astonished
+businessman Steve Jackson, was "a manual for computer crime."
+
+"It's science fiction," Jackson said.
+
+"No, this is real."
+
+This statement was repeated several times, by several agents.
+Jackson's ominously accurate game had passed from pure,
+obscure, small-scale fantasy into the impure, highly publicized,
+large-scale fantasy of the Hacker Crackdown.
+
+No mention was made of the real reason for the search.
+According to their search warrant, the raiders had expected
+to find the E911 Document stored on Jackson's bulletin board system.
+But that warrant was sealed; a procedure that most law enforcement agencies
+will use only when lives are demonstrably in danger. The raiders'
+true motives were not discovered until the Jackson search-warrant
+was unsealed by his lawyers, many months later. The Secret Service,
+and the Chicago Computer Fraud and Abuse Task Force,
+said absolutely nothing to Steve Jackson about any threat
+to the police 911 System. They said nothing about the Atlanta Three,
+nothing about Phrack or Knight Lightning, nothing about Terminus.
+
+Jackson was left to believe that his computers had been seized because
+he intended to publish a science fiction book that law enforcement
+considered too dangerous to see print.
+
+This misconception was repeated again and again, for months,
+to an ever-widening public audience. It was not the truth of the case;
+but as months passed, and this misconception was publicly printed again
+and again, it became one of the few publicly known "facts" about
+the mysterious Hacker Crackdown. The Secret Service had seized a computer
+to stop the publication of a cyberpunk science fiction book.
+
+The second section of this book, "The Digital Underground,"
+is almost finished now. We have become acquainted with all
+the major figures of this case who actually belong to the
+underground milieu of computer intrusion. We have some idea
+of their history, their motives, their general modus operandi.
+We now know, I hope, who they are, where they came from,
+and more or less what they want. In the next section of this book,
+"Law and Order," we will leave this milieu and directly enter the
+world of America's computer-crime police.
+
+At this point, however, I have another figure to introduce: myself.
+
+My name is Bruce Sterling. I live in Austin, Texas, where I am
+a science fiction writer by trade: specifically, a CYBERPUNK
+science fiction writer.
+
+Like my "cyberpunk" colleagues in the U.S. and Canada,
+I've never been entirely happy with this literary label--
+especially after it became a synonym for computer criminal.
+But I did once edit a book of stories by my colleagues,
+called Mirrorshades: the Cyberpunk Anthology, and I've
+long been a writer of literary-critical cyberpunk manifestos.
+I am not a "hacker" of any description, though I do have readers
+in the digital underground.
+
+When the Steve Jackson Games seizure occurred, I naturally took
+an intense interest. If "cyberpunk" books were being banned
+by federal police in my own home town, I reasonably wondered
+whether I myself might be next. Would my computer be seized
+by the Secret Service? At the time, I was in possession
+of an aging Apple IIe without so much as a hard disk.
+If I were to be raided as an author of computer-crime manuals,
+the loss of my feeble word-processor would likely provoke more
+snickers than sympathy.
+
+I'd known Steve Jackson for many years. We knew
+one another as colleagues, for we frequented
+the same local science-fiction conventions.
+I'd played Jackson games, and recognized his cleverness;
+but he certainly had never struck me as a potential mastermind
+of computer crime.
+
+I also knew a little about computer bulletin-board systems.
+In the mid-1980s I had taken an active role in an Austin board
+called "SMOF-BBS," one of the first boards dedicated to science fiction.
+I had a modem, and on occasion I'd logged on to Illuminati,
+which always looked entertainly wacky, but certainly harmless enough.
+
+At the time of the Jackson seizure, I had no experience
+whatsoever with underground boards. But I knew that no one
+on Illuminati talked about breaking into systems illegally,
+or about robbing phone companies. Illuminati didn't even
+offer pirated computer games. Steve Jackson, like many creative artists,
+was markedly touchy about theft of intellectual property.
+
+It seemed to me that Jackson was either seriously suspected
+of some crime--in which case, he would be charged soon,
+and would have his day in court--or else he was innocent,
+in which case the Secret Service would quickly return his equipment,
+and everyone would have a good laugh. I rather expected the good laugh.
+The situation was not without its comic side. The raid, known
+as the "Cyberpunk Bust" in the science fiction community,
+was winning a great deal of free national publicity both
+for Jackson himself and the "cyberpunk" science fiction
+writers generally.
+
+Besides, science fiction people are used to being misinterpreted.
+Science fiction is a colorful, disreputable, slipshod occupation,
+full of unlikely oddballs, which, of course, is why we like it.
+Weirdness can be an occupational hazard in our field. People who
+wear Halloween costumes are sometimes mistaken for monsters.
+
+Once upon a time--back in 1939, in New York City--
+science fiction and the U.S. Secret Service collided in
+a comic case of mistaken identity. This weird incident
+involved a literary group quite famous in science fiction,
+known as "the Futurians," whose membership included
+such future genre greats as Isaac Asimov, Frederik Pohl,
+and Damon Knight. The Futurians were every bit as
+offbeat and wacky as any of their spiritual descendants,
+including the cyberpunks, and were given to communal living,
+spontaneous group renditions of light opera, and midnight fencing
+exhibitions on the lawn. The Futurians didn't have bulletin
+board systems, but they did have the technological equivalent
+in 1939--mimeographs and a private printing press. These were
+in steady use, producing a stream of science-fiction fan magazines,
+literary manifestos, and weird articles, which were picked up
+in ink-sticky bundles by a succession of strange, gangly,
+spotty young men in fedoras and overcoats.
+
+The neighbors grew alarmed at the antics of the Futurians
+and reported them to the Secret Service as suspected counterfeiters.
+In the winter of 1939, a squad of USSS agents with drawn guns burst into
+"Futurian House," prepared to confiscate the forged currency and illicit
+printing presses. There they discovered a slumbering science fiction fan
+named George Hahn, a guest of the Futurian commune who had just arrived
+in New York. George Hahn managed to explain himself and his group,
+and the Secret Service agents left the Futurians in peace henceforth.
+(Alas, Hahn died in 1991, just before I had discovered this astonishing
+historical parallel, and just before I could interview him for this book.)
+
+But the Jackson case did not come to a swift and comic end.
+No quick answers came his way, or mine; no swift reassurances
+that all was right in the digital world, that matters were well
+in hand after all. Quite the opposite. In my alternate role
+as a sometime pop-science journalist, I interviewed Jackson
+and his staff for an article in a British magazine.
+The strange details of the raid left me more concerned than ever.
+Without its computers, the company had been financially
+and operationally crippled. Half the SJG workforce,
+a group of entirely innocent people, had been sorrowfully fired,
+deprived of their livelihoods by the seizure. It began to dawn on me
+that authors--American writers--might well have their computers seized,
+under sealed warrants, without any criminal charge; and that,
+as Steve Jackson had discovered, there was no immediate recourse for this.
+This was no joke; this wasn't science fiction; this was real.
+
+I determined to put science fiction aside until I had discovered
+what had happened and where this trouble had come from.
+It was time to enter the purportedly real world of electronic
+free expression and computer crime. Hence, this book.
+Hence, the world of the telcos; and the world of the digital underground;
+and next, the world of the police.
+
+
+
+PART THREE: LAW AND ORDER
+
+
+Of the various anti-hacker activities of 1990, "Operation Sundevil"
+had by far the highest public profile. The sweeping, nationwide
+computer seizures of May 8, 1990 were unprecedented in scope and highly,
+if rather selectively, publicized.
+
+Unlike the efforts of the Chicago Computer Fraud and Abuse Task Force,
+"Operation Sundevil" was not intended to combat "hacking" in the sense
+of computer intrusion or sophisticated raids on telco switching stations.
+Nor did it have anything to do with hacker misdeeds with AT&T's software,
+or with Southern Bell's proprietary documents.
+
+Instead, "Operation Sundevil" was a crackdown on those traditional scourges
+of the digital underground: credit-card theft and telephone code abuse.
+The ambitious activities out of Chicago, and the somewhat lesser-known
+but vigorous anti-hacker actions of the New York State Police in 1990,
+were never a part of "Operation Sundevil" per se, which was based in Arizona.
+
+Nevertheless, after the spectacular May 8 raids, the public, misled by
+police secrecy, hacker panic, and a puzzled national press-corps,
+conflated all aspects of the nationwide crackdown in 1990 under
+the blanket term "Operation Sundevil." "Sundevil" is still the best-known
+synonym for the crackdown of 1990. But the Arizona organizers of "Sundevil"
+did not really deserve this reputation--any more, for instance, than all
+hackers deserve a reputation as "hackers."
+
+There was some justice in this confused perception, though.
+For one thing, the confusion was abetted by the Washington office
+of the Secret Service, who responded to Freedom of Information Act
+requests on "Operation Sundevil" by referring investigators
+to the publicly known cases of Knight Lightning and the Atlanta Three.
+And "Sundevil" was certainly the largest aspect of the Crackdown,
+the most deliberate and the best-organized. As a crackdown on electronic
+fraud, "Sundevil" lacked the frantic pace of the war on the Legion of Doom;
+on the contrary, Sundevil's targets were picked out with cool deliberation
+over an elaborate investigation lasting two full years.
+
+And once again the targets were bulletin board systems.
+
+Boards can be powerful aids to organized fraud. Underground boards carry
+lively, extensive, detailed, and often quite flagrant "discussions" of
+lawbreaking techniques and lawbreaking activities. "Discussing" crime
+in the abstract, or "discussing" the particulars of criminal cases,
+is not illegal--but there are stern state and federal laws against
+coldbloodedly conspiring in groups in order to commit crimes.
+
+In the eyes of police, people who actively conspire to break the law
+are not regarded as "clubs," "debating salons," "users' groups," or
+"free speech advocates." Rather, such people tend to find themselves
+formally indicted by prosecutors as "gangs," "racketeers," "corrupt
+organizations" and "organized crime figures."
+
+What's more, the illicit data contained on outlaw boards goes well beyond
+mere acts of speech and/or possible criminal conspiracy. As we have seen,
+it was common practice in the digital underground to post purloined telephone
+codes on boards, for any phreak or hacker who cared to abuse them. Is posting
+digital booty of this sort supposed to be protected by the First Amendment?
+Hardly--though the issue, like most issues in cyberspace, is not entirely
+resolved. Some theorists argue that to merely RECITE a number publicly
+is not illegal--only its USE is illegal. But anti-hacker police point out
+that magazines and newspapers (more traditional forms of free expression)
+never publish stolen telephone codes (even though this might well
+raise their circulation).
+
+Stolen credit card numbers, being riskier and more valuable,
+were less often publicly posted on boards--but there is no question
+that some underground boards carried "carding" traffic,
+generally exchanged through private mail.
+
+Underground boards also carried handy programs for "scanning" telephone
+codes and raiding credit card companies, as well as the usual obnoxious
+galaxy of pirated software, cracked passwords, blue-box schematics,
+intrusion manuals, anarchy files, porn files, and so forth.
+
+But besides their nuisance potential for the spread of illicit knowledge,
+bulletin boards have another vitally interesting aspect for the
+professional investigator. Bulletin boards are cram-full of EVIDENCE.
+All that busy trading of electronic mail, all those hacker boasts,
+brags and struts, even the stolen codes and cards, can be neat,
+electronic, real-time recordings of criminal activity.
+As an investigator, when you seize a pirate board, you have
+scored a coup as effective as tapping phones or intercepting mail.
+However, you have not actually tapped a phone or intercepted a letter.
+The rules of evidence regarding phone-taps and mail interceptions are old,
+stern and well-understood by police, prosecutors and defense attorneys alike.
+The rules of evidence regarding boards are new, waffling, and understood
+by nobody at all.
+
+Sundevil was the largest crackdown on boards in world history.
+On May 7, 8, and 9, 1990, about forty-two computer systems were seized.
+Of those forty-two computers, about twenty-five actually were running boards.
+(The vagueness of this estimate is attributable to the vagueness of
+(a) what a "computer system" is, and (b) what it actually means to
+"run a board" with one--or with two computers, or with three.)
+
+About twenty-five boards vanished into police custody in May 1990.
+As we have seen, there are an estimated 30,000 boards in America today.
+If we assume that one board in a hundred is up to no good with codes
+and cards (which rather flatters the honesty of the board-using community),
+then that would leave 2,975 outlaw boards untouched by Sundevil.
+Sundevil seized about one tenth of one percent of all computer
+bulletin boards in America. Seen objectively, this is something less
+than a comprehensive assault. In 1990, Sundevil's organizers--
+the team at the Phoenix Secret Service office, and the Arizona
+Attorney General's office-- had a list of at least THREE HUNDRED
+boards that they considered fully deserving of search and seizure warrants.
+The twenty-five boards actually seized were merely among the most obvious
+and egregious of this much larger list of candidates. All these boards
+had been examined beforehand--either by informants, who had passed printouts
+to the Secret Service, or by Secret Service agents themselves, who not only
+come equipped with modems but know how to use them.
+
+There were a number of motives for Sundevil. First, it offered
+a chance to get ahead of the curve on wire-fraud crimes.
+Tracking back credit-card ripoffs to their perpetrators
+can be appallingly difficult. If these miscreants
+have any kind of electronic sophistication, they can snarl
+their tracks through the phone network into a mind-boggling,
+untraceable mess, while still managing to "reach out and rob someone."
+Boards, however, full of brags and boasts, codes and cards,
+offer evidence in the handy congealed form.
+
+Seizures themselves--the mere physical removal of machines--
+tends to take the pressure off. During Sundevil, a large number
+of code kids, warez d00dz, and credit card thieves would be deprived
+of those boards--their means of community and conspiracy--in one swift blow.
+As for the sysops themselves (commonly among the boldest offenders)
+they would be directly stripped of their computer equipment,
+and rendered digitally mute and blind.
+
+And this aspect of Sundevil was carried out with great success.
+Sundevil seems to have been a complete tactical surprise--
+unlike the fragmentary and continuing seizures of the war on the
+Legion of Doom, Sundevil was precisely timed and utterly overwhelming.
+At least forty "computers" were seized during May 7, 8 and 9, 1990,
+in Cincinnati, Detroit, Los Angeles, Miami, Newark, Phoenix, Tucson,
+Richmond, San Diego, San Jose, Pittsburgh and San Francisco.
+Some cities saw multiple raids, such as the five separate raids
+in the New York City environs. Plano, Texas (essentially a suburb of
+the Dallas/Fort Worth metroplex, and a hub of the telecommunications industry)
+saw four computer seizures. Chicago, ever in the forefront, saw its own
+local Sundevil raid, briskly carried out by Secret Service agents
+Timothy Foley and Barbara Golden.
+
+Many of these raids occurred, not in the cities proper,
+but in associated white-middle class suburbs--places like
+Mount Lebanon, Pennsylvania and Clark Lake, Michigan.
+There were a few raids on offices; most took place in people's homes,
+the classic hacker basements and bedrooms.
+
+The Sundevil raids were searches and seizures, not a group of mass arrests.
+There were only four arrests during Sundevil. "Tony the Trashman,"
+a longtime teenage bete noire of the Arizona Racketeering unit,
+was arrested in Tucson on May 9. "Dr. Ripco," sysop of an outlaw board
+with the misfortune to exist in Chicago itself, was also arrested--
+on illegal weapons charges. Local units also arrested a 19-year-old
+female phone phreak named "Electra" in Pennsylvania, and a male juvenile
+in California. Federal agents however were not seeking arrests, but computers.
+
+Hackers are generally not indicted (if at all) until the evidence
+in their seized computers is evaluated--a process that can take weeks,
+months--even years. When hackers are arrested on the spot, it's generally
+an arrest for other reasons. Drugs and/or illegal weapons show up in a good
+third of anti-hacker computer seizures (though not during Sundevil).
+
+That scofflaw teenage hackers (or their parents) should have marijuana
+in their homes is probably not a shocking revelation, but the surprisingly
+common presence of illegal firearms in hacker dens is a bit disquieting.
+A Personal Computer can be a great equalizer for the techno-cowboy--
+much like that more traditional American "Great Equalizer,"
+the Personal Sixgun. Maybe it's not all that surprising
+that some guy obsessed with power through illicit technology
+would also have a few illicit high-velocity-impact devices around.
+An element of the digital underground particularly dotes on those
+"anarchy philes," and this element tends to shade into the crackpot milieu
+of survivalists, gun-nuts, anarcho-leftists and the ultra-libertarian
+right-wing.
+
+This is not to say that hacker raids to date have uncovered any
+major crack-dens or illegal arsenals; but Secret Service agents
+do not regard "hackers" as "just kids." They regard hackers as
+unpredictable people, bright and slippery. It doesn't help matters
+that the hacker himself has been "hiding behind his keyboard"
+all this time. Commonly, police have no idea what he looks like.
+This makes him an unknown quantity, someone best treated with
+proper caution.
+
+To date, no hacker has come out shooting, though they do sometimes brag on
+boards that they will do just that. Threats of this sort are taken seriously.
+Secret Service hacker raids tend to be swift, comprehensive, well-manned
+(even over-manned); and agents generally burst through every door
+in the home at once, sometimes with drawn guns. Any potential resistance
+is swiftly quelled. Hacker raids are usually raids on people's homes.
+It can be a very dangerous business to raid an American home;
+people can panic when strangers invade their sanctum. Statistically speaking,
+the most dangerous thing a policeman can do is to enter someone's home.
+(The second most dangerous thing is to stop a car in traffic.)
+People have guns in their homes. More cops are hurt in homes
+than are ever hurt in biker bars or massage parlors.
+
+But in any case, no one was hurt during Sundevil,
+or indeed during any part of the Hacker Crackdown.
+
+Nor were there any allegations of any physical mistreatment of a suspect.
+Guns were pointed, interrogations were sharp and prolonged; but no one
+in 1990 claimed any act of brutality by any crackdown raider.
+
+In addition to the forty or so computers, Sundevil reaped floppy disks
+in particularly great abundance--an estimated 23,000 of them, which
+naturally included every manner of illegitimate data: pirated games,
+stolen codes, hot credit card numbers, the complete text and software
+of entire pirate bulletin-boards. These floppy disks, which remain
+in police custody today, offer a gigantic, almost embarrassingly
+rich source of possible criminal indictments. These 23,000 floppy disks
+also include a thus-far unknown quantity of legitimate computer games,
+legitimate software, purportedly "private" mail from boards,
+business records, and personal correspondence of all kinds.
+
+Standard computer-crime search warrants lay great emphasis on seizing
+written documents as well as computers--specifically including photocopies,
+computer printouts, telephone bills, address books, logs, notes,
+memoranda and correspondence. In practice, this has meant that diaries,
+gaming magazines, software documentation, nonfiction books on hacking
+and computer security, sometimes even science fiction novels, have all
+vanished out the door in police custody. A wide variety of electronic items
+have been known to vanish as well, including telephones, televisions, answering
+machines, Sony Walkmans, desktop printers, compact disks, and audiotapes.
+
+No fewer than 150 members of the Secret Service were sent into
+the field during Sundevil. They were commonly accompanied by
+squads of local and/or state police. Most of these officers--
+especially the locals--had never been on an anti-hacker raid before.
+(This was one good reason, in fact, why so many of them were invited along
+in the first place.) Also, the presence of a uniformed police officer
+assures the raidees that the people entering their homes are, in fact, police.
+Secret Service agents wear plain clothes. So do the telco security experts
+who commonly accompany the Secret Service on raids (and who make no particular
+effort to identify themselves as mere employees of telephone companies).
+
+A typical hacker raid goes something like this. First, police storm in
+rapidly, through every entrance, with overwhelming force,
+in the assumption that this tactic will keep casualties to a minimum.
+Second, possible suspects are immediately removed from the vicinity
+of any and all computer systems, so that they will have no chance
+to purge or destroy computer evidence. Suspects are herded into a room
+without computers, commonly the living room, and kept under guard--
+not ARMED guard, for the guns are swiftly holstered, but under guard
+nevertheless. They are presented with the search warrant and warned
+that anything they say may be held against them. Commonly they have
+a great deal to say, especially if they are unsuspecting parents.
+
+Somewhere in the house is the "hot spot"--a computer tied to a phone
+line (possibly several computers and several phones). Commonly it's
+a teenager's bedroom, but it can be anywhere in the house;
+there may be several such rooms. This "hot spot" is put in charge
+of a two-agent team, the "finder" and the "recorder." The "finder"
+is computer-trained, commonly the case agent who has actually obtained
+the search warrant from a judge. He or she understands what is being sought,
+and actually carries out the seizures: unplugs machines, opens drawers,
+desks, files, floppy-disk containers, etc. The "recorder" photographs
+all the equipment, just as it stands--especially the tangle of
+wired connections in the back, which can otherwise be a real nightmare
+to restore. The recorder will also commonly photograph every room
+in the house, lest some wily criminal claim that the police had robbed him
+during the search. Some recorders carry videocams or tape recorders;
+however, it's more common for the recorder to simply take written notes.
+Objects are described and numbered as the finder seizes them, generally
+on standard preprinted police inventory forms.
+
+Even Secret Service agents were not, and are not, expert computer users.
+They have not made, and do not make, judgements on the fly about potential
+threats posed by various forms of equipment. They may exercise discretion;
+they may leave Dad his computer, for instance, but they don't HAVE to.
+Standard computer-crime search warrants, which date back to the early 80s,
+use a sweeping language that targets computers, most anything attached
+to a computer, most anything used to operate a computer--most anything
+that remotely resembles a computer--plus most any and all written documents
+surrounding it. Computer-crime investigators have strongly urged agents
+to seize the works.
+
+In this sense, Operation Sundevil appears to have been a complete success.
+Boards went down all over America, and were shipped en masse to the computer
+investigation lab of the Secret Service, in Washington DC, along with the
+23,000 floppy disks and unknown quantities of printed material.
+
+But the seizure of twenty-five boards, and the multi-megabyte mountains
+of possibly useful evidence contained in these boards (and in their owners'
+other computers, also out the door), were far from the only motives for
+Operation Sundevil. An unprecedented action of great ambition and size,
+Sundevil's motives can only be described as political. It was a
+public-relations effort, meant to pass certain messages, meant to make
+certain situations clear: both in the mind of the general public,
+and in the minds of various constituencies of the electronic community.
+
+ First --and this motivation was vital--a "message" would be sent from
+law enforcement to the digital underground. This very message was recited
+in so many words by Garry M. Jenkins, the Assistant Director of the
+US Secret Service, at the Sundevil press conference in Phoenix on
+May 9, 1990, immediately after the raids. In brief, hackers were
+mistaken in their foolish belief that they could hide behind the
+"relative anonymity of their computer terminals." On the contrary,
+they should fully understand that state and federal cops were
+actively patrolling the beat in cyberspace--that they were
+on the watch everywhere, even in those sleazy and secretive
+dens of cybernetic vice, the underground boards.
+
+This is not an unusual message for police to publicly convey to crooks.
+The message is a standard message; only the context is new.
+
+In this respect, the Sundevil raids were the digital equivalent
+of the standard vice-squad crackdown on massage parlors, porno bookstores,
+head-shops, or floating crap-games. There may be few or no arrests in a raid
+of this sort; no convictions, no trials, no interrogations. In cases of this
+sort, police may well walk out the door with many pounds of sleazy magazines,
+X-rated videotapes, sex toys, gambling equipment, baggies of marijuana. . . .
+
+Of course, if something truly horrendous is discovered by the raiders,
+there will be arrests and prosecutions. Far more likely, however,
+there will simply be a brief but sharp disruption of the closed
+and secretive world of the nogoodniks. There will be "street hassle."
+"Heat." "Deterrence." And, of course, the immediate loss of the seized goods.
+It is very unlikely that any of this seized material will ever be returned.
+Whether charged or not, whether convicted or not, the perpetrators will
+almost surely lack the nerve ever to ask for this stuff to be given back.
+
+Arrests and trials--putting people in jail--may involve all kinds of
+formal legalities; but dealing with the justice system is far from the only
+task of police. Police do not simply arrest people. They don't simply
+put people in jail. That is not how the police perceive their jobs.
+Police "protect and serve." Police "keep the peace," they "keep public order."
+Like other forms of public relations, keeping public order is not an
+exact science. Keeping public order is something of an art-form.
+
+If a group of tough-looking teenage hoodlums was loitering on a street-corner,
+no one would be surprised to see a street-cop arrive and sternly order
+them to "break it up." On the contrary, the surprise would come if one
+of these ne'er-do-wells stepped briskly into a phone-booth,
+called a civil rights lawyer, and instituted a civil suit
+in defense of his Constitutional rights of free speech
+and free assembly. But something much along this line
+was one of the many anomolous outcomes of the Hacker Crackdown.
+
+Sundevil also carried useful "messages" for other constituents of
+the electronic community. These messages may not have been read
+aloud from the Phoenix podium in front of the press corps,
+but there was little mistaking their meaning. There was a message
+of reassurance for the primary victims of coding and carding:
+the telcos, and the credit companies. Sundevil was greeted with joy
+by the security officers of the electronic business community.
+After years of high-tech harassment and spiralling revenue losses,
+their complaints of rampant outlawry were being taken seriously by
+law enforcement. No more head-scratching or dismissive shrugs;
+no more feeble excuses about "lack of computer-trained officers" or
+the low priority of "victimless" white-collar telecommunication crimes.
+
+Computer-crime experts have long believed that computer-related offenses
+are drastically under-reported. They regard this as a major open scandal
+of their field. Some victims are reluctant to come forth, because they
+believe that police and prosecutors are not computer-literate,
+and can and will do nothing. Others are embarrassed by
+their vulnerabilities, and will take strong measures
+to avoid any publicity; this is especially true of banks,
+who fear a loss of investor confidence should an embezzlement-case
+or wire-fraud surface. And some victims are so helplessly confused
+by their own high technology that they never even realize that
+a crime has occurred--even when they have been fleeced to the bone.
+
+The results of this situation can be dire.
+Criminals escape apprehension and punishment.
+The computer-crime units that do exist, can't get work.
+The true scope of computer-crime: its size, its real nature,
+the scope of its threats, and the legal remedies for it--
+all remain obscured.
+
+Another problem is very little publicized, but it is a cause
+of genuine concern. Where there is persistent crime,
+but no effective police protection, then vigilantism can result.
+Telcos, banks, credit companies, the major corporations who
+maintain extensive computer networks vulnerable to hacking
+--these organizations are powerful, wealthy, and
+politically influential. They are disinclined to be
+pushed around by crooks (or by most anyone else,
+for that matter). They often maintain well-organized
+private security forces, commonly run by
+experienced veterans of military and police units,
+who have left public service for the greener pastures
+of the private sector. For police, the corporate
+security manager can be a powerful ally; but if this
+gentleman finds no allies in the police, and the
+pressure is on from his board-of-directors,
+he may quietly take certain matters into his own hands.
+
+Nor is there any lack of disposable hired-help in the
+corporate security business. Private security agencies--
+the `security business' generally--grew explosively in the 1980s.
+Today there are spooky gumshoed armies of "security consultants,"
+"rent-a- cops," "private eyes," "outside experts"--every manner
+of shady operator who retails in "results" and discretion.
+Or course, many of these gentlemen and ladies may be paragons
+of professional and moral rectitude. But as anyone
+who has read a hard-boiled detective novel knows,
+police tend to be less than fond of this sort
+of private-sector competition.
+
+Companies in search of computer-security have even been
+known to hire hackers. Police shudder at this prospect.
+
+Police treasure good relations with the business community.
+Rarely will you see a policeman so indiscreet as to allege
+publicly that some major employer in his state or city has succumbed
+to paranoia and gone off the rails. Nevertheless,
+police --and computer police in particular--are aware
+of this possibility. Computer-crime police can and do
+spend up to half of their business hours just doing
+public relations: seminars, "dog and pony shows,"
+sometimes with parents' groups or computer users,
+but generally with their core audience: the likely
+victims of hacking crimes. These, of course, are telcos,
+credit card companies and large computer-equipped corporations.
+The police strongly urge these people, as good citizens,
+to report offenses and press criminal charges;
+they pass the message that there is someone in authority who cares,
+understands, and, best of all, will take useful action
+should a computer-crime occur.
+
+But reassuring talk is cheap. Sundevil offered action.
+
+The final message of Sundevil was intended for internal consumption
+by law enforcement. Sundevil was offered as proof that the community
+of American computer-crime police had come of age. Sundevil was
+proof that enormous things like Sundevil itself could now be accomplished.
+Sundevil was proof that the Secret Service and its local law-enforcement
+allies could act like a well-oiled machine--(despite the hampering use
+of those scrambled phones). It was also proof that the Arizona Organized
+Crime and Racketeering Unit--the sparkplug of Sundevil--ranked with the best
+in the world in ambition, organization, and sheer conceptual daring.
+
+And, as a final fillip, Sundevil was a message from the Secret Service
+to their longtime rivals in the Federal Bureau of Investigation.
+By Congressional fiat, both USSS and FBI formally share jurisdiction
+over federal computer-crimebusting activities. Neither of these groups
+has ever been remotely happy with this muddled situation. It seems to
+suggest that Congress cannot make up its mind as to which of these groups
+is better qualified. And there is scarcely a G-man or a Special Agent
+anywhere without a very firm opinion on that topic.
+
+#
+
+For the neophyte, one of the most puzzling aspects of the crackdown
+on hackers is why the United States Secret Service has anything at all
+to do with this matter.
+
+The Secret Service is best known for its primary public role:
+its agents protect the President of the United States.
+They also guard the President's family, the Vice President and his family,
+former Presidents, and Presidential candidates. They sometimes guard
+foreign dignitaries who are visiting the United States, especially foreign
+heads of state, and have been known to accompany American officials
+on diplomatic missions overseas.
+
+Special Agents of the Secret Service don't wear uniforms, but the
+Secret Service also has two uniformed police agencies. There's the
+former White House Police (now known as the Secret Service Uniformed Division,
+since they currently guard foreign embassies in Washington, as well as the
+White House itself). And there's the uniformed Treasury Police Force.
+
+The Secret Service has been charged by Congress with a number
+of little-known duties. They guard the precious metals in Treasury vaults.
+They guard the most valuable historical documents of the United States:
+originals of the Constitution, the Declaration of Independence,
+Lincoln's Second Inaugural Address, an American-owned copy of
+the Magna Carta, and so forth. Once they were assigned to guard
+the Mona Lisa, on her American tour in the 1960s.
+
+The entire Secret Service is a division of the Treasury Department.
+Secret Service Special Agents (there are about 1,900 of them)
+are bodyguards for the President et al, but they all work for the Treasury.
+And the Treasury (through its divisions of the U.S. Mint and the
+Bureau of Engraving and Printing) prints the nation's money.
+
+As Treasury police, the Secret Service guards the nation's currency;
+it is the only federal law enforcement agency with direct jurisdiction
+over counterfeiting and forgery. It analyzes documents for authenticity,
+and its fight against fake cash is still quite lively (especially since
+the skilled counterfeiters of Medellin, Columbia have gotten into the act).
+Government checks, bonds, and other obligations, which exist in untold
+millions and are worth untold billions, are common targets for forgery,
+which the Secret Service also battles. It even handles forgery
+of postage stamps.
+
+But cash is fading in importance today as money has become electronic.
+As necessity beckoned, the Secret Service moved from fighting the
+counterfeiting of paper currency and the forging of checks,
+to the protection of funds transferred by wire.
+
+From wire-fraud, it was a simple skip-and-jump to what is formally
+known as "access device fraud." Congress granted the Secret Service
+the authority to investigate "access device fraud" under Title 18
+of the United States Code (U.S.C. Section 1029).
+
+The term "access device" seems intuitively simple. It's some kind
+of high-tech gizmo you use to get money with. It makes good sense
+to put this sort of thing in the charge of counterfeiting and
+wire-fraud experts.
+
+However, in Section 1029, the term "access device" is very
+generously defined. An access device is: "any card, plate,
+code, account number, or other means of account access
+that can be used, alone or in conjunction with another access device,
+to obtain money, goods, services, or any other thing of value,
+or that can be used to initiate a transfer of funds."
+
+"Access device" can therefore be construed to include credit cards
+themselves (a popular forgery item nowadays). It also includes credit card
+account NUMBERS, those standards of the digital underground. The same goes
+for telephone charge cards (an increasingly popular item with telcos,
+who are tired of being robbed of pocket change by phone-booth thieves).
+And also telephone access CODES, those OTHER standards of the digital
+underground. (Stolen telephone codes may not "obtain money," but they
+certainly do obtain valuable "services," which is specifically forbidden
+by Section 1029.)
+
+We can now see that Section 1029 already pits the United States Secret Service
+directly against the digital underground, without any mention at all of
+the word "computer."
+
+Standard phreaking devices, like "blue boxes," used to steal phone service
+from old-fashioned mechanical switches, are unquestionably "counterfeit
+access devices." Thanks to Sec.1029, it is not only illegal to USE
+counterfeit access devices, but it is even illegal to BUILD them.
+"Producing," "designing" "duplicating" or "assembling" blue boxes
+are all federal crimes today, and if you do this, the Secret Service
+has been charged by Congress to come after you.
+
+Automatic Teller Machines, which replicated all over America during the 1980s,
+are definitely "access devices," too, and an attempt to tamper with their
+punch-in codes and plastic bank cards falls directly under Sec. 1029.
+
+Section 1029 is remarkably elastic. Suppose you find a computer password
+in somebody's trash. That password might be a "code"--it's certainly a
+"means of account access." Now suppose you log on to a computer
+and copy some software for yourself. You've certainly obtained
+"service" (computer service) and a "thing of value" (the software).
+Suppose you tell a dozen friends about your swiped password,
+and let them use it, too. Now you're "trafficking in unauthorized
+access devices." And when the Prophet, a member of the Legion of Doom,
+passed a stolen telephone company document to Knight Lightning
+at Phrack magazine, they were both charged under Sec. 1029!
+
+There are two limitations on Section 1029. First, the offense must
+"affect interstate or foreign commerce" in order to become a matter
+of federal jurisdiction. The term "affecting commerce" is not well defined;
+but you may take it as a given that the Secret Service can take an interest
+if you've done most anything that happens to cross a state line.
+State and local police can be touchy about their jurisdictions,
+and can sometimes be mulish when the feds show up. But when it comes
+to computer-crime, the local police are pathetically grateful
+for federal help--in fact they complain that they can't get enough of it.
+If you're stealing long-distance service, you're almost certainly crossing
+state lines, and you're definitely "affecting the interstate commerce"
+of the telcos. And if you're abusing credit cards by ordering stuff
+out of glossy catalogs from, say, Vermont, you're in for it.
+
+The second limitation is money. As a rule, the feds don't pursue
+penny-ante offenders. Federal judges will dismiss cases that appear
+to waste their time. Federal crimes must be serious; Section 1029
+specifies a minimum loss of a thousand dollars.
+
+We now come to the very next section of Title 18, which is Section 1030,
+"Fraud and related activity in connection with computers." This statute
+gives the Secret Service direct jurisdiction over acts of computer intrusion.
+On the face of it, the Secret Service would now seem to command the field.
+Section 1030, however, is nowhere near so ductile as Section 1029.
+
+The first annoyance is Section 1030(d), which reads:
+
+"(d) The United States Secret Service shall,
+IN ADDITION TO ANY OTHER AGENCY HAVING SUCH AUTHORITY,
+have the authority to investigate offenses under this section.
+Such authority of the United States Secret Service shall be
+exercised in accordance with an agreement which shall be entered
+into by the Secretary of the Treasury AND THE ATTORNEY GENERAL."
+(Author's italics.) [Represented by capitals.]
+
+The Secretary of the Treasury is the titular head of the Secret Service,
+while the Attorney General is in charge of the FBI. In Section (d),
+Congress shrugged off responsibility for the computer-crime turf-battle
+between the Service and the Bureau, and made them fight it out all
+by themselves. The result was a rather dire one for the Secret Service,
+for the FBI ended up with exclusive jurisdiction over computer break-ins
+having to do with national security, foreign espionage, federally insured
+banks, and U.S. military bases, while retaining joint jurisdiction over
+all the other computer intrusions. Essentially, when it comes to Section 1030,
+the FBI not only gets the real glamor stuff for itself, but can peer over the
+shoulder of the Secret Service and barge in to meddle whenever it suits them.
+
+The second problem has to do with the dicey term
+"Federal interest computer." Section 1030(a)(2)
+makes it illegal to "access a computer without authorization"
+if that computer belongs to a financial institution or an issuer
+of credit cards (fraud cases, in other words). Congress was quite
+willing to give the Secret Service jurisdiction over
+money-transferring computers, but Congress balked at
+letting them investigate any and all computer intrusions.
+Instead, the USSS had to settle for the money machines
+and the "Federal interest computers." A "Federal interest computer"
+is a computer which the government itself owns, or is using.
+Large networks of interstate computers, linked over state lines,
+are also considered to be of "Federal interest." (This notion of
+"Federal interest" is legally rather foggy and has never been
+clearly defined in the courts. The Secret Service has never yet
+had its hand slapped for investigating computer break-ins that were NOT
+of "Federal interest," but conceivably someday this might happen.)
+
+So the Secret Service's authority over "unauthorized access"
+to computers covers a lot of territory, but by no means the
+whole ball of cyberspatial wax. If you are, for instance,
+a LOCAL computer retailer, or the owner of a LOCAL bulletin
+board system, then a malicious LOCAL intruder can break in,
+crash your system, trash your files and scatter viruses,
+and the U.S. Secret Service cannot do a single thing about it.
+
+At least, it can't do anything DIRECTLY. But the Secret Service
+will do plenty to help the local people who can.
+
+The FBI may have dealt itself an ace off the bottom of the deck
+when it comes to Section 1030; but that's not the whole story;
+that's not the street. What's Congress thinks is one thing,
+and Congress has been known to change its mind. The REAL
+turf-struggle is out there in the streets where it's happening.
+If you're a local street-cop with a computer problem,
+the Secret Service wants you to know where you can find
+the real expertise. While the Bureau crowd are off having
+their favorite shoes polished--(wing-tips)--and making derisive
+fun of the Service's favorite shoes--("pansy-ass tassels")--
+the tassel-toting Secret Service has a crew of ready-and-able
+hacker-trackers installed in the capital of every state in the Union.
+Need advice? They'll give you advice, or at least point you in
+the right direction. Need training? They can see to that, too.
+
+If you're a local cop and you call in the FBI, the FBI
+(as is widely and slanderously rumored) will order you around
+like a coolie, take all the credit for your busts,
+and mop up every possible scrap of reflected glory.
+The Secret Service, on the other hand, doesn't brag a lot.
+They're the quiet types. VERY quiet. Very cool. Efficient.
+High-tech. Mirrorshades, icy stares, radio ear-plugs,
+an Uzi machine-pistol tucked somewhere in that well-cut jacket.
+American samurai, sworn to give their lives to protect our President.
+"The granite agents." Trained in martial arts, absolutely fearless.
+Every single one of 'em has a top-secret security clearance.
+Something goes a little wrong, you're not gonna hear any whining
+and moaning and political buck-passing out of these guys.
+
+The facade of the granite agent is not, of course, the reality.
+Secret Service agents are human beings. And the real glory
+in Service work is not in battling computer crime--not yet,
+anyway--but in protecting the President. The real glamour
+of Secret Service work is in the White House Detail.
+If you're at the President's side, then the kids and the wife
+see you on television; you rub shoulders with the most powerful
+people in the world. That's the real heart of Service work,
+the number one priority. More than one computer investigation
+has stopped dead in the water when Service agents vanished at
+the President's need.
+
+There's romance in the work of the Service. The intimate access
+to circles of great power; the esprit-de-corps of a highly trained
+and disciplined elite; the high responsibility of defending the
+Chief Executive; the fulfillment of a patriotic duty. And as police
+work goes, the pay's not bad. But there's squalor in Service work, too.
+You may get spat upon by protesters howling abuse--and if they get violent,
+if they get too close, sometimes you have to knock one of them down--
+discreetly.
+
+The real squalor in Service work is drudgery such as "the quarterlies,"
+traipsing out four times a year, year in, year out, to interview the various
+pathetic wretches, many of them in prisons and asylums, who have seen fit
+to threaten the President's life. And then there's the grinding stress
+of searching all those faces in the endless bustling crowds, looking for
+hatred, looking for psychosis, looking for the tight, nervous face
+of an Arthur Bremer, a Squeaky Fromme, a Lee Harvey Oswald.
+It's watching all those grasping, waving hands for sudden movements,
+while your ears strain at your radio headphone for the long-rehearsed
+cry of "Gun!"
+
+It's poring, in grinding detail, over the biographies of every rotten
+loser who ever shot at a President. It's the unsung work of the
+Protective Research Section, who study scrawled, anonymous death threats
+with all the meticulous tools of anti-forgery techniques.
+
+And it's maintaining the hefty computerized files on anyone
+who ever threatened the President's life. Civil libertarians
+have become increasingly concerned at the Government's use
+of computer files to track American citizens--but the
+Secret Service file of potential Presidential assassins,
+which has upward of twenty thousand names, rarely causes
+a peep of protest. If you EVER state that you intend to
+kill the President, the Secret Service will want to know
+and record who you are, where you are, what you are,
+and what you're up to. If you're a serious threat--
+if you're officially considered "of protective interest"--
+then the Secret Service may well keep tabs on you
+for the rest of your natural life.
+
+Protecting the President has first call on all the Service's resources.
+But there's a lot more to the Service's traditions and history than
+standing guard outside the Oval Office.
+
+The Secret Service is the nation's oldest general federal
+law-enforcement agency. Compared to the Secret Service,
+the FBI are new-hires and the CIA are temps. The Secret Service
+was founded 'way back in 1865, at the suggestion of Hugh McCulloch,
+Abraham Lincoln's Secretary of the Treasury. McCulloch wanted
+a specialized Treasury police to combat counterfeiting.
+Abraham Lincoln agreed that this seemed a good idea, and,
+with a terrible irony, Abraham Lincoln was shot that
+very night by John Wilkes Booth.
+
+The Secret Service originally had nothing to do with protecting Presidents.
+They didn't take this on as a regular assignment until after the Garfield
+assassination in 1881. And they didn't get any Congressional money for it
+until President McKinley was shot in 1901. The Service was originally
+designed for one purpose: destroying counterfeiters.
+
+#
+
+There are interesting parallels between the Service's
+nineteenth-century entry into counterfeiting,
+and America's twentieth-century entry into computer-crime.
+
+In 1865, America's paper currency was a terrible muddle.
+Security was drastically bad. Currency was printed on the spot
+by local banks in literally hundreds of different designs.
+No one really knew what the heck a dollar bill was supposed to look like.
+Bogus bills passed easily. If some joker told you that a one-dollar bill
+from the Railroad Bank of Lowell, Massachusetts had a woman leaning on
+a shield, with a locomotive, a cornucopia, a compass, various agricultural
+implements, a railroad bridge, and some factories, then you pretty much had
+to take his word for it. (And in fact he was telling the truth!)
+
+SIXTEEN HUNDRED local American banks designed and printed their own
+paper currency, and there were no general standards for security.
+Like a badly guarded node in a computer network, badly designed bills
+were easy to fake, and posed a security hazard for the entire monetary system.
+
+No one knew the exact extent of the threat to the currency.
+There were panicked estimates that as much as a third of
+the entire national currency was faked. Counterfeiters--
+known as "boodlers" in the underground slang of the time--
+were mostly technically skilled printers who had gone to the bad.
+Many had once worked printing legitimate currency.
+Boodlers operated in rings and gangs. Technical experts
+engraved the bogus plates--commonly in basements in New York City.
+Smooth confidence men passed large wads of high-quality,
+high-denomination fakes, including the really sophisticated stuff--
+government bonds, stock certificates, and railway shares.
+Cheaper, botched fakes were sold or sharewared to low-level
+gangs of boodler wannabes. (The really cheesy lowlife boodlers
+merely upgraded real bills by altering face values,
+changing ones to fives, tens to hundreds, and so on.)
+
+The techniques of boodling were little-known and regarded
+with a certain awe by the mid- nineteenth-century public.
+The ability to manipulate the system for rip-off seemed
+diabolically clever. As the skill and daring of the
+boodlers increased, the situation became intolerable.
+The federal government stepped in, and began offering
+its own federal currency, which was printed in fancy green ink,
+but only on the back--the original "greenbacks." And at first,
+the improved security of the well-designed, well-printed
+federal greenbacks seemed to solve the problem; but then
+the counterfeiters caught on. Within a few years things were
+worse than ever: a CENTRALIZED system where ALL security was bad!
+
+The local police were helpless. The Government tried offering
+blood money to potential informants, but this met with little success.
+Banks, plagued by boodling, gave up hope of police help and hired
+private security men instead. Merchants and bankers queued up
+by the thousands to buy privately-printed manuals on currency security,
+slim little books like Laban Heath's INFALLIBLE GOVERNMENT
+COUNTERFEIT DETECTOR. The back of the book offered Laban Heath's
+patent microscope for five bucks.
+
+Then the Secret Service entered the picture. The first agents
+were a rough and ready crew. Their chief was one William P. Wood,
+a former guerilla in the Mexican War who'd won a reputation busting
+contractor fraudsters for the War Department during the Civil War.
+Wood, who was also Keeper of the Capital Prison, had a sideline
+as a counterfeiting expert, bagging boodlers for the federal bounty money.
+
+Wood was named Chief of the new Secret Service in July 1865.
+There were only ten Secret Service agents in all: Wood himself,
+a handful who'd worked for him in the War Department, and a few
+former private investigators--counterfeiting experts--whom Wood
+had won over to public service. (The Secret Service of 1865 was
+much the size of the Chicago Computer Fraud Task Force or the
+Arizona Racketeering Unit of 1990.) These ten "Operatives"
+had an additional twenty or so "Assistant Operatives" and "Informants."
+Besides salary and per diem, each Secret Service employee received
+a whopping twenty-five dollars for each boodler he captured.
+
+Wood himself publicly estimated that at least HALF of America's currency
+was counterfeit, a perhaps pardonable perception. Within a year the
+Secret Service had arrested over 200 counterfeiters. They busted about
+two hundred boodlers a year for four years straight.
+
+Wood attributed his success to travelling fast and light, hitting the
+bad-guys hard, and avoiding bureaucratic baggage. "Because my raids
+were made without military escort and I did not ask the assistance
+of state officers, I surprised the professional counterfeiter."
+
+Wood's social message to the once-impudent boodlers bore an eerie ring
+of Sundevil: "It was also my purpose to convince such characters that
+it would no longer be healthy for them to ply their vocation without
+being handled roughly, a fact they soon discovered."
+
+William P. Wood, the Secret Service's guerilla pioneer,
+did not end well. He succumbed to the lure of aiming for
+the really big score. The notorious Brockway Gang of New York City,
+headed by William E. Brockway, the "King of the Counterfeiters,"
+had forged a number of government bonds. They'd passed these
+brilliant fakes on the prestigious Wall Street investment
+firm of Jay Cooke and Company. The Cooke firm were frantic
+and offered a huge reward for the forgers' plates.
+
+Laboring diligently, Wood confiscated the plates
+(though not Mr. Brockway) and claimed the reward.
+But the Cooke company treacherously reneged.
+Wood got involved in a down-and-dirty lawsuit
+with the Cooke capitalists. Wood's boss,
+Secretary of the Treasury McCulloch, felt that
+Wood's demands for money and glory were unseemly,
+and even when the reward money finally came through,
+McCulloch refused to pay Wood anything.
+Wood found himself mired in a seemingly endless
+round of federal suits and Congressional lobbying.
+
+Wood never got his money. And he lost his job to boot.
+He resigned in 1869.
+
+Wood's agents suffered, too. On May 12, 1869, the second Chief
+of the Secret Service took over, and almost immediately fired
+most of Wood's pioneer Secret Service agents: Operatives,
+Assistants and Informants alike. The practice of receiving $25
+per crook was abolished. And the Secret Service began the long,
+uncertain process of thorough professionalization.
+
+Wood ended badly. He must have felt stabbed in the back.
+In fact his entire organization was mangled.
+
+On the other hand, William P. Wood WAS the first head of the Secret Service.
+William Wood was the pioneer. People still honor his name. Who remembers
+the name of the SECOND head of the Secret Service?
+
+As for William Brockway (also known as "Colonel Spencer"),
+he was finally arrested by the Secret Service in 1880.
+He did five years in prison, got out, and was still boodling
+at the age of seventy-four.
+
+#
+
+Anyone with an interest in Operation Sundevil--
+or in American computer-crime generally--
+could scarcely miss the presence of Gail Thackeray,
+Assistant Attorney General of the State of Arizona.
+Computer-crime training manuals often cited
+Thackeray's group and her work; she was the
+highest-ranking state official to specialize
+in computer-related offenses. Her name had been
+on the Sundevil press release (though modestly ranked
+well after the local federal prosecuting attorney and
+the head of the Phoenix Secret Service office).
+
+As public commentary, and controversy, began to mount
+about the Hacker Crackdown, this Arizonan state official
+began to take a higher and higher public profile.
+Though uttering almost nothing specific about
+the Sundevil operation itself, she coined some
+of the most striking soundbites of the growing propaganda war:
+"Agents are operating in good faith, and I don't think
+you can say that for the hacker community," was one.
+Another was the memorable "I am not a mad dog prosecutor"
+(Houston Chronicle, Sept 2, 1990.) In the meantime,
+the Secret Service maintained its usual extreme discretion;
+the Chicago Unit, smarting from the backlash
+of the Steve Jackson scandal, had gone completely to earth.
+
+As I collated my growing pile of newspaper clippings,
+Gail Thackeray ranked as a comparative fount of public
+knowledge on police operations.
+
+I decided that I had to get to know Gail Thackeray.
+I wrote to her at the Arizona Attorney General's Office.
+Not only did she kindly reply to me, but, to my astonishment,
+she knew very well what "cyberpunk" science fiction was.
+
+Shortly after this, Gail Thackeray lost her job.
+And I temporarily misplaced my own career as
+a science-fiction writer, to become a full-time
+computer-crime journalist. In early March, 1991,
+I flew to Phoenix, Arizona, to interview Gail Thackeray
+for my book on the hacker crackdown.
+
+#
+
+"Credit cards didn't used to cost anything to get,"
+says Gail Thackeray. "Now they cost forty bucks--
+and that's all just to cover the costs from RIP-OFF ARTISTS."
+
+Electronic nuisance criminals are parasites.
+One by one they're not much harm, no big deal.
+But they never come just one by one. They come in swarms,
+heaps, legions, sometimes whole subcultures. And they bite.
+Every time we buy a credit card today, we lose a little financial
+vitality to a particular species of bloodsucker.
+
+What, in her expert opinion, are the worst forms of electronic crime,
+I ask, consulting my notes. Is it--credit card fraud? Breaking into
+ATM bank machines? Phone-phreaking? Computer intrusions?
+Software viruses? Access-code theft? Records tampering?
+Software piracy? Pornographic bulletin boards?
+Satellite TV piracy? Theft of cable service?
+It's a long list. By the time I reach the end
+of it I feel rather depressed.
+
+"Oh no," says Gail Thackeray, leaning forward over the table,
+her whole body gone stiff with energetic indignation,
+"the biggest damage is telephone fraud. Fake sweepstakes,
+fake charities. Boiler-room con operations. You could pay off
+the national debt with what these guys steal. . . .
+They target old people, they get hold of credit ratings
+and demographics, they rip off the old and the weak."
+The words come tumbling out of her.
+
+It's low-tech stuff, your everyday boiler-room fraud.
+Grifters, conning people out of money over the phone,
+have been around for decades. This is where the word "phony" came from!
+
+It's just that it's so much EASIER now, horribly facilitated by advances
+in technology and the byzantine structure of the modern phone system.
+The same professional fraudsters do it over and over, Thackeray tells me,
+they hide behind dense onion-shells of fake companies. . . fake holding
+corporations nine or ten layers deep, registered all over the map.
+They get a phone installed under a false name in an empty safe-house.
+And then they call-forward everything out of that phone to yet
+another phone, a phone that may even be in another STATE.
+And they don't even pay the charges on their phones;
+after a month or so, they just split; set up somewhere else
+in another Podunkville with the same seedy crew of veteran phone-crooks.
+They buy or steal commercial credit card reports, slap them on the PC,
+have a program pick out people over sixty-five who pay a lot to charities.
+A whole subculture living off this, merciless folks on the con.
+
+"The `light-bulbs for the blind' people," Thackeray muses,
+with a special loathing. "There's just no end to them."
+
+We're sitting in a downtown diner in Phoenix, Arizona.
+It's a tough town, Phoenix. A state capital seeing some hard times.
+Even to a Texan like myself, Arizona state politics seem rather baroque.
+There was, and remains, endless trouble over the Martin Luther King holiday,
+the sort of stiff-necked, foot-shooting incident for which Arizona politics
+seem famous. There was Evan Mecham, the eccentric Republican millionaire
+governor who was impeached, after reducing state government to a
+ludicrous shambles. Then there was the national Keating scandal,
+involving Arizona savings and loans, in which both of Arizona's
+U.S. senators, DeConcini and McCain, played sadly prominent roles.
+
+And the very latest is the bizarre AzScam case,
+in which state legislators were videotaped,
+eagerly taking cash from an informant of the Phoenix city
+police department, who was posing as a Vegas mobster.
+
+"Oh," says Thackeray cheerfully. "These people are amateurs here,
+they thought they were finally getting to play with the big boys.
+They don't have the least idea how to take a bribe!
+It's not institutional corruption. It's not like back in Philly."
+
+Gail Thackeray was a former prosecutor in Philadelphia.
+Now she's a former assistant attorney general of the State of Arizona.
+Since moving to Arizona in 1986, she had worked under the aegis
+of Steve Twist, her boss in the Attorney General's office.
+Steve Twist wrote Arizona's pioneering computer crime laws
+and naturally took an interest in seeing them enforced.
+It was a snug niche, and Thackeray's Organized Crime and
+Racketeering Unit won a national reputation for ambition
+and technical knowledgeability. . . . Until the latest
+election in Arizona. Thackeray's boss ran for the top
+job, and lost. The victor, the new Attorney General,
+apparently went to some pains to eliminate the bureaucratic
+traces of his rival, including his pet group--Thackeray's group.
+Twelve people got their walking papers.
+
+Now Thackeray's painstakingly assembled computer lab
+sits gathering dust somewhere in the glass-and-concrete
+Attorney General's HQ on 1275 Washington Street.
+Her computer-crime books, her painstakingly garnered
+back issues of phreak and hacker zines, all bought
+at her own expense--are piled in boxes somewhere.
+The State of Arizona is simply not particularly
+interested in electronic racketeering at the moment.
+
+At the moment of our interview, Gail Thackeray,
+officially unemployed, is working out of the county
+sheriff's office, living on her savings, and prosecuting
+several cases--working 60-hour weeks, just as always--
+for no pay at all. "I'm trying to train people,"
+she mutters.
+
+Half her life seems to be spent training people--merely pointing out,
+to the naive and incredulous (such as myself) that this stuff
+is ACTUALLY GOING ON OUT THERE. It's a small world, computer crime.
+A young world. Gail Thackeray, a trim blonde Baby-Boomer who favors
+Grand Canyon white-water rafting to kill some slow time,
+is one of the world's most senior, most veteran "hacker-trackers."
+Her mentor was Donn Parker, the California think-tank theorist
+who got it all started `way back in the mid-70s, the "grandfather
+of the field," "the great bald eagle of computer crime."
+
+And what she has learned, Gail Thackeray teaches. Endlessly.
+Tirelessly. To anybody. To Secret Service agents and state police,
+at the Glynco, Georgia federal training center. To local police,
+on "roadshows" with her slide projector and notebook.
+To corporate security personnel. To journalists. To parents.
+
+Even CROOKS look to Gail Thackeray for advice.
+Phone-phreaks call her at the office. They know very
+well who she is. They pump her for information
+on what the cops are up to, how much they know.
+Sometimes whole CROWDS of phone phreaks,
+hanging out on illegal conference calls, will call Gail
+Thackeray up. They taunt her. And, as always,
+they boast. Phone-phreaks, real stone phone-phreaks,
+simply CANNOT SHUT UP. They natter on for hours.
+
+Left to themselves, they mostly talk about the intricacies
+of ripping-off phones; it's about as interesting as listening
+to hot-rodders talk about suspension and distributor-caps.
+They also gossip cruelly about each other. And when talking
+to Gail Thackeray, they incriminate themselves. "I have tapes,"
+Thackeray says coolly.
+
+Phone phreaks just talk like crazy. "Dial-Tone" out in Alabama
+has been known to spend half-an-hour simply reading stolen
+phone-codes aloud into voice-mail answering machines.
+Hundreds, thousands of numbers, recited in a monotone,
+without a break--an eerie phenomenon. When arrested,
+it's a rare phone phreak who doesn't inform at endless length
+on everybody he knows.
+
+Hackers are no better. What other group of criminals,
+she asks rhetorically, publishes newsletters and holds conventions?
+She seems deeply nettled by the sheer brazenness of this behavior,
+though to an outsider, this activity might make one wonder
+whether hackers should be considered "criminals" at all.
+Skateboarders have magazines, and they trespass a lot.
+Hot rod people have magazines and they break speed limits
+and sometimes kill people. . . .
+
+I ask her whether it would be any loss to society if phone phreaking
+and computer hacking, as hobbies, simply dried up and blew away,
+so that nobody ever did it again.
+
+She seems surprised. "No," she says swiftly. "Maybe a little. . .
+in the old days. . .the MIT stuff. . . . But there's a lot of wonderful,
+legal stuff you can do with computers now, you don't have to break into
+somebody else's just to learn. You don't have that excuse.
+You can learn all you like."
+
+Did you ever hack into a system? I ask.
+
+The trainees do it at Glynco. Just to demonstrate system vulnerabilities.
+She's cool to the notion. Genuinely indifferent.
+
+"What kind of computer do you have?"
+
+"A Compaq 286LE," she mutters.
+
+"What kind do you WISH you had?"
+
+At this question, the unmistakable light of true hackerdom flares in
+Gail Thackeray's eyes. She becomes tense, animated, the words pour out:
+"An Amiga 2000 with an IBM card and Mac emulation! The most common hacker
+machines are Amigas and Commodores. And Apples." If she had the Amiga,
+she enthuses, she could run a whole galaxy of seized computer-evidence disks
+on one convenient multifunctional machine. A cheap one, too. Not like the
+old Attorney General lab, where they had an ancient CP/M machine,
+assorted Amiga flavors and Apple flavors, a couple IBMS, all the
+utility software. . .but no Commodores. The workstations down
+at the Attorney General's are Wang dedicated word-processors.
+Lame machines tied in to an office net--though at least they get
+on- line to the Lexis and Westlaw legal data services.
+
+I don't say anything. I recognize the syndrome, though.
+This computer-fever has been running through segments of
+our society for years now. It's a strange kind of lust:
+K-hunger, Meg-hunger; but it's a shared disease;
+it can kill parties dead, as conversation spirals into
+the deepest and most deviant recesses of software releases
+and expensive peripherals. . . . The mark of the hacker beast.
+I have it too. The whole "electronic community," whatever the hell
+that is, has it. Gail Thackeray has it. Gail Thackeray is a hacker cop.
+My immediate reaction is a strong rush of indignant pity:
+WHY DOESN'T SOMEBODY BUY THIS WOMAN HER AMIGA?!
+It's not like she's asking for a Cray X-MP
+supercomputer mainframe; an Amiga's a sweet little
+cookie-box thing. We're losing zillions in organized fraud;
+prosecuting and defending a single hacker case in court can cost
+a hundred grand easy. How come nobody can come up with four lousy grand
+so this woman can do her job? For a hundred grand we could buy every
+computer cop in America an Amiga. There aren't that many of 'em.
+
+Computers. The lust, the hunger, for computers.
+The loyalty they inspire, the intense sense of possessiveness.
+The culture they have bred. I myself am sitting in downtown Phoenix,
+Arizona because it suddenly occurred to me that the police might--
+just MIGHT--come and take away my computer. The prospect of this,
+the mere IMPLIED THREAT, was unbearable. It literally changed my life.
+It was changing the lives of many others. Eventually it would change
+everybody's life.
+
+Gail Thackeray was one of the top computer-crime people in America.
+And I was just some novelist, and yet I had a better computer than hers.
+PRACTICALLY EVERYBODY I KNEW had a better computer than Gail Thackeray
+and her feeble laptop 286. It was like sending the sheriff in to clean
+up Dodge City and arming her with a slingshot cut from an old rubber tire.
+
+But then again, you don't need a howitzer to enforce the law.
+You can do a lot just with a badge. With a badge alone,
+you can basically wreak havoc, take a terrible vengeance on wrongdoers.
+Ninety percent of "computer crime investigation" is just "crime investigation:"
+names, places, dossiers, modus operandi, search warrants, victims,
+complainants, informants. . . .
+
+What will computer crime look like in ten years? Will it get better?
+Did "Sundevil" send 'em reeling back in confusion?
+
+It'll be like it is now, only worse, she tells me with perfect conviction.
+Still there in the background, ticking along, changing with the times:
+the criminal underworld. It'll be like drugs are. Like our problems
+with alcohol. All the cops and laws in the world never solved our problems
+with alcohol. If there's something people want, a certain percentage
+of them are just going to take it. Fifteen percent of the populace
+will never steal. Fifteen percent will steal most anything not nailed down.
+The battle is for the hearts and minds of the remaining seventy percent.
+
+And criminals catch on fast. If there's not "too steep a learning curve"--
+if it doesn't require a baffling amount of expertise and practice--
+then criminals are often some of the first through the gate of a
+new technology. Especially if it helps them to hide.
+They have tons of cash, criminals. The new communications tech--
+like pagers, cellular phones, faxes, Federal Express--were pioneered
+by rich corporate people, and by criminals. In the early years
+of pagers and beepers, dope dealers were so enthralled this technology
+that owing a beeper was practically prima facie evidence of cocaine dealing.
+CB radio exploded when the speed limit hit 55 and breaking the highway law
+became a national pastime. Dope dealers send cash by Federal Express,
+despite, or perhaps BECAUSE OF, the warnings in FedEx offices that tell you
+never to try this. Fed Ex uses X-rays and dogs on their mail,
+to stop drug shipments. That doesn't work very well.
+
+Drug dealers went wild over cellular phones.
+There are simple methods of faking ID on cellular phones,
+making the location of the call mobile, free of charge,
+and effectively untraceable. Now victimized cellular
+companies routinely bring in vast toll-lists of calls
+to Colombia and Pakistan.
+
+Judge Greene's fragmentation of the phone company
+is driving law enforcement nuts. Four thousand
+telecommunications companies. Fraud skyrocketing.
+Every temptation in the world available with a phone
+and a credit card number. Criminals untraceable.
+A galaxy of "new neat rotten things to do."
+
+If there were one thing Thackeray would like to have,
+it would be an effective legal end-run through this new
+fragmentation minefield.
+
+It would be a new form of electronic search warrant,
+an "electronic letter of marque" to be issued by a judge.
+It would create a new category of "electronic emergency."
+Like a wiretap, its use would be rare, but it would cut
+across state lines and force swift cooperation from all concerned.
+Cellular, phone, laser, computer network, PBXes, AT&T, Baby Bells,
+long-distance entrepreneurs, packet radio. Some document,
+some mighty court-order, that could slice through four thousand
+separate forms of corporate red-tape, and get her at once to
+the source of calls, the source of email threats and viruses,
+the sources of bomb threats, kidnapping threats. "From now on,"
+she says, "the Lindbergh baby will always die."
+
+Something that would make the Net sit still, if only for a moment.
+Something that would get her up to speed. Seven league boots.
+That's what she really needs. "Those guys move in nanoseconds
+and I'm on the Pony Express."
+
+And then, too, there's the coming international angle.
+Electronic crime has never been easy to localize,
+to tie to a physical jurisdiction. And phone-phreaks
+and hackers loathe boundaries, they jump them whenever they can.
+The English. The Dutch. And the Germans, especially the ubiquitous
+Chaos Computer Club. The Australians. They've all learned phone-phreaking
+from America. It's a growth mischief industry. The multinational
+networks are global, but governments and the police simply aren't.
+Neither are the laws. Or the legal frameworks for citizen protection.
+
+One language is global, though--English. Phone phreaks speak English;
+it's their native tongue even if they're Germans. English may have started
+in England but now it's the Net language; it might as well be called "CNNese."
+
+Asians just aren't much into phone phreaking. They're the world masters
+at organized software piracy. The French aren't into phone-phreaking either.
+The French are into computerized industrial espionage.
+
+In the old days of the MIT righteous hackerdom, crashing systems
+didn't hurt anybody. Not all that much, anyway. Not permanently.
+Now the players are more venal. Now the consequences are worse.
+Hacking will begin killing people soon. Already there are methods
+of stacking calls onto 911 systems, annoying the police, and possibly
+causing the death of some poor soul calling in with a genuine emergency.
+Hackers in Amtrak computers, or air-traffic control computers, will kill
+somebody someday. Maybe a lot of people. Gail Thackeray expects it.
+
+And the viruses are getting nastier. The "Scud" virus is the latest one out.
+It wipes hard-disks.
+
+According to Thackeray, the idea that phone-phreaks are Robin Hoods is a fraud.
+They don't deserve this repute. Basically, they pick on the weak. AT&T now
+protects itself with the fearsome ANI (Automatic Number Identification)
+trace capability. When AT&T wised up and tightened security generally,
+the phreaks drifted into the Baby Bells. The Baby Bells lashed out in 1989
+and 1990, so the phreaks switched to smaller long-distance entrepreneurs.
+Today, they are moving into locally owned PBXes and voice-mail systems,
+which are full of security holes, dreadfully easy to hack. These victims
+aren't the moneybags Sheriff of Nottingham or Bad King John, but small groups
+of innocent people who find it hard to protect themselves, and who really
+suffer from these depredations. Phone phreaks pick on the weak. They do it
+for power. If it were legal, they wouldn't do it. They don't want service,
+or knowledge, they want the thrill of power-tripping. There's plenty of
+knowledge or service around if you're willing to pay. Phone phreaks don't pay,
+they steal. It's because it is illegal that it feels like power,
+that it gratifies their vanity.
+
+I leave Gail Thackeray with a handshake at the door of her office building--
+a vast International-Style office building downtown. The Sheriff's office
+is renting part of it. I get the vague impression that quite a lot of the
+building is empty--real estate crash.
+
+In a Phoenix sports apparel store, in a downtown mall, I meet
+the "Sun Devil" himself. He is the cartoon mascot of
+Arizona State University, whose football stadium, "Sundevil,"
+is near the local Secret Service HQ--hence the name Operation Sundevil.
+The Sun Devil himself is named "Sparky." Sparky the Sun Devil is maroon
+and bright yellow, the school colors. Sparky brandishes a three-tined
+yellow pitchfork. He has a small mustache, pointed ears, a barbed tail,
+and is dashing forward jabbing the air with the pitchfork,
+with an expression of devilish glee.
+
+Phoenix was the home of Operation Sundevil. The Legion of Doom
+ran a hacker bulletin board called "The Phoenix Project."
+An Australian hacker named "Phoenix" once burrowed through
+the Internet to attack Cliff Stoll, then bragged and boasted
+about it to The New York Times. This net of coincidence
+is both odd and meaningless.
+
+The headquarters of the Arizona Attorney General, Gail Thackeray's
+former workplace, is on 1275 Washington Avenue. Many of the downtown
+streets in Phoenix are named after prominent American presidents:
+Washington, Jefferson, Madison. . . .
+
+After dark, all the employees go home to their suburbs.
+Washington, Jefferson and Madison--what would be the
+Phoenix inner city, if there were an inner city in this
+sprawling automobile-bred town--become the haunts
+of transients and derelicts. The homeless. The sidewalks
+along Washington are lined with orange trees.
+Ripe fallen fruit lies scattered like croquet balls
+on the sidewalks and gutters. No one seems to be eating them.
+I try a fresh one. It tastes unbearably bitter.
+
+The Attorney General's office, built in 1981 during the
+Babbitt administration, is a long low two-story building
+of white cement and wall-sized sheets of curtain-glass.
+Behind each glass wall is a lawyer's office, quite open
+and visible to anyone strolling by. Across the street
+is a dour government building labelled simply ECONOMIC SECURITY,
+something that has not been in great supply in the American
+Southwest lately.
+
+The offices are about twelve feet square. They feature
+tall wooden cases full of red-spined lawbooks;
+Wang computer monitors; telephones; Post-it notes galore.
+Also framed law diplomas and a general excess of bad
+Western landscape art. Ansel Adams photos are a big favorite,
+perhaps to compensate for the dismal specter of the parking lot,
+two acres of striped black asphalt, which features gravel landscaping
+and some sickly-looking barrel cacti.
+
+It has grown dark. Gail Thackeray has told me that the people
+who work late here, are afraid of muggings in the parking lot.
+It seems cruelly ironic that a woman tracing electronic racketeers
+across the interstate labyrinth of Cyberspace should fear an assault
+by a homeless derelict in the parking lot of her own workplace.
+
+Perhaps this is less than coincidence. Perhaps these two seemingly
+disparate worlds are somehow generating one another. The poor and
+disenfranchised take to the streets, while the rich and computer-equipped,
+safe in their bedrooms, chatter over their modems. Quite often the derelicts
+kick the glass out and break in to the lawyers' offices, if they see something
+they need or want badly enough.
+
+I cross the parking lot to the street behind the Attorney General's office.
+A pair of young tramps are bedding down on flattened sheets of cardboard,
+under an alcove stretching over the sidewalk. One tramp wears a
+glitter-covered T-shirt reading "CALIFORNIA" in Coca-Cola cursive.
+His nose and cheeks look chafed and swollen; they glisten with
+what seems to be Vaseline. The other tramp has a ragged long-sleeved
+shirt and lank brown hair parted in the middle. They both wear blue jeans
+coated in grime. They are both drunk.
+
+"You guys crash here a lot?" I ask them.
+
+They look at me warily. I am wearing black jeans, a black pinstriped
+suit jacket and a black silk tie. I have odd shoes and a funny haircut.
+
+"It's our first time here," says the red-nosed tramp unconvincingly.
+There is a lot of cardboard stacked here. More than any two people could use.
+
+"We usually stay at the Vinnie's down the street," says the brown-haired tramp,
+puffing a Marlboro with a meditative air, as he sprawls with his head on
+a blue nylon backpack. "The Saint Vincent's."
+
+"You know who works in that building over there?" I ask, pointing.
+
+The brown-haired tramp shrugs. "Some kind of attorneys, it says."
+
+We urge one another to take it easy. I give them five bucks.
+
+A block down the street I meet a vigorous workman who is wheeling along
+some kind of industrial trolley; it has what appears to be a tank of
+propane on it.
+
+We make eye contact. We nod politely. I walk past him. "Hey!
+Excuse me sir!" he says.
+
+"Yes?" I say, stopping and turning.
+
+"Have you seen," the guy says rapidly, "a black guy, about 6'7",
+scars on both his cheeks like this--" he gestures-- "wears a
+black baseball cap on backwards, wandering around here anyplace?"
+
+"Sounds like I don't much WANT to meet him," I say.
+
+"He took my wallet," says my new acquaintance.
+"Took it this morning. Y'know, some people would be
+SCARED of a guy like that. But I'm not scared.
+I'm from Chicago. I'm gonna hunt him down.
+We do things like that in Chicago."
+
+"Yeah?"
+
+"I went to the cops and now he's got an APB out on his ass,"
+he says with satisfaction. "You run into him, you let me know."
+
+"Okay," I say. "What is your name, sir?"
+
+"Stanley. . . ."
+
+"And how can I reach you?"
+
+"Oh," Stanley says, in the same rapid voice,
+"you don't have to reach, uh, me.
+You can just call the cops. Go straight to the cops."
+He reaches into a pocket and pulls out a greasy piece of pasteboard.
+"See, here's my report on him."
+
+I look. The "report," the size of an index card, is labelled PRO-ACT:
+Phoenix Residents Opposing Active Crime Threat. . . . or is it
+Organized Against Crime Threat? In the darkening street it's hard
+to read. Some kind of vigilante group? Neighborhood watch?
+I feel very puzzled.
+
+"Are you a police officer, sir?"
+
+He smiles, seems very pleased by the question.
+
+"No," he says.
+
+"But you are a `Phoenix Resident?'"
+
+"Would you believe a homeless person," Stanley says.
+
+"Really? But what's with the. . . ." For the first time I take a close look
+at Stanley's trolley. It's a rubber-wheeled thing of industrial metal,
+but the device I had mistaken for a tank of propane is in fact a water-cooler.
+Stanley also has an Army duffel-bag, stuffed tight as a sausage with clothing
+or perhaps a tent, and, at the base of his trolley, a cardboard box and a
+battered leather briefcase.
+
+"I see," I say, quite at a loss. For the first time I notice that Stanley
+has a wallet. He has not lost his wallet at all. It is in his back pocket
+and chained to his belt. It's not a new wallet. It seems to have seen
+a lot of wear.
+
+"Well, you know how it is, brother," says Stanley.
+Now that I know that he is homeless--A POSSIBLE
+THREAT--my entire perception of him has changed
+in an instant. His speech, which once seemed just
+bright and enthusiastic, now seems to have a
+dangerous tang of mania. "I have to do this!"
+he assures me. "Track this guy down. . . .
+It's a thing I do. . . you know. . .to keep myself together!"
+He smiles, nods, lifts his trolley by its decaying rubber handgrips.
+
+"Gotta work together, y'know," Stanley booms, his face alight
+with cheerfulness, "the police can't do everything!"
+The gentlemen I met in my stroll in downtown Phoenix
+are the only computer illiterates in this book.
+To regard them as irrelevant, however, would be a grave mistake.
+
+As computerization spreads across society, the populace at large
+is subjected to wave after wave of future shock. But, as a
+necessary converse, the "computer community" itself is subjected
+to wave after wave of incoming computer illiterates.
+How will those currently enjoying America's digital bounty regard,
+and treat, all this teeming refuse yearning to breathe free?
+Will the electronic frontier be another Land of Opportunity--
+or an armed and monitored enclave, where the disenfranchised
+snuggle on their cardboard at the locked doors of our houses of justice?
+
+Some people just don't get along with computers. They can't read.
+They can't type. They just don't have it in their heads to master
+arcane instructions in wirebound manuals. Somewhere, the process
+of computerization of the populace will reach a limit. Some people--
+quite decent people maybe, who might have thrived in any other situation--
+will be left irretrievably outside the bounds. What's to be done with
+these people, in the bright new shiny electroworld? How will they
+be regarded, by the mouse-whizzing masters of cyberspace? With contempt?
+Indifference? Fear?
+
+In retrospect, it astonishes me to realize how quickly poor Stanley
+became a perceived threat. Surprise and fear are closely allied feelings.
+And the world of computing is full of surprises.
+
+I met one character in the streets of Phoenix whose role in this book
+is supremely and directly relevant. That personage was Stanley's giant
+thieving scarred phantom. This phantasm is everywhere in this book.
+He is the specter haunting cyberspace.
+
+Sometimes he's a maniac vandal ready to smash the phone system
+for no sane reason at all. Sometimes he's a fascist fed,
+coldly programming his mighty mainframes to destroy our Bill of Rights.
+Sometimes he's a telco bureaucrat, covertly conspiring to register all modems
+in the service of an Orwellian surveillance regime. Mostly, though,
+this fearsome phantom is a "hacker." He's strange, he doesn't belong,
+he's not authorized, he doesn't smell right, he's not keeping his proper place,
+he's not one of us. The focus of fear is the hacker, for much the same
+reasons that Stanley's fancied assailant is black.
+
+Stanley's demon can't go away, because he doesn't exist.
+Despite singleminded and tremendous effort, he can't be arrested,
+sued, jailed, or fired. The only constructive way to do ANYTHING
+about him is to learn more about Stanley himself. This learning process
+may be repellent, it may be ugly, it may involve grave elements of paranoiac
+confusion, but it's necessary. Knowing Stanley requires something more
+than class-crossing condescension. It requires more than steely
+legal objectivity. It requires human compassion and sympathy.
+
+To know Stanley is to know his demon. If you know the other guy's demon,
+then maybe you'll come to know some of your own. You'll be able to
+separate reality from illusion. And then you won't do your cause,
+and yourself, more harm than good. Like poor damned Stanley from Chicago did.
+
+#
+
+The Federal Computer Investigations Committee (FCIC) is the most important
+and influential organization in the realm of American computer-crime.
+Since the police of other countries have largely taken their computer-crime
+cues from American methods, the FCIC might well be called the most important
+computer crime group in the world.
+
+It is also, by federal standards, an organization of great unorthodoxy.
+State and local investigators mix with federal agents. Lawyers,
+financial auditors and computer-security programmers trade notes
+with street cops. Industry vendors and telco security people show up
+to explain their gadgetry and plead for protection and justice.
+Private investigators, think-tank experts and industry pundits throw in
+their two cents' worth. The FCIC is the antithesis of a formal bureaucracy.
+
+Members of the FCIC are obscurely proud of this fact; they recognize their
+group as aberrant, but are entirely convinced that this, for them,
+outright WEIRD behavior is nevertheless ABSOLUTELY NECESSARY
+to get their jobs done.
+
+FCIC regulars --from the Secret Service, the FBI, the IRS,
+the Department of Labor, the offices of federal attorneys,
+state police, the Air Force, from military intelligence--
+often attend meetings, held hither and thither across the country,
+at their own expense. The FCIC doesn't get grants. It doesn't
+charge membership fees. It doesn't have a boss. It has no headquarters--
+just a mail drop in Washington DC, at the Fraud Division of the Secret Service.
+It doesn't have a budget. It doesn't have schedules. It meets three times
+a year--sort of. Sometimes it issues publications, but the FCIC
+has no regular publisher, no treasurer, not even a secretary.
+There are no minutes of FCIC meetings. Non-federal people are considered
+"non-voting members," but there's not much in the way of elections.
+There are no badges, lapel pins or certificates of membership.
+Everyone is on a first-name basis. There are about forty of them.
+Nobody knows how many, exactly. People come, people go--
+sometimes people "go" formally but still hang around anyway.
+Nobody has ever exactly figured out what "membership" of this
+"Committee" actually entails.
+
+Strange as this may seem to some, to anyone familiar with the social world
+of computing, the "organization" of the FCIC is very recognizable.
+
+For years now, economists and management theorists have speculated
+that the tidal wave of the information revolution would destroy rigid,
+pyramidal bureaucracies, where everything is top-down and
+centrally controlled. Highly trained "employees" would take on
+much greater autonomy, being self-starting, and self-motivating,
+moving from place to place, task to task, with great speed and fluidity.
+"Ad-hocracy" would rule, with groups of people spontaneously knitting
+together across organizational lines, tackling the problem at hand,
+applying intense computer-aided expertise to it, and then vanishing
+whence they came.
+
+This is more or less what has actually happened in the world of
+federal computer investigation. With the conspicuous exception
+of the phone companies, which are after all over a hundred years old,
+practically EVERY organization that plays any important role in this book
+functions just like the FCIC. The Chicago Task Force, the Arizona
+Racketeering Unit, the Legion of Doom, the Phrack crowd, the
+Electronic Frontier Foundation--they ALL look and act like "tiger teams"
+or "user's groups." They are all electronic ad-hocracies leaping up
+spontaneously to attempt to meet a need.
+
+Some are police. Some are, by strict definition, criminals.
+Some are political interest-groups. But every single group
+has that same quality of apparent spontaneity--"Hey, gang!
+My uncle's got a barn--let's put on a show!"
+
+Every one of these groups is embarrassed by this "amateurism,"
+and, for the sake of their public image in a world of non-computer people,
+they all attempt to look as stern and formal and impressive as possible.
+These electronic frontier-dwellers resemble groups of nineteenth-century
+pioneers hankering after the respectability of statehood.
+There are however, two crucial differences in the historical experience
+of these "pioneers" of the nineteeth and twenty-first centuries.
+
+First, powerful information technology DOES play into the hands of small,
+fluid, loosely organized groups. There have always been "pioneers,"
+"hobbyists," "amateurs," "dilettantes," "volunteers," "movements,"
+"users' groups" and "blue-ribbon panels of experts" around.
+But a group of this kind--when technically equipped to ship
+huge amounts of specialized information, at lightning speed,
+to its members, to government, and to the press--is simply
+a different kind of animal. It's like the difference between
+an eel and an electric eel.
+
+The second crucial change is that American society is currently
+in a state approaching permanent technological revolution.
+In the world of computers particularly, it is practically impossible
+to EVER stop being a "pioneer," unless you either drop dead or
+deliberately jump off the bus. The scene has never slowed down
+enough to become well-institutionalized. And after twenty, thirty,
+forty years the "computer revolution" continues to spread,
+to permeate new corners of society. Anything that really works
+is already obsolete.
+
+If you spend your entire working life as a "pioneer," the word "pioneer"
+begins to lose its meaning. Your way of life looks less and less like
+an introduction to something else" more stable and organized,
+and more and more like JUST THE WAY THINGS ARE. A "permanent revolution"
+is really a contradiction in terms. If "turmoil" lasts long enough,
+it simply becomes A NEW KIND OF SOCIETY--still the same game of history,
+but new players, new rules.
+
+Apply this to the world of late twentieth-century law enforcement,
+and the implications are novel and puzzling indeed. Any bureaucratic
+rulebook you write about computer-crime will be flawed when you write it,
+and almost an antique by the time it sees print. The fluidity and fast
+reactions of the FCIC give them a great advantage in this regard,
+which explains their success. Even with the best will in the world
+(which it does not, in fact, possess) it is impossible for an organization
+the size of the U.S. Federal Bureau of Investigation to get up to speed
+on the theory and practice of computer crime. If they tried to train all
+their agents to do this, it would be SUICIDAL, as they would NEVER BE ABLE
+TO DO ANYTHING ELSE.
+
+The FBI does try to train its agents in the basics of electronic crime,
+at their base in Quantico, Virginia. And the Secret Service, along with
+many other law enforcement groups, runs quite successful and well-attended
+training courses on wire fraud, business crime, and computer intrusion
+at the Federal Law Enforcement Training Center (FLETC, pronounced "fletsy")
+in Glynco, Georgia. But the best efforts of these bureaucracies does not
+remove the absolute need for a "cutting-edge mess" like the FCIC.
+
+For you see--the members of FCIC ARE the trainers of the rest
+of law enforcement. Practically and literally speaking,
+they are the Glynco computer-crime faculty by another name.
+If the FCIC went over a cliff on a bus, the U.S. law enforcement
+community would be rendered deaf dumb and blind in the world
+of computer crime, and would swiftly feel a desperate need
+to reinvent them. And this is no time to go starting from scratch.
+
+On June 11, 1991, I once again arrived in Phoenix, Arizona,
+for the latest meeting of the Federal Computer Investigations Committee.
+This was more or less the twentieth meeting of this stellar group.
+The count was uncertain, since nobody could figure out whether to
+include the meetings of "the Colluquy," which is what the FCIC
+was called in the mid-1980s before it had even managed to obtain
+the dignity of its own acronym.
+
+Since my last visit to Arizona, in May, the local AzScam bribery scandal
+had resolved itself in a general muddle of humiliation. The Phoenix chief
+of police, whose agents had videotaped nine state legislators up to no good,
+had resigned his office in a tussle with the Phoenix city council over
+the propriety of his undercover operations.
+
+The Phoenix Chief could now join Gail Thackeray and eleven of her closest
+associates in the shared experience of politically motivated unemployment.
+As of June, resignations were still continuing at the Arizona Attorney
+General's office, which could be interpreted as either a New Broom
+Sweeping Clean or a Night of the Long Knives Part II, depending on
+your point of view.
+
+The meeting of FCIC was held at the Scottsdale Hilton Resort.
+Scottsdale is a wealthy suburb of Phoenix, known as "Scottsdull"
+to scoffing local trendies, but well-equipped with posh shopping-malls
+and manicured lawns, while conspicuously undersupplied with homeless derelicts.
+The Scottsdale Hilton Resort was a sprawling hotel in postmodern
+crypto-Southwestern style. It featured a "mission bell tower"
+plated in turquoise tile and vaguely resembling a Saudi minaret.
+
+Inside it was all barbarically striped Santa Fe Style decor.
+There was a health spa downstairs and a large oddly-shaped
+pool in the patio. A poolside umbrella-stand offered Ben and Jerry's
+politically correct Peace Pops.
+
+I registered as a member of FCIC, attaining a handy discount rate,
+then went in search of the Feds. Sure enough, at the back of the
+hotel grounds came the unmistakable sound of Gail Thackeray
+holding forth.
+
+Since I had also attended the Computers Freedom and Privacy conference
+(about which more later), this was the second time I had seen Thackeray
+in a group of her law enforcement colleagues. Once again I was struck
+by how simply pleased they seemed to see her. It was natural that she'd
+get SOME attention, as Gail was one of two women in a group of some thirty men;
+but there was a lot more to it than that.
+
+Gail Thackeray personifies the social glue of the FCIC. They could give
+a damn about her losing her job with the Attorney General. They were sorry
+about it, of course, but hell, they'd all lost jobs. If they were the kind
+of guys who liked steady boring jobs, they would never have gotten into
+computer work in the first place.
+
+I wandered into her circle and was immediately introduced to five strangers.
+The conditions of my visit at FCIC were reviewed. I would not quote
+anyone directly. I would not tie opinions expressed to the agencies
+of the attendees. I would not (a purely hypothetical example)
+report the conversation of a guy from the Secret Service talking
+quite civilly to a guy from the FBI, as these two agencies NEVER
+talk to each other, and the IRS (also present, also hypothetical)
+NEVER TALKS TO ANYBODY.
+
+Worse yet, I was forbidden to attend the first conference. And I didn't.
+I have no idea what the FCIC was up to behind closed doors that afternoon.
+I rather suspect that they were engaging in a frank and thorough confession
+of their errors, goof-ups and blunders, as this has been a feature of every
+FCIC meeting since their legendary Memphis beer-bust of 1986. Perhaps the
+single greatest attraction of FCIC is that it is a place where you can go,
+let your hair down, and completely level with people who actually comprehend
+what you are talking about. Not only do they understand you, but they
+REALLY PAY ATTENTION, they are GRATEFUL FOR YOUR INSIGHTS, and they
+FORGIVE YOU, which in nine cases out of ten is something even your
+boss can't do, because as soon as you start talking "ROM," "BBS,"
+or "T-1 trunk," his eyes glaze over.
+
+I had nothing much to do that afternoon. The FCIC were beavering away
+in their conference room. Doors were firmly closed, windows too dark
+to peer through. I wondered what a real hacker, a computer intruder,
+would do at a meeting like this.
+
+The answer came at once. He would "trash" the place. Not reduce the place
+to trash in some orgy of vandalism; that's not the use of the term in the
+hacker milieu. No, he would quietly EMPTY THE TRASH BASKETS and silently
+raid any valuable data indiscreetly thrown away.
+
+Journalists have been known to do this. (Journalists hunting information
+have been known to do almost every single unethical thing that hackers
+have ever done. They also throw in a few awful techniques all their own.)
+The legality of `trashing' is somewhat dubious but it is not in fact
+flagrantly illegal. It was, however, absurd to contemplate trashing the FCIC.
+These people knew all about trashing. I wouldn't last fifteen seconds.
+
+The idea sounded interesting, though. I'd been hearing a lot about
+the practice lately. On the spur of the moment, I decided I would try
+trashing the office ACROSS THE HALL from the FCIC, an area which had
+nothing to do with the investigators.
+
+The office was tiny; six chairs, a table. . . . Nevertheless, it was open,
+so I dug around in its plastic trash can.
+
+To my utter astonishment, I came up with the torn scraps of a SPRINT
+long-distance phone bill. More digging produced a bank statement
+and the scraps of a hand-written letter, along with gum, cigarette ashes,
+candy wrappers and a day-old-issue of USA TODAY.
+
+The trash went back in its receptacle while the scraps of data went into
+my travel bag. I detoured through the hotel souvenir shop for some
+Scotch tape and went up to my room.
+
+Coincidence or not, it was quite true. Some poor soul had, in fact,
+thrown a SPRINT bill into the hotel's trash. Date May 1991,
+total amount due: $252.36. Not a business phone, either,
+but a residential bill, in the name of someone called Evelyn
+(not her real name). Evelyn's records showed a ## PAST DUE BILL ##!
+Here was her nine-digit account ID. Here was a stern computer-printed warning:
+
+"TREAT YOUR FONCARD AS YOU WOULD ANY CREDIT CARD. TO SECURE AGAINST FRAUD,
+NEVER GIVE YOUR FONCARD NUMBER OVER THE PHONE UNLESS YOU INITIATED THE CALL.
+IF YOU RECEIVE SUSPICIOUS CALLS PLEASE NOTIFY CUSTOMER SERVICE IMMEDIATELY!"
+
+I examined my watch. Still plenty of time left for the FCIC to carry on.
+I sorted out the scraps of Evelyn's SPRINT bill and re-assembled them with
+fresh Scotch tape. Here was her ten-digit FONCARD number. Didn't seem
+to have the ID number necessary to cause real fraud trouble.
+
+I did, however, have Evelyn's home phone number. And the phone numbers
+for a whole crowd of Evelyn's long-distance friends and acquaintances.
+In San Diego, Folsom, Redondo, Las Vegas, La Jolla, Topeka, and Northampton
+Massachusetts. Even somebody in Australia!
+
+I examined other documents. Here was a bank statement. It was Evelyn's
+IRA account down at a bank in San Mateo California (total balance $1877.20).
+Here was a charge-card bill for $382.64. She was paying it off bit by bit.
+
+Driven by motives that were completely unethical and prurient,
+I now examined the handwritten notes. They had been torn fairly
+thoroughly, so much so that it took me almost an entire five minutes
+to reassemble them.
+
+They were drafts of a love letter. They had been written on
+the lined stationery of Evelyn's employer, a biomedical company.
+Probably written at work when she should have been doing something else.
+
+"Dear Bob," (not his real name) "I guess in everyone's life there comes
+a time when hard decisions have to be made, and this is a difficult one
+for me--very upsetting. Since you haven't called me, and I don't understand
+why, I can only surmise it's because you don't want to. I thought I would
+have heard from you Friday. I did have a few unusual problems with my phone
+and possibly you tried, I hope so.
+
+"Robert, you asked me to `let go'. . . ."
+
+The first note ended. UNUSUAL PROBLEMS WITH HER PHONE?
+I looked swiftly at the next note.
+
+"Bob, not hearing from you for the whole weekend has left me very perplexed. . . ."
+
+Next draft.
+
+"Dear Bob, there is so much I don't understand right now, and I wish I did.
+I wish I could talk to you, but for some unknown reason you have elected not
+to call--this is so difficult for me to understand. . . ."
+
+She tried again.
+
+"Bob, Since I have always held you in such high esteem, I had every hope that
+we could remain good friends, but now one essential ingredient is missing--
+respect. Your ability to discard people when their purpose is served is
+appalling to me. The kindest thing you could do for me now is to leave me
+alone. You are no longer welcome in my heart or home. . . ."
+
+Try again.
+
+"Bob, I wrote a very factual note to you to say how much respect I had lost
+for you, by the way you treat people, me in particular, so uncaring and cold.
+The kindest thing you can do for me is to leave me alone entirely,
+as you are no longer welcome in my heart or home. I would appreciate it
+if you could retire your debt to me as soon as possible--I wish no link
+to you in any way. Sincerely, Evelyn."
+
+Good heavens, I thought, the bastard actually owes her money!
+I turned to the next page.
+
+"Bob: very simple. GOODBYE! No more mind games--no more fascination--
+no more coldness--no more respect for you! It's over--Finis. Evie"
+
+There were two versions of the final brushoff letter, but they read about
+the same. Maybe she hadn't sent it. The final item in my illicit and
+shameful booty was an envelope addressed to "Bob" at his home address,
+but it had no stamp on it and it hadn't been mailed.
+
+Maybe she'd just been blowing off steam because her rascal boyfriend
+had neglected to call her one weekend. Big deal. Maybe they'd kissed
+and made up, maybe she and Bob were down at Pop's Chocolate Shop now,
+sharing a malted. Sure.
+
+Easy to find out. All I had to do was call Evelyn up. With a half-clever
+story and enough brass-plated gall I could probably trick the truth out of her.
+Phone-phreaks and hackers deceive people over the phone all the time.
+It's called "social engineering." Social engineering is a very common practice
+in the underground, and almost magically effective. Human beings are almost
+always the weakest link in computer security. The simplest way to learn
+Things You Are Not Meant To Know is simply to call up and exploit the
+knowledgeable people. With social engineering, you use the bits of specialized
+knowledge you already have as a key, to manipulate people into believing
+that you are legitimate. You can then coax, flatter, or frighten them into
+revealing almost anything you want to know. Deceiving people (especially
+over the phone) is easy and fun. Exploiting their gullibility is very
+gratifying; it makes you feel very superior to them.
+
+If I'd been a malicious hacker on a trashing raid, I would now have Evelyn
+very much in my power. Given all this inside data, it wouldn't take much
+effort at all to invent a convincing lie. If I were ruthless enough,
+and jaded enough, and clever enough, this momentary indiscretion of hers--
+maybe committed in tears, who knows--could cause her a whole world of
+confusion and grief.
+
+I didn't even have to have a MALICIOUS motive. Maybe I'd be "on her side,"
+and call up Bob instead, and anonymously threaten to break both his kneecaps
+if he didn't take Evelyn out for a steak dinner pronto. It was still
+profoundly NONE OF MY BUSINESS. To have gotten this knowledge at all
+was a sordid act and to use it would be to inflict a sordid injury.
+
+To do all these awful things would require exactly zero high-tech expertise.
+All it would take was the willingness to do it and a certain amount
+of bent imagination.
+
+I went back downstairs. The hard-working FCIC, who had labored forty-five
+minutes over their schedule, were through for the day, and adjourned to the
+hotel bar. We all had a beer.
+
+I had a chat with a guy about "Isis," or rather IACIS,
+the International Association of Computer Investigation Specialists.
+They're into "computer forensics," the techniques of picking computer-
+systems apart without destroying vital evidence. IACIS, currently run
+out of Oregon, is comprised of investigators in the U.S., Canada, Taiwan
+and Ireland. "Taiwan and Ireland?" I said. Are TAIWAN and IRELAND
+really in the forefront of this stuff? Well not exactly, my informant
+admitted. They just happen to have been the first ones to have caught
+on by word of mouth. Still, the international angle counts, because this
+is obviously an international problem. Phone-lines go everywhere.
+
+There was a Mountie here from the Royal Canadian Mounted Police.
+He seemed to be having quite a good time. Nobody had flung this
+Canadian out because he might pose a foreign security risk.
+These are cyberspace cops. They still worry a lot about "jurisdictions,"
+but mere geography is the least of their troubles.
+
+NASA had failed to show. NASA suffers a lot from computer intrusions,
+in particular from Australian raiders and a well-trumpeted Chaos
+Computer Club case, and in 1990 there was a brief press flurry
+when it was revealed that one of NASA's Houston branch-exchanges
+had been systematically ripped off by a gang of phone-phreaks.
+But the NASA guys had had their funding cut. They were stripping everything.
+
+Air Force OSI, its Office of Special Investigations, is the ONLY federal
+entity dedicated full-time to computer security. They'd been expected
+to show up in force, but some of them had cancelled--a Pentagon budget pinch.
+
+As the empties piled up, the guys began joshing around and telling war-stories.
+"These are cops," Thackeray said tolerantly. "If they're not talking shop
+they talk about women and beer."
+
+I heard the story about the guy who, asked for "a copy" of a computer disk,
+PHOTOCOPIED THE LABEL ON IT. He put the floppy disk onto the glass plate
+of a photocopier. The blast of static when the copier worked completely
+erased all the real information on the disk.
+
+Some other poor souls threw a whole bag of confiscated diskettes
+into the squad-car trunk next to the police radio. The powerful radio
+signal blasted them, too.
+
+We heard a bit about Dave Geneson, the first computer prosecutor,
+a mainframe-runner in Dade County, turned lawyer. Dave Geneson
+was one guy who had hit the ground running, a signal virtue
+in making the transition to computer-crime. It was generally
+agreed that it was easier to learn the world of computers first,
+then police or prosecutorial work. You could take certain computer
+people and train 'em to successful police work--but of course they
+had to have the COP MENTALITY. They had to have street smarts.
+Patience. Persistence. And discretion. You've got to make sure
+they're not hot-shots, show-offs, "cowboys."
+
+Most of the folks in the bar had backgrounds in military intelligence,
+or drugs, or homicide. It was rudely opined that "military intelligence"
+was a contradiction in terms, while even the grisly world of homicide
+was considered cleaner than drug enforcement. One guy had been 'way
+undercover doing dope-work in Europe for four years straight.
+"I'm almost recovered now," he said deadpan, with the acid black humor
+that is pure cop. "Hey, now I can say FUCKER without putting MOTHER
+in front of it."
+
+"In the cop world," another guy said earnestly, "everything is good and bad,
+black and white. In the computer world everything is gray."
+
+One guy--a founder of the FCIC, who'd been with the group
+since it was just the Colluquy--described his own introduction
+to the field. He'd been a Washington DC homicide guy called in
+on a "hacker" case. From the word "hacker," he naturally assumed
+he was on the trail of a knife-wielding marauder, and went to the
+computer center expecting blood and a body. When he finally figured
+out what was happening there (after loudly demanding, in vain,
+that the programmers "speak English"), he called headquarters
+and told them he was clueless about computers. They told him nobody
+else knew diddly either, and to get the hell back to work.
+
+So, he said, he had proceeded by comparisons. By analogy. By metaphor.
+"Somebody broke in to your computer, huh?" Breaking and entering;
+I can understand that. How'd he get in? "Over the phone-lines."
+Harassing phone-calls, I can understand that! What we need here
+is a tap and a trace!
+
+It worked. It was better than nothing. And it worked a lot faster
+when he got hold of another cop who'd done something similar.
+And then the two of them got another, and another, and pretty soon
+the Colluquy was a happening thing. It helped a lot that everybody
+seemed to know Carlton Fitzpatrick, the data-processing trainer in Glynco.
+
+The ice broke big-time in Memphis in '86. The Colluquy had attracted
+a bunch of new guys--Secret Service, FBI, military, other feds, heavy guys.
+Nobody wanted to tell anybody anything. They suspected that if word got back
+to the home office they'd all be fired. They passed an uncomfortably
+guarded afternoon.
+
+The formalities got them nowhere. But after the formal session was over,
+the organizers brought in a case of beer. As soon as the participants
+knocked it off with the bureaucratic ranks and turf-fighting, everything
+changed. "I bared my soul," one veteran reminisced proudly. By nightfall
+they were building pyramids of empty beer-cans and doing everything
+but composing a team fight song.
+
+FCIC were not the only computer-crime people around. There was DATTA
+(District Attorneys' Technology Theft Association), though they mostly
+specialized in chip theft, intellectual property, and black-market cases.
+There was HTCIA (High Tech Computer Investigators Association),
+also out in Silicon Valley, a year older than FCIC and featuring
+brilliant people like Donald Ingraham. There was LEETAC
+(Law Enforcement Electronic Technology Assistance Committee)
+in Florida, and computer-crime units in Illinois and Maryland
+and Texas and Ohio and Colorado and Pennsylvania. But these were
+local groups. FCIC were the first to really network nationally
+and on a federal level.
+
+FCIC people live on the phone lines. Not on bulletin board systems--
+they know very well what boards are, and they know that boards aren't secure.
+Everyone in the FCIC has a voice-phone bill like you wouldn't believe.
+FCIC people have been tight with the telco people for a long time.
+Telephone cyberspace is their native habitat.
+
+FCIC has three basic sub-tribes: the trainers, the security people,
+and the investigators. That's why it's called an "Investigations
+Committee" with no mention of the term "computer-crime"--the dreaded
+"C-word." FCIC, officially, is "an association of agencies rather
+than individuals;" unofficially, this field is small enough that
+the influence of individuals and individual expertise is paramount.
+Attendance is by invitation only, and most everyone in FCIC considers
+himself a prophet without honor in his own house.
+
+Again and again I heard this, with different terms but identical
+sentiments. "I'd been sitting in the wilderness talking to myself."
+"I was totally isolated." "I was desperate." "FCIC is the best
+thing there is about computer crime in America." "FCIC is what
+really works." "This is where you hear real people telling you
+what's really happening out there, not just lawyers picking nits."
+"We taught each other everything we knew."
+
+The sincerity of these statements convinces me that this is true.
+FCIC is the real thing and it is invaluable. It's also very sharply
+at odds with the rest of the traditions and power structure
+in American law enforcement. There probably hasn't been anything
+around as loose and go-getting as the FCIC since the start of the
+U.S. Secret Service in the 1860s. FCIC people are living like
+twenty-first-century people in a twentieth-century environment,
+and while there's a great deal to be said for that, there's also
+a great deal to be said against it, and those against it happen
+to control the budgets.
+
+I listened to two FCIC guys from Jersey compare life histories.
+One of them had been a biker in a fairly heavy-duty gang in the 1960s.
+"Oh, did you know so-and-so?" said the other guy from Jersey.
+"Big guy, heavyset?"
+
+"Yeah, I knew him."
+
+"Yeah, he was one of ours. He was our plant in the gang."
+
+"Really? Wow! Yeah, I knew him. Helluva guy."
+
+Thackeray reminisced at length about being tear-gassed blind
+in the November 1969 antiwar protests in Washington Circle,
+covering them for her college paper. "Oh yeah, I was there,"
+said another cop. "Glad to hear that tear gas hit somethin'.
+Haw haw haw." He'd been so blind himself, he confessed,
+that later that day he'd arrested a small tree.
+
+FCIC are an odd group, sifted out by coincidence and necessity,
+and turned into a new kind of cop. There are a lot of specialized
+cops in the world--your bunco guys, your drug guys, your tax guys,
+but the only group that matches FCIC for sheer isolation are probably
+the child-pornography people. Because they both deal with conspirators
+who are desperate to exchange forbidden data and also desperate to hide;
+and because nobody else in law enforcement even wants to hear about it.
+
+FCIC people tend to change jobs a lot. They tend not to get the equipment
+and training they want and need. And they tend to get sued quite often.
+
+As the night wore on and a band set up in the bar, the talk grew darker.
+Nothing ever gets done in government, someone opined, until there's
+a DISASTER. Computing disasters are awful, but there's no denying
+that they greatly help the credibility of FCIC people. The Internet Worm,
+for instance. "For years we'd been warning about that--but it's nothing
+compared to what's coming." They expect horrors, these people.
+They know that nothing will really get done until there is a horror.
+
+#
+
+Next day we heard an extensive briefing from a guy who'd been a computer cop,
+gotten into hot water with an Arizona city council, and now installed
+computer networks for a living (at a considerable rise in pay).
+He talked about pulling fiber-optic networks apart.
+
+Even a single computer, with enough peripherals, is a literal
+"network"--a bunch of machines all cabled together, generally
+with a complexity that puts stereo units to shame. FCIC people
+invent and publicize methods of seizing computers and maintaining
+their evidence. Simple things, sometimes, but vital rules of thumb
+for street cops, who nowadays often stumble across a busy computer
+in the midst of a drug investigation or a white-collar bust.
+For instance: Photograph the system before you touch it.
+Label the ends of all the cables before you detach anything.
+"Park" the heads on the disk drives before you move them.
+Get the diskettes. Don't put the diskettes in magnetic fields.
+Don't write on diskettes with ballpoint pens. Get the manuals.
+Get the printouts. Get the handwritten notes. Copy data before
+you look at it, and then examine the copy instead of the original.
+
+Now our lecturer distributed copied diagrams of a typical LAN
+or "Local Area Network", which happened to be out of Connecticut.
+ONE HUNDRED AND FIFTY-NINE desktop computers, each with its own
+peripherals. Three "file servers." Five "star couplers"
+each with thirty-two ports. One sixteen-port coupler
+off in the corner office. All these machines talking to each other,
+distributing electronic mail, distributing software, distributing,
+quite possibly, criminal evidence. All linked by high-capacity
+fiber-optic cable. A bad guy--cops talk a about "bad guys"
+--might be lurking on PC #47 lot or #123 and distributing
+his ill doings onto some dupe's "personal" machine in
+another office--or another floor--or, quite possibly,
+two or three miles away! Or, conceivably, the evidence might
+be "data-striped"--split up into meaningless slivers stored,
+one by one, on a whole crowd of different disk drives.
+
+The lecturer challenged us for solutions. I for one was utterly clueless.
+As far as I could figure, the Cossacks were at the gate; there were probably
+more disks in this single building than were seized during the entirety
+of Operation Sundevil.
+
+"Inside informant," somebody said. Right. There's always the human angle,
+something easy to forget when contemplating the arcane recesses of high
+technology. Cops are skilled at getting people to talk, and computer people,
+given a chair and some sustained attention, will talk about their computers
+till their throats go raw. There's a case on record of a single question--
+"How'd you do it?"--eliciting a forty-five-minute videotaped confession
+from a computer criminal who not only completely incriminated himself
+but drew helpful diagrams.
+
+Computer people talk. Hackers BRAG. Phone-phreaks
+talk PATHOLOGICALLY--why else are they stealing phone-codes,
+if not to natter for ten hours straight to their friends
+on an opposite seaboard? Computer-literate people do
+in fact possess an arsenal of nifty gadgets and techniques
+that would allow them to conceal all kinds of exotic skullduggery,
+and if they could only SHUT UP about it, they could probably
+get away with all manner of amazing information-crimes.
+But that's just not how it works--or at least,
+that's not how it's worked SO FAR.
+
+Most every phone-phreak ever busted has swiftly implicated his mentors,
+his disciples, and his friends. Most every white-collar computer-criminal,
+smugly convinced that his clever scheme is bulletproof, swiftly learns
+otherwise when, for the first time in his life, an actual no-kidding
+policeman leans over, grabs the front of his shirt, looks him right
+in the eye and says: "All right, ASSHOLE--you and me are going downtown!"
+All the hardware in the world will not insulate your nerves from
+these actual real-life sensations of terror and guilt.
+
+Cops know ways to get from point A to point Z without thumbing
+through every letter in some smart-ass bad-guy's alphabet.
+Cops know how to cut to the chase. Cops know a lot of things
+other people don't know.
+
+Hackers know a lot of things other people don't know, too.
+Hackers know, for instance, how to sneak into your computer
+through the phone-lines. But cops can show up RIGHT ON YOUR DOORSTEP
+and carry off YOU and your computer in separate steel boxes.
+A cop interested in hackers can grab them and grill them.
+A hacker interested in cops has to depend on hearsay,
+underground legends, and what cops are willing to publicly reveal.
+And the Secret Service didn't get named "the SECRET Service"
+because they blab a lot.
+
+Some people, our lecturer informed us, were under the mistaken
+impression that it was "impossible" to tap a fiber-optic line.
+Well, he announced, he and his son had just whipped up a
+fiber-optic tap in his workshop at home. He passed it around
+the audience, along with a circuit-covered LAN plug-in card
+so we'd all recognize one if we saw it on a case. We all had a look.
+
+The tap was a classic "Goofy Prototype"--a thumb-length rounded
+metal cylinder with a pair of plastic brackets on it.
+From one end dangled three thin black cables, each of which ended
+in a tiny black plastic cap. When you plucked the safety-cap
+off the end of a cable, you could see the glass fiber--
+no thicker than a pinhole.
+
+Our lecturer informed us that the metal cylinder was a
+"wavelength division multiplexer." Apparently, what one did
+was to cut the fiber-optic cable, insert two of the legs into
+the cut to complete the network again, and then read any passing data
+on the line by hooking up the third leg to some kind of monitor.
+Sounded simple enough. I wondered why nobody had thought of it before.
+I also wondered whether this guy's son back at the workshop had any
+teenage friends.
+
+We had a break. The guy sitting next to me was wearing a giveaway
+baseball cap advertising the Uzi submachine gun. We had a desultory chat
+about the merits of Uzis. Long a favorite of the Secret Service,
+it seems Uzis went out of fashion with the advent of the Persian Gulf War,
+our Arab allies taking some offense at Americans toting Israeli weapons.
+Besides, I was informed by another expert, Uzis jam. The equivalent weapon
+of choice today is the Heckler & Koch, manufactured in Germany.
+
+The guy with the Uzi cap was a forensic photographer. He also did a lot
+of photographic surveillance work in computer crime cases. He used to,
+that is, until the firings in Phoenix. He was now a private investigator and,
+with his wife, ran a photography salon specializing in weddings and portrait
+photos. At--one must repeat--a considerable rise in income.
+
+He was still FCIC. If you were FCIC, and you needed to talk
+to an expert about forensic photography, well, there he was,
+willing and able. If he hadn't shown up, people would have missed him.
+
+Our lecturer had raised the point that preliminary investigation
+of a computer system is vital before any seizure is undertaken.
+It's vital to understand how many machines are in there, what kinds
+there are, what kind of operating system they use, how many people
+use them, where the actual data itself is stored. To simply barge into
+an office demanding "all the computers" is a recipe for swift disaster.
+
+This entails some discreet inquiries beforehand. In fact, what it
+entails is basically undercover work. An intelligence operation.
+SPYING, not to put too fine a point on it.
+
+In a chat after the lecture, I asked an attendee whether "trashing" might work.
+
+I received a swift briefing on the theory and practice of "trash covers."
+Police "trash covers," like "mail covers" or like wiretaps, require the
+agreement of a judge. This obtained, the "trashing" work of cops is just
+like that of hackers, only more so and much better organized. So much so,
+I was informed, that mobsters in Phoenix make extensive use of locked
+garbage cans picked up by a specialty high-security trash company.
+
+In one case, a tiger team of Arizona cops had trashed a local residence
+for four months. Every week they showed up on the municipal garbage truck,
+disguised as garbagemen, and carried the contents of the suspect cans off
+to a shade tree, where they combed through the garbage--a messy task,
+especially considering that one of the occupants was undergoing
+kidney dialysis. All useful documents were cleaned, dried and examined.
+A discarded typewriter-ribbon was an especially valuable source of data,
+as its long one-strike ribbon of film contained the contents of every
+letter mailed out of the house. The letters were neatly retyped by
+a police secretary equipped with a large desk-mounted magnifying glass.
+
+There is something weirdly disquieting about the whole subject of
+"trashing"-- an unsuspected and indeed rather disgusting mode of
+deep personal vulnerability. Things that we pass by every day,
+that we take utterly for granted, can be exploited with so little work.
+Once discovered, the knowledge of these vulnerabilities tend to spread.
+
+Take the lowly subject of MANHOLE COVERS. The humble manhole cover
+reproduces many of the dilemmas of computer-security in miniature.
+Manhole covers are, of course, technological artifacts, access-points
+to our buried urban infrastructure. To the vast majority of us,
+manhole covers are invisible. They are also vulnerable. For many years now,
+the Secret Service has made a point of caulking manhole covers along all routes
+of the Presidential motorcade. This is, of course, to deter terrorists from
+leaping out of underground ambush or, more likely, planting remote-control
+car-smashing bombs beneath the street.
+
+Lately, manhole covers have seen more and more criminal exploitation,
+especially in New York City. Recently, a telco in New York City
+discovered that a cable television service had been sneaking into
+telco manholes and installing cable service alongside the phone-lines--
+WITHOUT PAYING ROYALTIES. New York companies have also suffered a
+general plague of (a) underground copper cable theft; (b) dumping of garbage,
+including toxic waste, and (c) hasty dumping of murder victims.
+
+Industry complaints reached the ears of an innovative New England
+industrial-security company, and the result was a new product known
+as "the Intimidator," a thick titanium-steel bolt with a precisely machined
+head that requires a special device to unscrew. All these "keys" have registered
+serial numbers kept on file with the manufacturer. There are now some
+thousands of these "Intimidator" bolts being sunk into American pavements
+wherever our President passes, like some macabre parody of strewn roses.
+They are also spreading as fast as steel dandelions around US military bases
+and many centers of private industry.
+
+Quite likely it has never occurred to you to peer under a manhole cover,
+perhaps climb down and walk around down there with a flashlight, just to see
+what it's like. Formally speaking, this might be trespassing, but if you
+didn't hurt anything, and didn't make an absolute habit of it, nobody would
+really care. The freedom to sneak under manholes was likely a freedom
+you never intended to exercise.
+
+You now are rather less likely to have that freedom at all.
+You may never even have missed it until you read about it here,
+but if you're in New York City it's gone, and elsewhere it's likely going.
+This is one of the things that crime, and the reaction to
+crime, does to us.
+
+The tenor of the meeting now changed as the Electronic Frontier Foundation
+arrived. The EFF, whose personnel and history will be examined in detail
+in the next chapter, are a pioneering civil liberties group who arose in
+direct response to the Hacker Crackdown of 1990.
+
+Now Mitchell Kapor, the Foundation's president, and Michael Godwin,
+its chief attorney, were confronting federal law enforcement MANO A MANO
+for the first time ever. Ever alert to the manifold uses of publicity,
+Mitch Kapor and Mike Godwin had brought their own journalist in tow:
+Robert Draper, from Austin, whose recent well-received book about
+ROLLING STONE magazine was still on the stands. Draper was on assignment
+for TEXAS MONTHLY.
+
+The Steve Jackson/EFF civil lawsuit against the Chicago Computer Fraud
+and Abuse Task Force was a matter of considerable regional interest in Texas.
+There were now two Austinite journalists here on the case. In fact,
+counting Godwin (a former Austinite and former journalist) there were
+three of us. Lunch was like Old Home Week.
+
+Later, I took Draper up to my hotel room. We had a long frank talk
+about the case, networking earnestly like a miniature freelance-journo
+version of the FCIC: privately confessing the numerous blunders
+of journalists covering the story, and trying hard to figure out
+who was who and what the hell was really going on out there.
+I showed Draper everything I had dug out of the Hilton trashcan.
+We pondered the ethics of "trashing" for a while, and agreed
+that they were dismal. We also agreed that finding a SPRINT
+bill on your first time out was a heck of a coincidence.
+
+First I'd "trashed"--and now, mere hours later, I'd bragged to someone else.
+Having entered the lifestyle of hackerdom, I was now, unsurprisingly,
+following its logic. Having discovered something remarkable through
+a surreptitious action, I of course HAD to "brag," and to drag the passing
+Draper into my iniquities. I felt I needed a witness. Otherwise nobody
+would have believed what I'd discovered. . . .
+
+Back at the meeting, Thackeray cordially, if rather tentatively,
+introduced Kapor and Godwin to her colleagues. Papers were distributed.
+Kapor took center stage. The brilliant Bostonian high-tech entrepreneur,
+normally the hawk in his own administration and quite an effective
+public speaker, seemed visibly nervous, and frankly admitted as much.
+He began by saying he consided computer-intrusion to be morally wrong,
+and that the EFF was not a "hacker defense fund," despite what had appeared
+in print. Kapor chatted a bit about the basic motivations of his group,
+emphasizing their good faith and willingness to listen and seek common ground
+with law enforcement--when, er, possible.
+
+Then, at Godwin's urging, Kapor suddenly remarked that EFF's own Internet
+machine had been "hacked" recently, and that EFF did not consider
+this incident amusing.
+
+After this surprising confession, things began to loosen up
+quite rapidly. Soon Kapor was fielding questions, parrying objections,
+challenging definitions, and juggling paradigms with something akin
+to his usual gusto.
+
+Kapor seemed to score quite an effect with his shrewd and skeptical analysis
+of the merits of telco "Caller-ID" services. (On this topic, FCIC and EFF
+have never been at loggerheads, and have no particular established earthworks
+to defend.) Caller-ID has generally been promoted as a privacy service
+for consumers, a presentation Kapor described as a "smokescreen,"
+the real point of Caller-ID being to ALLOW CORPORATE CUSTOMERS TO BUILD
+EXTENSIVE COMMERCIAL DATABASES ON EVERYBODY WHO PHONES OR FAXES THEM.
+Clearly, few people in the room had considered this possibility,
+except perhaps for two late-arrivals from US WEST RBOC security,
+who chuckled nervously.
+
+Mike Godwin then made an extensive presentation on
+"Civil Liberties Implications of Computer Searches and Seizures."
+Now, at last, we were getting to the real nitty-gritty here,
+real political horse-trading. The audience listened with close
+attention, angry mutters rising occasionally: "He's trying to
+teach us our jobs!" "We've been thinking about this for years!
+We think about these issues every day!" "If I didn't seize the works,
+I'd be sued by the guy's victims!" "I'm violating the law if I leave
+ten thousand disks full of illegal PIRATED SOFTWARE and STOLEN CODES!"
+"It's our job to make sure people don't trash the Constitution--
+we're the DEFENDERS of the Constitution!" "We seize stuff when
+we know it will be forfeited anyway as restitution for the victim!"
+
+"If it's forfeitable, then don't get a search warrant, get a
+forfeiture warrant," Godwin suggested coolly. He further remarked
+that most suspects in computer crime don't WANT to see their computers
+vanish out the door, headed God knew where, for who knows how long.
+They might not mind a search, even an extensive search, but they want
+their machines searched on-site.
+
+"Are they gonna feed us?" somebody asked sourly.
+
+"How about if you take copies of the data?" Godwin parried.
+
+"That'll never stand up in court."
+
+"Okay, you make copies, give THEM the copies, and take the originals."
+
+Hmmm.
+
+Godwin championed bulletin-board systems as repositories of First Amendment
+protected free speech. He complained that federal computer-crime training
+manuals gave boards a bad press, suggesting that they are hotbeds of crime
+haunted by pedophiles and crooks, whereas the vast majority of the nation's
+thousands of boards are completely innocuous, and nowhere near so
+romantically suspicious.
+
+People who run boards violently resent it when their systems are seized,
+and their dozens (or hundreds) of users look on in abject horror.
+Their rights of free expression are cut short. Their right to associate
+with other people is infringed. And their privacy is violated as their
+private electronic mail becomes police property.
+
+Not a soul spoke up to defend the practice of seizing boards.
+The issue passed in chastened silence. Legal principles aside--
+(and those principles cannot be settled without laws passed or
+court precedents)--seizing bulletin boards has become public-relations
+poison for American computer police.
+
+And anyway, it's not entirely necessary. If you're a cop, you can get 'most
+everything you need from a pirate board, just by using an inside informant.
+Plenty of vigilantes--well, CONCERNED CITIZENS--will inform police the moment
+they see a pirate board hit their area (and will tell the police all about it,
+in such technical detail, actually, that you kinda wish they'd shut up).
+They will happily supply police with extensive downloads or printouts.
+It's IMPOSSIBLE to keep this fluid electronic information out of the
+hands of police.
+
+Some people in the electronic community become enraged at the prospect
+of cops "monitoring" bulletin boards. This does have touchy aspects,
+as Secret Service people in particular examine bulletin boards with
+some regularity. But to expect electronic police to be deaf dumb
+and blind in regard to this particular medium rather flies in the face
+of common sense. Police watch television, listen to radio, read newspapers
+and magazines; why should the new medium of boards be different?
+Cops can exercise the same access to electronic information
+as everybody else. As we have seen, quite a few computer
+police maintain THEIR OWN bulletin boards, including anti-hacker
+"sting" boards, which have generally proven quite effective.
+
+As a final clincher, their Mountie friends in Canada (and colleagues
+in Ireland and Taiwan) don't have First Amendment or American
+constitutional restrictions, but they do have phone lines,
+and can call any bulletin board in America whenever they please.
+The same technological determinants that play into the hands of hackers,
+phone phreaks and software pirates can play into the hands of police.
+"Technological determinants" don't have ANY human allegiances.
+They're not black or white, or Establishment or Underground,
+or pro-or-anti anything.
+
+Godwin complained at length about what he called "the Clever Hobbyist
+hypothesis" --the assumption that the "hacker" you're busting is clearly
+a technical genius, and must therefore by searched with extreme thoroughness.
+So: from the law's point of view, why risk missing anything? Take the works.
+Take the guy's computer. Take his books. Take his notebooks.
+Take the electronic drafts of his love letters. Take his Walkman.
+Take his wife's computer. Take his dad's computer. Take his kid
+sister's computer. Take his employer's computer. Take his compact disks--
+they MIGHT be CD-ROM disks, cunningly disguised as pop music.
+Take his laser printer--he might have hidden something vital in the
+printer's 5meg of memory. Take his software manuals and hardware
+documentation. Take his science-fiction novels and his simulation-
+gaming books. Take his Nintendo Game-Boy and his Pac-Man arcade game.
+Take his answering machine, take his telephone out of the wall.
+Take anything remotely suspicious.
+
+Godwin pointed out that most "hackers" are not, in fact, clever
+genius hobbyists. Quite a few are crooks and grifters who don't
+have much in the way of technical sophistication; just some rule-of-thumb
+rip-off techniques. The same goes for most fifteen-year-olds who've
+downloaded a code-scanning program from a pirate board. There's no
+real need to seize everything in sight. It doesn't require an entire
+computer system and ten thousand disks to prove a case in court.
+
+What if the computer is the instrumentality of a crime? someone demanded.
+
+Godwin admitted quietly that the doctrine of seizing the instrumentality
+of a crime was pretty well established in the American legal system.
+
+The meeting broke up. Godwin and Kapor had to leave. Kapor was testifying
+next morning before the Massachusetts Department Of Public Utility,
+about ISDN narrowband wide-area networking.
+
+As soon as they were gone, Thackeray seemed elated.
+She had taken a great risk with this. Her colleagues had not,
+in fact, torn Kapor and Godwin's heads off. She was very proud of them,
+and told them so.
+
+"Did you hear what Godwin said about INSTRUMENTALITY OF A CRIME?"
+she exulted, to nobody in particular. "Wow, that means
+MITCH ISN'T GOING TO SUE ME."
+
+#
+
+America's computer police are an interesting group.
+As a social phenomenon they are far more interesting,
+and far more important, than teenage phone phreaks
+and computer hackers. First, they're older and wiser;
+not dizzy hobbyists with leaky morals, but seasoned adult
+professionals with all the responsibilities of public service.
+And, unlike hackers, they possess not merely TECHNICAL
+power alone, but heavy-duty legal and social authority.
+
+And, very interestingly, they are just as much at
+sea in cyberspace as everyone else. They are not
+happy about this. Police are authoritarian by nature,
+and prefer to obey rules and precedents. (Even those police
+who secretly enjoy a fast ride in rough territory will soberly
+disclaim any "cowboy" attitude.) But in cyberspace there ARE
+no rules and precedents. They are groundbreaking pioneers,
+Cyberspace Rangers, whether they like it or not.
+
+In my opinion, any teenager enthralled by computers,
+fascinated by the ins and outs of computer security,
+and attracted by the lure of specialized forms of knowledge and power,
+would do well to forget all about "hacking" and set his (or her)
+sights on becoming a fed. Feds can trump hackers at almost every
+single thing hackers do, including gathering intelligence,
+undercover disguise, trashing, phone-tapping, building dossiers,
+networking, and infiltrating computer systems--CRIMINAL computer systems.
+Secret Service agents know more about phreaking, coding and carding
+than most phreaks can find out in years, and when it comes to viruses,
+break-ins, software bombs and trojan horses, Feds have direct access to red-hot
+confidential information that is only vague rumor in the underground.
+
+And if it's an impressive public rep you're after, there are few people
+in the world who can be so chillingly impressive as a well-trained,
+well-armed United States Secret Service agent.
+
+Of course, a few personal sacrifices are necessary in order to obtain
+that power and knowledge. First, you'll have the galling discipline
+of belonging to a large organization; but the world of computer crime
+is still so small, and so amazingly fast-moving, that it will remain
+spectacularly fluid for years to come. The second sacrifice is that
+you'll have to give up ripping people off. This is not a great loss.
+Abstaining from the use of illegal drugs, also necessary, will be a boon
+to your health.
+
+A career in computer security is not a bad choice for a young man
+or woman today. The field will almost certainly expand drastically
+in years to come. If you are a teenager today, by the time you
+become a professional, the pioneers you have read about in this book
+will be the grand old men and women of the field, swamped by their many
+disciples and successors. Of course, some of them, like William P. Wood
+of the 1865 Secret Service, may well be mangled in the whirring machinery
+of legal controversy; but by the time you enter the computer-crime field,
+it may have stabilized somewhat, while remaining entertainingly challenging.
+
+But you can't just have a badge. You have to win it. First, there's the
+federal law enforcement training. And it's hard--it's a challenge.
+A real challenge--not for wimps and rodents.
+
+Every Secret Service agent must complete gruelling courses at the
+Federal Law Enforcement Training Center. (In fact, Secret Service
+agents are periodically re-trained during their entire careers.)
+
+In order to get a glimpse of what this might be like,
+I myself travelled to FLETC.
+
+#
+
+The Federal Law Enforcement Training Center is a 1500-acre facility
+on Georgia's Atlantic coast. It's a milieu of marshgrass, seabirds,
+damp, clinging sea-breezes, palmettos, mosquitos, and bats.
+Until 1974, it was a Navy Air Base, and still features a working runway,
+and some WWII vintage blockhouses and officers' quarters.
+The Center has since benefitted by a forty-million-dollar retrofit,
+but there's still enough forest and swamp on the facility for the
+Border Patrol to put in tracking practice.
+
+As a town, "Glynco" scarcely exists. The nearest real town is Brunswick,
+a few miles down Highway 17, where I stayed at the aptly named Marshview
+Holiday Inn. I had Sunday dinner at a seafood restaurant called "Jinright's,"
+where I feasted on deep-fried alligator tail. This local favorite was
+a heaped basket of bite-sized chunks of white, tender, almost fluffy
+reptile meat, steaming in a peppered batter crust. Alligator makes
+a culinary experience that's hard to forget, especially when liberally
+basted with homemade cocktail sauce from a Jinright squeeze-bottle.
+
+The crowded clientele were tourists, fishermen, local black folks
+in their Sunday best, and white Georgian locals who all seemed
+to bear an uncanny resemblance to Georgia humorist Lewis Grizzard.
+
+The 2,400 students from 75 federal agencies who make up the FLETC
+population scarcely seem to make a dent in the low-key local scene.
+The students look like tourists, and the teachers seem to have taken
+on much of the relaxed air of the Deep South. My host was Mr. Carlton
+Fitzpatrick, the Program Coordinator of the Financial Fraud Institute.
+Carlton Fitzpatrick is a mustached, sinewy, well-tanned Alabama native
+somewhere near his late forties, with a fondness for chewing tobacco,
+powerful computers, and salty, down-home homilies. We'd met before,
+at FCIC in Arizona.
+
+The Financial Fraud Institute is one of the nine divisions at FLETC.
+Besides Financial Fraud, there's Driver & Marine, Firearms,
+and Physical Training. These are specialized pursuits.
+There are also five general training divisions: Basic Training,
+Operations, Enforcement Techniques, Legal Division, and Behavioral Science.
+
+Somewhere in this curriculum is everything necessary to turn green college
+graduates into federal agents. First they're given ID cards. Then they get
+the rather miserable-looking blue coveralls known as "smurf suits."
+The trainees are assigned a barracks and a cafeteria, and immediately
+set on FLETC's bone-grinding physical training routine. Besides the
+obligatory daily jogging--(the trainers run up danger flags beside
+the track when the humidity rises high enough to threaten heat stroke)--
+here's the Nautilus machines, the martial arts, the survival skills. . . .
+
+The eighteen federal agencies who maintain on-site academies at FLETC
+employ a wide variety of specialized law enforcement units, some of them
+rather arcane. There's Border Patrol, IRS Criminal Investigation Division,
+Park Service, Fish and Wildlife, Customs, Immigration, Secret Service and
+the Treasury's uniformed subdivisions. . . . If you're a federal cop
+and you don't work for the FBI, you train at FLETC. This includes people
+as apparently obscure as the agents of the Railroad Retirement Board
+Inspector General. Or the Tennessee Valley Authority Police,
+who are in fact federal police officers, and can and do arrest criminals
+on the federal property of the Tennessee Valley Authority.
+
+And then there are the computer-crime people. All sorts, all backgrounds.
+Mr. Fitzpatrick is not jealous of his specialized knowledge. Cops all over,
+in every branch of service, may feel a need to learn what he can teach.
+Backgrounds don't matter much. Fitzpatrick himself was originally a
+Border Patrol veteran, then became a Border Patrol instructor at FLETC.
+His Spanish is still fluent--but he found himself strangely fascinated
+when the first computers showed up at the Training Center. Fitzpatrick
+did have a background in electrical engineering, and though he never
+considered himself a computer hacker, he somehow found himself writing
+useful little programs for this new and promising gizmo.
+
+He began looking into the general subject of computers and crime,
+reading Donn Parker's books and articles, keeping an ear cocked
+for war stories, useful insights from the field, the up-and-coming
+people of the local computer-crime and high-technology units. . . .
+Soon he got a reputation around FLETC as the resident "computer expert,"
+and that reputation alone brought him more exposure, more experience--
+until one day he looked around, and sure enough he WAS a federal
+computer-crime expert.
+
+In fact, this unassuming, genial man may be THE federal computer-crime expert.
+There are plenty of very good computer people, and plenty of very good
+federal investigators, but the area where these worlds of expertise overlap
+is very slim. And Carlton Fitzpatrick has been right at the center of that
+since 1985, the first year of the Colluquy, a group which owes much to
+his influence.
+
+He seems quite at home in his modest, acoustic-tiled office,
+with its Ansel Adams-style Western photographic art, a gold-framed
+Senior Instructor Certificate, and a towering bookcase crammed with
+three-ring binders with ominous titles such as Datapro Reports on
+Information Security and CFCA Telecom Security '90.
+
+The phone rings every ten minutes; colleagues show up at the door
+to chat about new developments in locksmithing or to shake their heads
+over the latest dismal developments in the BCCI global banking scandal.
+
+Carlton Fitzpatrick is a fount of computer-crime war-stories,
+related in an acerbic drawl. He tells me the colorful tale
+of a hacker caught in California some years back. He'd been
+raiding systems, typing code without a detectable break,
+for twenty, twenty-four, thirty-six hours straight. Not just
+logged on--TYPING. Investigators were baffled. Nobody
+could do that. Didn't he have to go to the bathroom?
+Was it some kind of automatic keyboard-whacking device
+that could actually type code?
+
+A raid on the suspect's home revealed a situation of astonishing squalor.
+The hacker turned out to be a Pakistani computer-science student who had
+flunked out of a California university. He'd gone completely underground
+as an illegal electronic immigrant, and was selling stolen phone-service
+to stay alive. The place was not merely messy and dirty, but in a state
+of psychotic disorder. Powered by some weird mix of culture shock,
+computer addiction, and amphetamines, the suspect had in fact been sitting
+in front of his computer for a day and a half straight, with snacks and
+drugs at hand on the edge of his desk and a chamber-pot under his chair.
+
+Word about stuff like this gets around in the hacker-tracker community.
+
+Carlton Fitzpatrick takes me for a guided tour by car around the
+FLETC grounds. One of our first sights is the biggest indoor
+firing range in the world. There are federal trainees in there,
+Fitzpatrick assures me politely, blasting away with a wide variety
+of automatic weapons: Uzis, Glocks, AK-47s. . . . He's willing to
+take me inside. I tell him I'm sure that's really interesting,
+but I'd rather see his computers. Carlton Fitzpatrick seems quite
+surprised and pleased. I'm apparently the first journalist he's ever
+seen who has turned down the shooting gallery in favor of microchips.
+
+Our next stop is a favorite with touring Congressmen: the three-mile
+long FLETC driving range. Here trainees of the Driver & Marine Division
+are taught high-speed pursuit skills, setting and breaking road-blocks,
+diplomatic security driving for VIP limousines. . . . A favorite FLETC
+pastime is to strap a passing Senator into the passenger seat beside a
+Driver & Marine trainer, hit a hundred miles an hour, then take it right into
+"the skid-pan," a section of greased track where two tons of Detroit iron
+can whip and spin like a hockey puck.
+
+Cars don't fare well at FLETC. First they're rifled again and again
+for search practice. Then they do 25,000 miles of high-speed
+pursuit training; they get about seventy miles per set
+of steel-belted radials. Then it's off to the skid pan,
+where sometimes they roll and tumble headlong in the grease.
+When they're sufficiently grease-stained, dented, and creaky,
+they're sent to the roadblock unit, where they're battered without pity.
+And finally then they're sacrificed to the Bureau of Alcohol,
+Tobacco and Firearms, whose trainees learn the ins and outs
+of car-bomb work by blowing them into smoking wreckage.
+
+There's a railroad box-car on the FLETC grounds, and a large
+grounded boat, and a propless plane; all training-grounds for searches.
+The plane sits forlornly on a patch of weedy tarmac next to an eerie
+blockhouse known as the "ninja compound," where anti-terrorism specialists
+practice hostage rescues. As I gaze on this creepy paragon of modern
+low-intensity warfare, my nerves are jangled by a sudden staccato outburst
+of automatic weapons fire, somewhere in the woods to my right.
+"Nine-millimeter," Fitzpatrick judges calmly.
+
+Even the eldritch ninja compound pales somewhat compared
+to the truly surreal area known as "the raid-houses."
+This is a street lined on both sides with nondescript
+concrete-block houses with flat pebbled roofs.
+They were once officers' quarters. Now they are training grounds.
+The first one to our left, Fitzpatrick tells me, has been specially
+adapted for computer search-and-seizure practice. Inside it has been
+wired for video from top to bottom, with eighteen pan-and-tilt
+remotely controlled videocams mounted on walls and in corners.
+Every movement of the trainee agent is recorded live by teachers,
+for later taped analysis. Wasted movements, hesitations, possibly lethal
+tactical mistakes--all are gone over in detail.
+
+Perhaps the weirdest single aspect of this building is its front door,
+scarred and scuffed all along the bottom, from the repeated impact,
+day after day, of federal shoe-leather.
+
+Down at the far end of the row of raid-houses some people are practicing
+a murder. We drive by slowly as some very young and rather nervous-looking
+federal trainees interview a heavyset bald man on the raid-house lawn.
+Dealing with murder takes a lot of practice; first you have to learn
+to control your own instinctive disgust and panic, then you have to learn
+to control the reactions of a nerve-shredded crowd of civilians,
+some of whom may have just lost a loved one, some of whom may be murderers--
+quite possibly both at once.
+
+A dummy plays the corpse. The roles of the bereaved, the morbidly curious,
+and the homicidal are played, for pay, by local Georgians: waitresses,
+musicians, most anybody who needs to moonlight and can learn a script.
+These people, some of whom are FLETC regulars year after year,
+must surely have one of the strangest jobs in the world.
+
+Something about the scene: "normal" people in a weird situation,
+standing around talking in bright Georgia sunshine, unsuccessfully
+pretending that something dreadful has gone on, while a dummy lies
+inside on faked bloodstains. . . . While behind this weird masquerade,
+like a nested set of Russian dolls, are grim future realities of real death,
+real violence, real murders of real people, that these young agents
+will really investigate, many times during their careers. . . .
+Over and over. . . . Will those anticipated murders look like this,
+feel like this--not as "real" as these amateur actors are trying to
+make it seem, but both as "real," and as numbingly unreal, as watching
+fake people standing around on a fake lawn? Something about this scene
+unhinges me. It seems nightmarish to me, Kafkaesque. I simply don't
+know how to take it; my head is turned around; I don't know whether to laugh,
+cry, or just shudder.
+
+When the tour is over, Carlton Fitzpatrick and I talk about computers.
+For the first time cyberspace seems like quite a comfortable place.
+It seems very real to me suddenly, a place where I know what I'm talking about,
+a place I'm used to. It's real. "Real." Whatever.
+
+Carlton Fitzpatrick is the only person I've met in cyberspace circles
+who is happy with his present equipment. He's got a 5 Meg RAM PC with
+a 112 meg hard disk; a 660 meg's on the way. He's got a Compaq 386 desktop,
+and a Zenith 386 laptop with 120 meg. Down the hall is a NEC Multi-Sync 2A
+with a CD-ROM drive and a 9600 baud modem with four com-lines.
+There's a training minicomputer, and a 10-meg local mini just for the Center,
+and a lab-full of student PC clones and half-a-dozen Macs or so.
+There's a Data General MV 2500 with 8 meg on board and a 370 meg disk.
+
+Fitzpatrick plans to run a UNIX board on the Data General when he's
+finished beta-testing the software for it, which he wrote himself.
+It'll have E-mail features, massive files on all manner of computer-crime
+and investigation procedures, and will follow the computer-security
+specifics of the Department of Defense "Orange Book." He thinks
+it will be the biggest BBS in the federal government.
+
+Will it have Phrack on it? I ask wryly.
+
+Sure, he tells me. Phrack, TAP, Computer Underground Digest,
+all that stuff. With proper disclaimers, of course.
+
+I ask him if he plans to be the sysop. Running a system that size is very
+time-consuming, and Fitzpatrick teaches two three-hour courses every day.
+
+No, he says seriously, FLETC has to get its money worth out of the instructors.
+He thinks he can get a local volunteer to do it, a high-school student.
+
+He says a bit more, something I think about an Eagle Scout law-enforcement
+liaison program, but my mind has rocketed off in disbelief.
+
+"You're going to put a TEENAGER in charge of a federal security BBS?"
+I'm speechless. It hasn't escaped my notice that the FLETC Financial
+Fraud Institute is the ULTIMATE hacker-trashing target; there is stuff in here,
+stuff of such utter and consummate cool by every standard of the
+digital underground. . . .
+
+I imagine the hackers of my acquaintance, fainting dead-away from
+forbidden-knowledge greed-fits, at the mere prospect of cracking
+the superultra top-secret computers used to train the Secret Service
+in computer-crime. . . .
+
+"Uhm, Carlton," I babble, "I'm sure he's a really nice kid and all,
+but that's a terrible temptation to set in front of somebody who's,
+you know, into computers and just starting out. . . ."
+
+"Yeah," he says, "that did occur to me." For the first time I begin
+to suspect that he's pulling my leg.
+
+He seems proudest when he shows me an ongoing project called JICC,
+Joint Intelligence Control Council. It's based on the services provided
+by EPIC, the El Paso Intelligence Center, which supplies data and intelligence
+to the Drug Enforcement Administration, the Customs Service, the Coast Guard,
+and the state police of the four southern border states. Certain EPIC files
+can now be accessed by drug-enforcement police of Central America,
+South America and the Caribbean, who can also trade information
+among themselves. Using a telecom program called "White Hat,"
+written by two brothers named Lopez from the Dominican Republic,
+police can now network internationally on inexpensive PCs.
+Carlton Fitzpatrick is teaching a class of drug-war agents
+from the Third World, and he's very proud of their progress.
+Perhaps soon the sophisticated smuggling networks of the
+Medellin Cartel will be matched by a sophisticated computer
+network of the Medellin Cartel's sworn enemies. They'll track boats,
+track contraband, track the international drug-lords who now leap over
+borders with great ease, defeating the police through the clever use
+of fragmented national jurisdictions.
+
+JICC and EPIC must remain beyond the scope of this book.
+They seem to me to be very large topics fraught with complications
+that I am not fit to judge. I do know, however, that the international,
+computer-assisted networking of police, across national boundaries,
+is something that Carlton Fitzpatrick considers very important,
+a harbinger of a desirable future. I also know that networks
+by their nature ignore physical boundaries. And I also know
+that where you put communications you put a community,
+and that when those communities become self-aware
+they will fight to preserve themselves and to expand their influence.
+I make no judgements whether this is good or bad.
+It's just cyberspace; it's just the way things are.
+
+I asked Carlton Fitzpatrick what advice he would have for
+a twenty-year-old who wanted to shine someday in the world
+of electronic law enforcement.
+
+He told me that the number one rule was simply not to be
+scared of computers. You don't need to be an obsessive
+"computer weenie," but you mustn't be buffaloed just because
+some machine looks fancy. The advantages computers give
+smart crooks are matched by the advantages they give smart cops.
+Cops in the future will have to enforce the law "with their heads,
+not their holsters." Today you can make good cases without ever
+leaving your office. In the future, cops who resist the computer
+revolution will never get far beyond walking a beat.
+
+I asked Carlton Fitzpatrick if he had some single message for the public;
+some single thing that he would most like the American public to know
+about his work.
+
+He thought about it while. "Yes," he said finally. "TELL me the rules,
+and I'll TEACH those rules!" He looked me straight in the eye.
+"I do the best that I can."
+
+
+
+PART FOUR: THE CIVIL LIBERTARIANS
+
+
+The story of the Hacker Crackdown, as we have followed it thus far,
+has been technological, subcultural, criminal and legal.
+The story of the Civil Libertarians, though it partakes
+of all those other aspects, is profoundly and thoroughly POLITICAL.
+
+In 1990, the obscure, long-simmering struggle over the ownership
+and nature of cyberspace became loudly and irretrievably public.
+People from some of the oddest corners of American society suddenly
+found themselves public figures. Some of these people found this
+situation much more than they had ever bargained for. They backpedalled,
+and tried to retreat back to the mandarin obscurity of their cozy
+subcultural niches. This was generally to prove a mistake.
+
+But the civil libertarians seized the day in 1990. They found themselves
+organizing, propagandizing, podium-pounding, persuading, touring,
+negotiating, posing for publicity photos, submitting to interviews,
+squinting in the limelight as they tried a tentative, but growingly
+sophisticated, buck-and-wing upon the public stage.
+
+It's not hard to see why the civil libertarians should have
+this competitive advantage.
+
+The hackers of the digital underground are an hermetic elite.
+They find it hard to make any remotely convincing case for
+their actions in front of the general public. Actually,
+hackers roundly despise the "ignorant" public, and have never
+trusted the judgement of "the system." Hackers do propagandize,
+but only among themselves, mostly in giddy, badly spelled manifestos
+of class warfare, youth rebellion or naive techie utopianism.
+Hackers must strut and boast in order to establish and preserve
+their underground reputations. But if they speak out too loudly
+and publicly, they will break the fragile surface-tension of the underground,
+and they will be harrassed or arrested. Over the longer term,
+most hackers stumble, get busted, get betrayed, or simply give up.
+As a political force, the digital underground is hamstrung.
+
+The telcos, for their part, are an ivory tower under protracted seige.
+They have plenty of money with which to push their calculated public image,
+but they waste much energy and goodwill attacking one another with
+slanderous and demeaning ad campaigns. The telcos have suffered
+at the hands of politicians, and, like hackers, they don't trust
+the public's judgement. And this distrust may be well-founded.
+Should the general public of the high-tech 1990s come to understand
+its own best interests in telecommunications, that might well pose
+a grave threat to the specialized technical power and authority
+that the telcos have relished for over a century. The telcos do
+have strong advantages: loyal employees, specialized expertise,
+influence in the halls of power, tactical allies in law enforcement,
+and unbelievably vast amounts of money. But politically speaking, they lack
+genuine grassroots support; they simply don't seem to have many friends.
+
+Cops know a lot of things other people don't know.
+But cops willingly reveal only those aspects of their
+knowledge that they feel will meet their institutional
+purposes and further public order. Cops have respect,
+they have responsibilities, they have power in the streets
+and even power in the home, but cops don't do particularly
+well in limelight. When pressed, they will step out in the
+public gaze to threaten bad-guys, or to cajole prominent citizens,
+or perhaps to sternly lecture the naive and misguided.
+But then they go back within their time-honored fortress
+of the station-house, the courtroom and the rule-book.
+
+The electronic civil libertarians, however, have proven to be
+born political animals. They seemed to grasp very early on
+the postmodern truism that communication is power. Publicity is power.
+Soundbites are power. The ability to shove one's issue onto the public
+agenda--and KEEP IT THERE--is power. Fame is power. Simple personal
+fluency and eloquence can be power, if you can somehow catch the
+public's eye and ear.
+
+The civil libertarians had no monopoly on "technical power"--
+though they all owned computers, most were not particularly
+advanced computer experts. They had a good deal of money,
+but nowhere near the earthshaking wealth and the galaxy
+of resources possessed by telcos or federal agencies.
+They had no ability to arrest people. They carried
+out no phreak and hacker covert dirty-tricks.
+
+But they really knew how to network.
+
+Unlike the other groups in this book, the civil libertarians
+have operated very much in the open, more or less right
+in the public hurly-burly. They have lectured audiences galore
+and talked to countless journalists, and have learned to
+refine their spiels. They've kept the cameras clicking,
+kept those faxes humming, swapped that email,
+run those photocopiers on overtime, licked envelopes
+and spent small fortunes on airfare and long-distance.
+In an information society, this open, overt, obvious activity
+has proven to be a profound advantage.
+
+In 1990, the civil libertarians of cyberspace assembled
+out of nowhere in particular, at warp speed. This "group"
+(actually, a networking gaggle of interested parties
+which scarcely deserves even that loose term) has almost nothing
+in the way of formal organization. Those formal civil libertarian
+organizations which did take an interest in cyberspace issues,
+mainly the Computer Professionals for Social Responsibility
+and the American Civil Liberties Union, were carried along
+by events in 1990, and acted mostly as adjuncts,
+underwriters or launching-pads.
+
+The civil libertarians nevertheless enjoyed the greatest success
+of any of the groups in the Crackdown of 1990. At this writing,
+their future looks rosy and the political initiative is firmly in their hands.
+This should be kept in mind as we study the highly unlikely lives
+and lifestyles of the people who actually made this happen.
+
+#
+
+In June 1989, Apple Computer, Inc., of Cupertino,
+California, had a problem. Someone had illicitly copied
+a small piece of Apple's proprietary software, software
+which controlled an internal chip driving the Macintosh
+screen display. This Color QuickDraw source code was
+a closely guarded piece of Apple's intellectual property.
+Only trusted Apple insiders were supposed to possess it.
+
+But the "NuPrometheus League" wanted things otherwise.
+This person (or persons) made several illicit copies
+of this source code, perhaps as many as two dozen.
+He (or she, or they) then put those illicit floppy disks
+into envelopes and mailed them to people all over America:
+people in the computer industry who were associated with,
+but not directly employed by, Apple Computer.
+
+The NuPrometheus caper was a complex, highly ideological,
+and very hacker-like crime. Prometheus, it will be recalled,
+stole the fire of the Gods and gave this potent gift to the
+general ranks of downtrodden mankind. A similar god-in-the-manger
+attitude was implied for the corporate elite of Apple Computer,
+while the "Nu" Prometheus had himself cast in the role of rebel demigod.
+The illicitly copied data was given away for free.
+
+The new Prometheus, whoever he was, escaped the
+fate of the ancient Greek Prometheus, who was chained
+to a rock for centuries by the vengeful gods while an eagle
+tore and ate his liver. On the other hand, NuPrometheus
+chickened out somewhat by comparison with his role model.
+The small chunk of Color QuickDraw code he had filched
+and replicated was more or less useless to Apple's
+industrial rivals (or, in fact, to anyone else).
+Instead of giving fire to mankind, it was more as if
+NuPrometheus had photocopied the schematics for part of a Bic lighter.
+The act was not a genuine work of industrial espionage.
+It was best interpreted as a symbolic, deliberate slap
+in the face for the Apple corporate heirarchy.
+
+Apple's internal struggles were well-known in the industry. Apple's founders,
+Jobs and Wozniak, had both taken their leave long since. Their raucous core
+of senior employees had been a barnstorming crew of 1960s Californians,
+many of them markedly less than happy with the new button-down multimillion
+dollar regime at Apple. Many of the programmers and developers who had
+invented the Macintosh model in the early 1980s had also taken their leave of
+the company. It was they, not the current masters of Apple's corporate fate,
+who had invented the stolen Color QuickDraw code. The NuPrometheus stunt
+was well-calculated to wound company morale.
+
+Apple called the FBI. The Bureau takes an interest in high-profile
+intellectual-property theft cases, industrial espionage and theft
+of trade secrets. These were likely the right people to call,
+and rumor has it that the entities responsible were in fact discovered
+by the FBI, and then quietly squelched by Apple management. NuPrometheus
+was never publicly charged with a crime, or prosecuted, or jailed.
+But there were no further illicit releases of Macintosh internal software.
+Eventually the painful issue of NuPrometheus was allowed to fade.
+
+In the meantime, however, a large number of puzzled bystanders
+found themselves entertaining surprise guests from the FBI.
+
+One of these people was John Perry Barlow. Barlow is a most unusual man,
+difficult to describe in conventional terms. He is perhaps best known as
+a songwriter for the Grateful Dead, for he composed lyrics for
+"Hell in a Bucket," "Picasso Moon," "Mexicali Blues," "I Need a Miracle,"
+and many more; he has been writing for the band since 1970.
+
+Before we tackle the vexing question as to why a rock lyricist
+should be interviewed by the FBI in a computer-crime case,
+it might be well to say a word or two about the Grateful Dead.
+The Grateful Dead are perhaps the most successful and long-lasting
+of the numerous cultural emanations from the Haight-Ashbury district
+of San Francisco, in the glory days of Movement politics and
+lysergic transcendance. The Grateful Dead are a nexus, a veritable
+whirlwind, of applique decals, psychedelic vans, tie-dyed T-shirts,
+earth-color denim, frenzied dancing and open and unashamed drug use.
+The symbols, and the realities, of Californian freak power surround
+the Grateful Dead like knotted macrame.
+
+The Grateful Dead and their thousands of Deadhead devotees
+are radical Bohemians. This much is widely understood.
+Exactly what this implies in the 1990s is rather more problematic.
+
+The Grateful Dead are among the world's most popular
+and wealthy entertainers: number 20, according to Forbes magazine,
+right between M.C. Hammer and Sean Connery. In 1990, this jeans-clad
+group of purported raffish outcasts earned seventeen million dollars.
+They have been earning sums much along this line for quite some time now.
+
+And while the Dead are not investment bankers or three-piece-suit
+tax specialists--they are, in point of fact, hippie musicians--
+this money has not been squandered in senseless Bohemian excess.
+The Dead have been quietly active for many years, funding various
+worthy activities in their extensive and widespread cultural community.
+
+The Grateful Dead are not conventional players in the American
+power establishment. They nevertheless are something of a force
+to be reckoned with. They have a lot of money and a lot of friends
+in many places, both likely and unlikely.
+
+The Dead may be known for back-to-the-earth environmentalist rhetoric,
+but this hardly makes them anti-technological Luddites. On the contrary,
+like most rock musicians, the Grateful Dead have spent their entire adult
+lives in the company of complex electronic equipment. They have funds to burn
+on any sophisticated tool and toy that might happen to catch their fancy.
+And their fancy is quite extensive.
+
+The Deadhead community boasts any number of recording engineers,
+lighting experts, rock video mavens, electronic technicians
+of all descriptions. And the drift goes both ways. Steve Wozniak,
+Apple's co-founder, used to throw rock festivals. Silicon Valley rocks out.
+
+These are the 1990s, not the 1960s. Today, for a surprising number of people
+all over America, the supposed dividing line between Bohemian and technician
+simply no longer exists. People of this sort may have a set of windchimes
+and a dog with a knotted kerchief 'round its neck, but they're also quite
+likely to own a multimegabyte Macintosh running MIDI synthesizer software
+and trippy fractal simulations. These days, even Timothy Leary himself,
+prophet of LSD, does virtual-reality computer-graphics demos in
+his lecture tours.
+
+John Perry Barlow is not a member of the Grateful Dead. He is, however,
+a ranking Deadhead.
+
+Barlow describes himself as a "techno-crank." A vague term like
+"social activist" might not be far from the mark, either.
+But Barlow might be better described as a "poet"--if one keeps in mind
+Percy Shelley's archaic definition of poets as "unacknowledged legislators
+of the world."
+
+Barlow once made a stab at acknowledged legislator status. In 1987,
+he narrowly missed the Republican nomination for a seat in the
+Wyoming State Senate. Barlow is a Wyoming native, the third-generation
+scion of a well-to-do cattle-ranching family. He is in his early forties,
+married and the father of three daughters.
+
+Barlow is not much troubled by other people's narrow notions of consistency.
+In the late 1980s, this Republican rock lyricist cattle rancher sold his ranch
+and became a computer telecommunications devotee.
+
+The free-spirited Barlow made this transition with ease. He genuinely
+enjoyed computers. With a beep of his modem, he leapt from small-town
+Pinedale, Wyoming, into electronic contact with a large and lively crowd
+of bright, inventive, technological sophisticates from all over the world.
+Barlow found the social milieu of computing attractive: its fast-lane pace,
+its blue-sky rhetoric, its open-endedness. Barlow began dabbling in
+computer journalism, with marked success, as he was a quick study,
+and both shrewd and eloquent. He frequently travelled to San Francisco
+to network with Deadhead friends. There Barlow made extensive contacts
+throughout the Californian computer community, including friendships
+among the wilder spirits at Apple.
+
+In May 1990, Barlow received a visit from a local Wyoming agent of the FBI.
+The NuPrometheus case had reached Wyoming.
+
+Barlow was troubled to find himself under investigation in an
+area of his interests once quite free of federal attention.
+He had to struggle to explain the very nature of computer-crime
+to a headscratching local FBI man who specialized in cattle-rustling.
+Barlow, chatting helpfully and demonstrating the wonders of his modem
+to the puzzled fed, was alarmed to find all "hackers" generally under
+FBI suspicion as an evil influence in the electronic community.
+The FBI, in pursuit of a hacker called "NuPrometheus," were tracing
+attendees of a suspect group called the Hackers Conference.
+
+The Hackers Conference, which had been started in 1984, was a
+yearly Californian meeting of digital pioneers and enthusiasts.
+The hackers of the Hackers Conference had little if anything to do
+with the hackers of the digital underground. On the contrary,
+the hackers of this conference were mostly well-to-do Californian
+high-tech CEOs, consultants, journalists and entrepreneurs.
+(This group of hackers were the exact sort of "hackers"
+most likely to react with militant fury at any criminal
+degradation of the term "hacker.")
+
+Barlow, though he was not arrested or accused of a crime,
+and though his computer had certainly not gone out the door,
+was very troubled by this anomaly. He carried the word to the Well.
+
+Like the Hackers Conference, "the Well" was an emanation of the
+Point Foundation. Point Foundation, the inspiration of a wealthy
+Californian 60s radical named Stewart Brand, was to be a major
+launch-pad of the civil libertarian effort.
+
+Point Foundation's cultural efforts, like those of their fellow Bay Area
+Californians the Grateful Dead, were multifaceted and multitudinous.
+Rigid ideological consistency had never been a strong suit of the
+Whole Earth Catalog. This Point publication had enjoyed a strong
+vogue during the late 60s and early 70s, when it offered hundreds
+of practical (and not so practical) tips on communitarian living,
+environmentalism, and getting back-to-the-land. The Whole Earth Catalog,
+and its sequels, sold two and half million copies and won a
+National Book Award.
+
+With the slow collapse of American radical dissent, the Whole Earth Catalog
+had slipped to a more modest corner of the cultural radar; but in its
+magazine incarnation, CoEvolution Quarterly, the Point Foundation
+continued to offer a magpie potpourri of "access to tools and ideas."
+
+CoEvolution Quarterly, which started in 1974, was never a widely
+popular magazine. Despite periodic outbreaks of millenarian fervor,
+CoEvolution Quarterly failed to revolutionize Western civilization
+and replace leaden centuries of history with bright new Californian paradigms.
+Instead, this propaganda arm of Point Foundation cakewalked a fine line between
+impressive brilliance and New Age flakiness. CoEvolution Quarterly carried
+no advertising, cost a lot, and came out on cheap newsprint with modest
+black-and-white graphics. It was poorly distributed, and spread mostly
+by subscription and word of mouth.
+
+It could not seem to grow beyond 30,000 subscribers.
+And yet--it never seemed to shrink much, either.
+Year in, year out, decade in, decade out, some strange
+demographic minority accreted to support the magazine.
+The enthusiastic readership did not seem to have much
+in the way of coherent politics or ideals. It was sometimes
+hard to understand what held them together (if the often bitter
+debate in the letter-columns could be described as "togetherness").
+
+But if the magazine did not flourish, it was resilient; it got by.
+Then, in 1984, the birth-year of the Macintosh computer,
+CoEvolution Quarterly suddenly hit the rapids. Point Foundation
+had discovered the computer revolution. Out came the Whole Earth
+Software Catalog of 1984, arousing headscratching doubts among
+the tie-dyed faithful, and rabid enthusiasm among the nascent
+"cyberpunk" milieu, present company included. Point Foundation
+started its yearly Hackers Conference, and began to take an
+extensive interest in the strange new possibilities of
+digital counterculture. CoEvolution Quarterlyfolded its teepee,
+replaced by Whole Earth Software Review and eventually by Whole Earth
+Review (the magazine's present incarnation, currently under
+the editorship of virtual-reality maven Howard Rheingold).
+
+1985 saw the birth of the "WELL"--the "Whole Earth 'Lectronic Link."
+The Well was Point Foundation's bulletin board system.
+
+As boards went, the Well was an anomaly from the beginning,
+and remained one. It was local to San Francisco.
+It was huge, with multiple phonelines and enormous files
+of commentary. Its complex UNIX-based software might be
+most charitably described as "user-opaque." It was run on
+a mainframe out of the rambling offices of a non-profit
+cultural foundation in Sausalito. And it was crammed with
+fans of the Grateful Dead.
+
+Though the Well was peopled by chattering hipsters of the Bay Area
+counterculture, it was by no means a "digital underground" board.
+Teenagers were fairly scarce; most Well users (known as "Wellbeings")
+were thirty- and forty-something Baby Boomers. They tended to work
+in the information industry: hardware, software, telecommunications,
+media, entertainment. Librarians, academics, and journalists were
+especially common on the Well, attracted by Point Foundation's
+open-handed distribution of "tools and ideas."
+
+There were no anarchy files on the Well, scarcely a
+dropped hint about access codes or credit-card theft.
+No one used handles. Vicious "flame-wars" were held to
+a comparatively civilized rumble. Debates were sometimes sharp,
+but no Wellbeing ever claimed that a rival had disconnected his phone,
+trashed his house, or posted his credit card numbers.
+
+The Well grew slowly as the 1980s advanced. It charged a modest sum
+for access and storage, and lost money for years--but not enough to hamper
+the Point Foundation, which was nonprofit anyway. By 1990, the Well
+had about five thousand users. These users wandered about a gigantic
+cyberspace smorgasbord of "Conferences", each conference itself consisting
+of a welter of "topics," each topic containing dozens, sometimes hundreds
+of comments, in a tumbling, multiperson debate that could last for months
+or years on end.
+
+
+In 1991, the Well's list of conferences looked like this:
+
+
+CONFERENCES ON THE WELL
+
+WELL "Screenzine" Digest (g zine)
+
+Best of the WELL - vintage material - (g best)
+
+Index listing of new topics in all conferences - (g newtops)
+
+Business - Education
+----------------------
+
+Apple Library Users Group(g alug) Agriculture (g agri)
+Brainstorming (g brain) Classifieds (g cla)
+Computer Journalism (g cj) Consultants (g consult)
+Consumers (g cons) Design (g design)
+Desktop Publishing (g desk) Disability (g disability)
+Education (g ed) Energy (g energy91)
+Entrepreneurs (g entre) Homeowners (g home)
+Indexing (g indexing) Investments (g invest)
+Kids91 (g kids) Legal (g legal)
+One Person Business (g one)
+Periodical/newsletter (g per)
+Telecomm Law (g tcl) The Future (g fut)
+Translators (g trans) Travel (g tra)
+Work (g work)
+
+Electronic Frontier Foundation (g eff)
+Computers, Freedom & Privacy (g cfp)
+Computer Professionals for Social Responsibility (g cpsr)
+
+Social - Political - Humanities
+---------------------------------
+
+Aging (g gray) AIDS (g aids)
+Amnesty International (g amnesty) Archives (g arc)
+Berkeley (g berk) Buddhist (g wonderland)
+Christian (g cross) Couples (g couples)
+Current Events (g curr) Dreams (g dream)
+Drugs (g dru) East Coast (g east)
+Emotional Health@@@@ (g private) Erotica (g eros)
+Environment (g env) Firearms (g firearms)
+First Amendment (g first) Fringes of Reason (g fringes)
+Gay (g gay) Gay (Private)# (g gaypriv)
+Geography (g geo) German (g german)
+Gulf War (g gulf) Hawaii (g aloha)
+Health (g heal) History (g hist)
+Holistic (g holi) Interview (g inter)
+Italian (g ital) Jewish (g jew)
+Liberty (g liberty) Mind (g mind)
+Miscellaneous (g misc) Men on the WELL@@ (g mow)
+Network Integration (g origin) Nonprofits (g non)
+North Bay (g north) Northwest (g nw)
+Pacific Rim (g pacrim) Parenting (g par)
+Peace (g pea) Peninsula (g pen)
+Poetry (g poetry) Philosophy (g phi)
+Politics (g pol) Psychology (g psy)
+Psychotherapy (g therapy) Recovery## (g recovery)
+San Francisco (g sanfran) Scams (g scam)
+Sexuality (g sex) Singles (g singles)
+Southern (g south) Spanish (g spanish)
+Spirituality (g spirit) Tibet (g tibet)
+Transportation (g transport) True Confessions (g tru)
+Unclear (g unclear) WELL Writer's Workshop@@@(g www)
+Whole Earth (g we) Women on the WELL@(g wow)
+Words (g words) Writers (g wri)
+
+@@@@Private Conference - mail wooly for entry
+@@@Private conference - mail sonia for entry
+@@Private conference - mail flash for entry
+@ Private conference - mail reva for entry
+# Private Conference - mail hudu for entry
+## Private Conference - mail dhawk for entry
+
+Arts - Recreation - Entertainment
+-----------------------------------
+ArtCom Electronic Net (g acen)
+Audio-Videophilia (g aud)
+Bicycles (g bike) Bay Area Tonight@@(g bat)
+Boating (g wet) Books (g books)
+CD's (g cd) Comics (g comics)
+Cooking (g cook) Flying (g flying)
+Fun (g fun) Games (g games)
+Gardening (g gard) Kids (g kids)
+Nightowls@ (g owl) Jokes (g jokes)
+MIDI (g midi) Movies (g movies)
+Motorcycling (g ride) Motoring (g car)
+Music (g mus) On Stage (g onstage)
+Pets (g pets) Radio (g rad)
+Restaurant (g rest) Science Fiction (g sf)
+Sports (g spo) Star Trek (g trek)
+Television (g tv) Theater (g theater)
+Weird (g weird) Zines/Factsheet Five(g f5)
+@Open from midnight to 6am
+@@Updated daily
+
+Grateful Dead
+-------------
+Grateful Dead (g gd) Deadplan@ (g dp)
+Deadlit (g deadlit) Feedback (g feedback)
+GD Hour (g gdh) Tapes (g tapes)
+Tickets (g tix) Tours (g tours)
+
+@Private conference - mail tnf for entry
+
+Computers
+-----------
+AI/Forth/Realtime (g realtime) Amiga (g amiga)
+Apple (g app) Computer Books (g cbook)
+Art & Graphics (g gra) Hacking (g hack)
+HyperCard (g hype) IBM PC (g ibm)
+LANs (g lan) Laptop (g lap)
+Macintosh (g mac) Mactech (g mactech)
+Microtimes (g microx) Muchomedia (g mucho)
+NeXt (g next) OS/2 (g os2)
+Printers (g print) Programmer's Net (g net)
+Siggraph (g siggraph) Software Design (g sdc)
+Software/Programming (g software)
+Software Support (g ssc)
+Unix (g unix) Windows (g windows)
+Word Processing (g word)
+
+Technical - Communications
+----------------------------
+Bioinfo (g bioinfo) Info (g boing)
+Media (g media) NAPLPS (g naplps)
+Netweaver (g netweaver) Networld (g networld)
+Packet Radio (g packet) Photography (g pho)
+Radio (g rad) Science (g science)
+Technical Writers (g tec) Telecommunications(g tele)
+Usenet (g usenet) Video (g vid)
+Virtual Reality (g vr)
+
+The WELL Itself
+---------------
+Deeper (g deeper) Entry (g ent)
+General (g gentech) Help (g help)
+Hosts (g hosts) Policy (g policy)
+System News (g news) Test (g test)
+
+The list itself is dazzling, bringing to the untutored eye
+a dizzying impression of a bizarre milieu of mountain-climbing
+Hawaiian holistic photographers trading true-life confessions
+with bisexual word-processing Tibetans.
+
+But this confusion is more apparent than real. Each of these conferences
+was a little cyberspace world in itself, comprising dozens and perhaps
+hundreds of sub-topics. Each conference was commonly frequented by
+a fairly small, fairly like-minded community of perhaps a few dozen people.
+It was humanly impossible to encompass the entire Well (especially since
+access to the Well's mainframe computer was billed by the hour).
+Most long-time users contented themselves with a few favorite
+topical neighborhoods, with the occasional foray elsewhere
+for a taste of exotica. But especially important news items,
+and hot topical debates, could catch the attention of the entire
+Well community.
+
+Like any community, the Well had its celebrities, and John Perry Barlow,
+the silver-tongued and silver-modemed lyricist of the Grateful Dead,
+ranked prominently among them. It was here on the Well that Barlow
+posted his true-life tale of computer-crime encounter with the FBI.
+
+The story, as might be expected, created a great stir. The Well was
+already primed for hacker controversy. In December 1989, Harper's magazine
+had hosted a debate on the Well about the ethics of illicit computer intrusion.
+While over forty various computer-mavens took part, Barlow proved a star
+in the debate. So did "Acid Phreak" and "Phiber Optik," a pair of young
+New York hacker-phreaks whose skills at telco switching-station intrusion
+were matched only by their apparently limitless hunger for fame.
+The advent of these two boldly swaggering outlaws in the precincts
+of the Well created a sensation akin to that of Black Panthers
+at a cocktail party for the radically chic.
+
+Phiber Optik in particular was to seize the day in 1990.
+A devotee of the 2600 circle and stalwart of the New York
+hackers' group "Masters of Deception," Phiber Optik was
+a splendid exemplar of the computer intruder as committed dissident.
+The eighteen-year-old Optik, a high-school dropout and part-time
+computer repairman, was young, smart, and ruthlessly obsessive,
+a sharp-dressing, sharp-talking digital dude who was utterly
+and airily contemptuous of anyone's rules but his own.
+By late 1991, Phiber Optik had appeared in Harper's,
+Esquire, The New York Times, in countless public debates
+and conventions, even on a television show hosted by Geraldo Rivera.
+
+Treated with gingerly respect by Barlow and other Well mavens,
+Phiber Optik swiftly became a Well celebrity. Strangely, despite
+his thorny attitude and utter single-mindedness, Phiber Optik seemed
+to arouse strong protective instincts in most of the people who met him.
+He was great copy for journalists, always fearlessly ready to swagger,
+and, better yet, to actually DEMONSTRATE some off-the-wall digital stunt.
+He was a born media darling.
+
+Even cops seemed to recognize that there was something peculiarly unworldly
+and uncriminal about this particular troublemaker. He was so bold,
+so flagrant, so young, and so obviously doomed, that even those
+who strongly disapproved of his actions grew anxious for his welfare,
+and began to flutter about him as if he were an endangered seal pup.
+
+In January 24, 1990 (nine days after the Martin Luther King Day Crash),
+Phiber Optik, Acid Phreak, and a third NYC scofflaw named Scorpion were
+raided by the Secret Service. Their computers went out the door,
+along with the usual blizzard of papers, notebooks, compact disks,
+answering machines, Sony Walkmans, etc. Both Acid Phreak and
+Phiber Optik were accused of having caused the Crash.
+
+The mills of justice ground slowly. The case eventually fell into
+the hands of the New York State Police. Phiber had lost his machinery
+in the raid, but there were no charges filed against him for over a year.
+His predicament was extensively publicized on the Well, where it caused
+much resentment for police tactics. It's one thing to merely hear about
+a hacker raided or busted; it's another to see the police attacking someone
+you've come to know personally, and who has explained his motives at length.
+Through the Harper's debate on the Well, it had become clear to the
+Wellbeings that Phiber Optik was not in fact going to "hurt anything."
+In their own salad days, many Wellbeings had tasted tear-gas in pitched
+street-battles with police. They were inclined to indulgence for
+acts of civil disobedience.
+
+Wellbeings were also startled to learn of the draconian thoroughness
+of a typical hacker search-and-seizure. It took no great stretch of
+imagination for them to envision themselves suffering much the same treatment.
+
+As early as January 1990, sentiment on the Well had already begun to sour,
+and people had begun to grumble that "hackers" were getting a raw deal
+from the ham-handed powers-that-be. The resultant issue of Harper's
+magazine posed the question as to whether computer-intrusion was a "crime"
+at all. As Barlow put it later: "I've begun to wonder if we wouldn't
+also regard spelunkers as desperate criminals if AT&T owned all the caves."
+
+In February 1991, more than a year after the raid on his home,
+Phiber Optik was finally arrested, and was charged with first-degree
+Computer Tampering and Computer Trespass, New York state offenses.
+He was also charged with a theft-of-service misdemeanor, involving a complex
+free-call scam to a 900 number. Phiber Optik pled guilty to the misdemeanor
+charge, and was sentenced to 35 hours of community service.
+
+This passing harassment from the unfathomable world of straight people
+seemed to bother Optik himself little if at all. Deprived of his computer
+by the January search-and-seizure, he simply bought himself a portable
+computer so the cops could no longer monitor the phone where he lived
+with his Mom, and he went right on with his depredations, sometimes on
+live radio or in front of television cameras.
+
+The crackdown raid may have done little to dissuade Phiber Optik,
+but its galling affect on the Wellbeings was profound. As 1990 rolled on,
+the slings and arrows mounted: the Knight Lightning raid,
+the Steve Jackson raid, the nation-spanning Operation Sundevil.
+The rhetoric of law enforcement made it clear that there was,
+in fact, a concerted crackdown on hackers in progress.
+
+The hackers of the Hackers Conference, the Wellbeings, and their ilk,
+did not really mind the occasional public misapprehension of "hacking;"
+if anything, this membrane of differentiation from straight society
+made the "computer community" feel different, smarter, better.
+They had never before been confronted, however, by a concerted
+vilification campaign.
+
+Barlow's central role in the counter-struggle was one of the major
+anomalies of 1990. Journalists investigating the controversy
+often stumbled over the truth about Barlow, but they commonly
+dusted themselves off and hurried on as if nothing had happened.
+It was as if it were TOO MUCH TO BELIEVE that a 1960s freak
+from the Grateful Dead had taken on a federal law enforcement operation
+head-to-head and ACTUALLY SEEMED TO BE WINNING!
+
+Barlow had no easily detectable power-base for a political struggle
+of this kind. He had no formal legal or technical credentials.
+Barlow was, however, a computer networker of truly stellar brilliance.
+He had a poet's gift of concise, colorful phrasing. He also had a
+journalist's shrewdness, an off-the-wall, self-deprecating wit,
+and a phenomenal wealth of simple personal charm.
+
+The kind of influence Barlow possessed is fairly common currency
+in literary, artistic, or musical circles. A gifted critic can
+wield great artistic influence simply through defining
+the temper of the times, by coining the catch-phrases
+and the terms of debate that become the common currency of the period.
+(And as it happened, Barlow WAS a part-time art critic,
+with a special fondness for the Western art of Frederic Remington.)
+
+Barlow was the first commentator to adopt William Gibson's
+striking science-fictional term "cyberspace" as a synonym
+for the present-day nexus of computer and telecommunications networks.
+Barlow was insistent that cyberspace should be regarded as
+a qualitatively new world, a "frontier." According to Barlow,
+the world of electronic communications, now made visible through
+the computer screen, could no longer be usefully regarded
+as just a tangle of high-tech wiring. Instead, it had become
+a PLACE, cyberspace, which demanded a new set of metaphors,
+a new set of rules and behaviors. The term, as Barlow employed it,
+struck a useful chord, and this concept of cyberspace was picked up
+by Time, Scientific American, computer police, hackers, and even
+Constitutional scholars. "Cyberspace" now seems likely to become
+a permanent fixture of the language.
+
+Barlow was very striking in person: a tall, craggy-faced, bearded,
+deep-voiced Wyomingan in a dashing Western ensemble of jeans, jacket,
+cowboy boots, a knotted throat-kerchief and an ever-present Grateful Dead
+cloisonne lapel pin.
+
+Armed with a modem, however, Barlow was truly in his element.
+Formal hierarchies were not Barlow's strong suit; he rarely missed
+a chance to belittle the "large organizations and their drones,"
+with their uptight, institutional mindset. Barlow was very much
+of the free-spirit persuasion, deeply unimpressed by brass-hats
+and jacks-in-office. But when it came to the digital grapevine,
+Barlow was a cyberspace ad-hocrat par excellence.
+
+There was not a mighty army of Barlows. There was only one Barlow,
+and he was a fairly anomolous individual. However, the situation only
+seemed to REQUIRE a single Barlow. In fact, after 1990, many people
+must have concluded that a single Barlow was far more than
+they'd ever bargained for.
+
+Barlow's querulous mini-essay about his encounter with the FBI
+struck a strong chord on the Well. A number of other free spirits
+on the fringes of Apple Computing had come under suspicion,
+and they liked it not one whit better than he did.
+
+One of these was Mitchell Kapor, the co-inventor of the spreadsheet
+program "Lotus 1-2-3" and the founder of Lotus Development Corporation.
+Kapor had written-off the passing indignity of being fingerprinted
+down at his own local Boston FBI headquarters, but Barlow's post
+made the full national scope of the FBI's dragnet clear to Kapor.
+The issue now had Kapor's full attention. As the Secret Service
+swung into anti-hacker operation nationwide in 1990, Kapor watched
+every move with deep skepticism and growing alarm.
+
+As it happened, Kapor had already met Barlow, who had interviewed Kapor
+for a California computer journal. Like most people who met Barlow,
+Kapor had been very taken with him. Now Kapor took it upon himself
+to drop in on Barlow for a heart-to-heart talk about the situation.
+
+Kapor was a regular on the Well. Kapor had been a devotee of the
+Whole Earth Catalogsince the beginning, and treasured a complete run
+of the magazine. And Kapor not only had a modem, but a private jet.
+In pursuit of the scattered high-tech investments of Kapor Enterprises Inc.,
+his personal, multi-million dollar holding company, Kapor commonly crossed
+state lines with about as much thought as one might give to faxing a letter.
+
+The Kapor-Barlow council of June 1990, in Pinedale, Wyoming, was the start
+of the Electronic Frontier Foundation. Barlow swiftly wrote a manifesto,
+"Crime and Puzzlement," which announced his, and Kapor's, intention
+to form a political organization to "raise and disburse funds for education,
+lobbying, and litigation in the areas relating to digital speech and the
+extension of the Constitution into Cyberspace."
+
+Furthermore, proclaimed the manifesto, the foundation would
+"fund, conduct, and support legal efforts to demonstrate
+that the Secret Service has exercised prior restraint on publications,
+limited free speech, conducted improper seizure of equipment and data,
+used undue force, and generally conducted itself in a fashion which
+is arbitrary, oppressive, and unconstitutional."
+
+"Crime and Puzzlement" was distributed far and wide through computer
+networking channels, and also printed in the Whole Earth Review.
+The sudden declaration of a coherent, politicized counter-strike
+from the ranks of hackerdom electrified the community. Steve Wozniak
+(perhaps a bit stung by the NuPrometheus scandal) swiftly offered
+to match any funds Kapor offered the Foundation.
+
+John Gilmore, one of the pioneers of Sun Microsystems, immediately offered
+his own extensive financial and personal support. Gilmore, an ardent
+libertarian, was to prove an eloquent advocate of electronic privacy issues,
+especially freedom from governmental and corporate computer-assisted
+surveillance of private citizens.
+
+A second meeting in San Francisco rounded up further allies:
+Stewart Brand of the Point Foundation, virtual-reality pioneers
+Jaron Lanier and Chuck Blanchard, network entrepreneur and venture
+capitalist Nat Goldhaber. At this dinner meeting, the activists settled on
+a formal title: the Electronic Frontier Foundation, Incorporated.
+Kapor became its president. A new EFF Conference was opened on
+the Point Foundation's Well, and the Well was declared
+"the home of the Electronic Frontier Foundation."
+
+Press coverage was immediate and intense. Like their
+nineteenth-century spiritual ancestors, Alexander Graham Bell
+and Thomas Watson, the high-tech computer entrepreneurs
+of the 1970s and 1980s--people such as Wozniak, Jobs, Kapor,
+Gates, and H. Ross Perot, who had raised themselves by their bootstraps
+to dominate a glittering new industry--had always made very good copy.
+
+But while the Wellbeings rejoiced, the press in general seemed
+nonplussed by the self-declared "civilizers of cyberspace."
+EFF's insistence that the war against "hackers" involved grave
+Constitutional civil liberties issues seemed somewhat farfetched,
+especially since none of EFF's organizers were lawyers
+or established politicians. The business press in particular
+found it easier to seize on the apparent core of the story--
+that high-tech entrepreneur Mitchell Kapor had established
+a "defense fund for hackers." Was EFF a genuinely important
+political development--or merely a clique of wealthy eccentrics,
+dabbling in matters better left to the proper authorities?
+The jury was still out.
+
+But the stage was now set for open confrontation.
+And the first and the most critical battle was the
+hacker show-trial of "Knight Lightning."
+
+#
+
+It has been my practice throughout this book to refer to hackers
+only by their "handles." There is little to gain by giving
+the real names of these people, many of whom are juveniles,
+many of whom have never been convicted of any crime, and many
+of whom had unsuspecting parents who have already suffered enough.
+
+But the trial of Knight Lightning on July 24-27, 1990,
+made this particular "hacker" a nationally known public figure.
+It can do no particular harm to himself or his family if I repeat
+the long-established fact that his name is Craig Neidorf (pronounced NYE-dorf).
+
+Neidorf's jury trial took place in the United States District Court,
+Northern District of Illinois, Eastern Division, with the
+Honorable Nicholas J. Bua presiding. The United States of America
+was the plaintiff, the defendant Mr. Neidorf. The defendant's attorney
+was Sheldon T. Zenner of the Chicago firm of Katten, Muchin and Zavis.
+
+The prosecution was led by the stalwarts of the Chicago Computer Fraud
+and Abuse Task Force: William J. Cook, Colleen D. Coughlin, and
+David A. Glockner, all Assistant United States Attorneys.
+The Secret Service Case Agent was Timothy M. Foley.
+
+It will be recalled that Neidorf was the co-editor of an underground hacker
+"magazine" called Phrack. Phrack was an entirely electronic publication,
+distributed through bulletin boards and over electronic networks.
+It was amateur publication given away for free. Neidorf had never made
+any money for his work in Phrack. Neither had his unindicted co-editor
+"Taran King" or any of the numerous Phrack contributors.
+
+The Chicago Computer Fraud and Abuse Task Force, however,
+had decided to prosecute Neidorf as a fraudster.
+To formally admit that Phrack was a "magazine"
+and Neidorf a "publisher" was to open a prosecutorial
+Pandora's Box of First Amendment issues. To do this
+was to play into the hands of Zenner and his EFF advisers,
+which now included a phalanx of prominent New York civil rights
+lawyers as well as the formidable legal staff of Katten, Muchin and Zavis.
+Instead, the prosecution relied heavily on the issue of access device fraud:
+Section 1029 of Title 18, the section from which the Secret Service drew
+its most direct jurisdiction over computer crime.
+
+Neidorf's alleged crimes centered around the E911 Document.
+He was accused of having entered into a fraudulent scheme with the Prophet,
+who, it will be recalled, was the Atlanta LoD member who had illicitly
+copied the E911 Document from the BellSouth AIMSX system.
+
+The Prophet himself was also a co-defendant in the Neidorf case,
+part-and-parcel of the alleged "fraud scheme" to "steal" BellSouth's
+E911 Document (and to pass the Document across state lines,
+which helped establish the Neidorf trial as a federal case).
+The Prophet, in the spirit of full co-operation, had agreed
+to testify against Neidorf.
+
+In fact, all three of the Atlanta crew stood ready to testify against Neidorf.
+Their own federal prosecutors in Atlanta had charged the Atlanta Three with:
+(a) conspiracy, (b) computer fraud, (c) wire fraud, (d) access device fraud,
+and (e) interstate transportation of stolen property (Title 18, Sections 371,
+1030, 1343, 1029, and 2314).
+
+Faced with this blizzard of trouble, Prophet and Leftist had ducked
+any public trial and had pled guilty to reduced charges--one conspiracy
+count apiece. Urvile had pled guilty to that odd bit of Section 1029
+which makes it illegal to possess "fifteen or more" illegal access devices
+(in his case, computer passwords). And their sentences were scheduled
+for September 14, 1990--well after the Neidorf trial. As witnesses,
+they could presumably be relied upon to behave.
+
+Neidorf, however, was pleading innocent. Most everyone else caught up
+in the crackdown had "cooperated fully" and pled guilty in hope
+of reduced sentences. (Steve Jackson was a notable exception,
+of course, and had strongly protested his innocence from the
+very beginning. But Steve Jackson could not get a day in court--
+Steve Jackson had never been charged with any crime in the first place.)
+
+Neidorf had been urged to plead guilty. But Neidorf was a political science
+major and was disinclined to go to jail for "fraud" when he had not made
+any money, had not broken into any computer, and had been publishing
+a magazine that he considered protected under the First Amendment.
+
+Neidorf's trial was the ONLY legal action of the entire Crackdown
+that actually involved bringing the issues at hand out for a public test
+in front of a jury of American citizens.
+
+Neidorf, too, had cooperated with investigators. He had voluntarily
+handed over much of the evidence that had led to his own indictment.
+He had already admitted in writing that he knew that the E911 Document
+had been stolen before he had "published" it in Phrack--or, from the
+prosecution's point of view, illegally transported stolen property by wire
+in something purporting to be a "publication."
+
+But even if the "publication" of the E911 Document was not held to be a crime,
+that wouldn't let Neidorf off the hook. Neidorf had still received
+the E911 Document when Prophet had transferred it to him from Rich Andrews'
+Jolnet node. On that occasion, it certainly hadn't been "published"--
+it was hacker booty, pure and simple, transported across state lines.
+
+The Chicago Task Force led a Chicago grand jury to indict Neidorf
+on a set of charges that could have put him in jail for thirty years.
+When some of these charges were successfully challenged before Neidorf
+actually went to trial, the Chicago Task Force rearranged his
+indictment so that he faced a possible jail term of over sixty years!
+As a first offender, it was very unlikely that Neidorf would in fact
+receive a sentence so drastic; but the Chicago Task Force clearly
+intended to see Neidorf put in prison, and his conspiratorial "magazine"
+put permanently out of commission. This was a federal case, and Neidorf
+was charged with the fraudulent theft of property worth almost
+eighty thousand dollars.
+
+William Cook was a strong believer in high-profile prosecutions
+with symbolic overtones. He often published articles on his work
+in the security trade press, arguing that "a clear message had
+to be sent to the public at large and the computer community
+in particular that unauthorized attacks on computers and the theft
+of computerized information would not be tolerated by the courts."
+
+The issues were complex, the prosecution's tactics somewhat unorthodox,
+but the Chicago Task Force had proved sure-footed to date. "Shadowhawk"
+had been bagged on the wing in 1989 by the Task Force, and sentenced
+to nine months in prison, and a $10,000 fine. The Shadowhawk case involved
+charges under Section 1030, the "federal interest computer" section.
+
+Shadowhawk had not in fact been a devotee of "federal-interest" computers
+per se. On the contrary, Shadowhawk, who owned an AT&T home computer,
+seemed to cherish a special aggression toward AT&T. He had bragged on
+the underground boards "Phreak Klass 2600" and "Dr. Ripco" of his skills
+at raiding AT&T, and of his intention to crash AT&T's national phone system.
+Shadowhawk's brags were noticed by Henry Kluepfel of Bellcore Security,
+scourge of the outlaw boards, whose relations with the Chicago Task Force
+were long and intimate.
+
+The Task Force successfully established that Section 1030 applied to
+the teenage Shadowhawk, despite the objections of his defense attorney.
+Shadowhawk had entered a computer "owned" by U.S. Missile Command
+and merely "managed" by AT&T. He had also entered an AT&T computer
+located at Robbins Air Force Base in Georgia. Attacking AT&T was
+of "federal interest" whether Shadowhawk had intended it or not.
+
+The Task Force also convinced the court that a piece of AT&T
+software that Shadowhawk had illicitly copied from Bell Labs,
+the "Artificial Intelligence C5 Expert System," was worth a cool
+one million dollars. Shadowhawk's attorney had argued that
+Shadowhawk had not sold the program and had made no profit from
+the illicit copying. And in point of fact, the C5 Expert System
+was experimental software, and had no established market value
+because it had never been on the market in the first place.
+AT&T's own assessment of a "one million dollar" figure for its
+own intangible property was accepted without challenge
+by the court, however. And the court concurred with
+the government prosecutors that Shadowhawk showed clear
+"intent to defraud" whether he'd gotten any money or not.
+Shadowhawk went to jail.
+
+The Task Force's other best-known triumph had been the conviction
+and jailing of "Kyrie." Kyrie, a true denizen of the digital
+criminal underground, was a 36-year-old Canadian woman,
+convicted and jailed for telecommunications fraud in Canada.
+After her release from prison, she had fled the wrath of Canada Bell
+and the Royal Canadian Mounted Police, and eventually settled,
+very unwisely, in Chicago.
+
+"Kyrie," who also called herself "Long Distance Information,"
+specialized in voice-mail abuse. She assembled large numbers
+of hot long-distance codes, then read them aloud into a series
+of corporate voice-mail systems. Kyrie and her friends were
+electronic squatters in corporate voice-mail systems,
+using them much as if they were pirate bulletin boards,
+then moving on when their vocal chatter clogged the system
+and the owners necessarily wised up. Kyrie's camp followers
+were a loose tribe of some hundred and fifty phone-phreaks,
+who followed her trail of piracy from machine to machine,
+ardently begging for her services and expertise.
+
+Kyrie's disciples passed her stolen credit-card numbers,
+in exchange for her stolen "long distance information."
+Some of Kyrie's clients paid her off in cash, by scamming
+credit-card cash advances from Western Union.
+
+Kyrie travelled incessantly, mostly through airline tickets
+and hotel rooms that she scammed through stolen credit cards.
+Tiring of this, she found refuge with a fellow female phone
+phreak in Chicago. Kyrie's hostess, like a surprising number
+of phone phreaks, was blind. She was also physically disabled.
+Kyrie allegedly made the best of her new situation by applying for,
+and receiving, state welfare funds under a false identity as
+a qualified caretaker for the handicapped.
+
+Sadly, Kyrie's two children by a former marriage had also vanished
+underground with her; these pre-teen digital refugees had no legal
+American identity, and had never spent a day in school.
+
+Kyrie was addicted to technical mastery and enthralled by her own
+cleverness and the ardent worship of her teenage followers.
+This foolishly led her to phone up Gail Thackeray in Arizona,
+to boast, brag, strut, and offer to play informant.
+Thackeray, however, had already learned far more
+than enough about Kyrie, whom she roundly despised
+as an adult criminal corrupting minors, a "female Fagin."
+Thackeray passed her tapes of Kyrie's boasts to the Secret Service.
+
+Kyrie was raided and arrested in Chicago in May 1989.
+She confessed at great length and pled guilty.
+
+In August 1990, Cook and his Task Force colleague Colleen Coughlin
+sent Kyrie to jail for 27 months, for computer and telecommunications fraud.
+This was a markedly severe sentence by the usual wrist-slapping standards
+of "hacker" busts. Seven of Kyrie's foremost teenage disciples were also
+indicted and convicted. The Kyrie "high-tech street gang," as Cook
+described it, had been crushed. Cook and his colleagues had been
+the first ever to put someone in prison for voice-mail abuse.
+Their pioneering efforts had won them attention and kudos.
+
+In his article on Kyrie, Cook drove the message home to the readers
+of Security Management magazine, a trade journal for corporate
+security professionals. The case, Cook said, and Kyrie's stiff sentence,
+"reflect a new reality for hackers and computer crime victims in the
+'90s. . . . Individuals and corporations who report computer
+and telecommunications crimes can now expect that their cooperation
+with federal law enforcement will result in meaningful punishment.
+Companies and the public at large must report computer-enhanced
+crimes if they want prosecutors and the course to protect their rights
+to the tangible and intangible property developed and stored on computers."
+
+Cook had made it his business to construct this "new reality for hackers."
+He'd also made it his business to police corporate property rights
+to the intangible.
+
+Had the Electronic Frontier Foundation been a "hacker defense fund"
+as that term was generally understood, they presumably would have stood up
+for Kyrie. Her 1990 sentence did indeed send a "message" that federal heat
+was coming down on "hackers." But Kyrie found no defenders at EFF,
+or anywhere else, for that matter. EFF was not a bail-out fund
+for electronic crooks.
+
+The Neidorf case paralleled the Shadowhawk case in certain ways.
+The victim once again was allowed to set the value of the "stolen" property.
+Once again Kluepfel was both investigator and technical advisor.
+Once again no money had changed hands, but the "intent to defraud" was central.
+
+The prosecution's case showed signs of weakness early on. The Task Force
+had originally hoped to prove Neidorf the center of a nationwide
+Legion of Doom criminal conspiracy. The Phrack editors threw physical
+get-togethers every summer, which attracted hackers from across the country;
+generally two dozen or so of the magazine's favorite contributors and readers.
+(Such conventions were common in the hacker community; 2600 Magazine,
+for instance, held public meetings of hackers in New York, every month.)
+LoD heavy-dudes were always a strong presence at these Phrack-sponsored
+"Summercons."
+
+In July 1988, an Arizona hacker named "Dictator" attended Summercon
+in Neidorf's home town of St. Louis. Dictator was one of Gail Thackeray's
+underground informants; Dictator's underground board in Phoenix was
+a sting operation for the Secret Service. Dictator brought an undercover
+crew of Secret Service agents to Summercon. The agents bored spyholes
+through the wall of Dictator's hotel room in St Louis, and videotaped
+the frolicking hackers through a one-way mirror. As it happened,
+however, nothing illegal had occurred on videotape, other than the
+guzzling of beer by a couple of minors. Summercons were social events,
+not sinister cabals. The tapes showed fifteen hours of raucous laughter,
+pizza-gobbling, in-jokes and back-slapping.
+
+Neidorf's lawyer, Sheldon Zenner, saw the Secret Service tapes
+before the trial. Zenner was shocked by the complete harmlessness
+of this meeting, which Cook had earlier characterized as a sinister
+interstate conspiracy to commit fraud. Zenner wanted to show the
+Summercon tapes to the jury. It took protracted maneuverings
+by the Task Force to keep the tapes from the jury as "irrelevant."
+
+The E911 Document was also proving a weak reed. It had originally
+been valued at $79,449. Unlike Shadowhawk's arcane Artificial Intelligence
+booty, the E911 Document was not software--it was written in English.
+Computer-knowledgeable people found this value--for a twelve-page
+bureaucratic document--frankly incredible. In his "Crime and Puzzlement"
+manifesto for EFF, Barlow commented: "We will probably never know how
+this figure was reached or by whom, though I like to imagine an appraisal
+team consisting of Franz Kafka, Joseph Heller, and Thomas Pynchon."
+
+As it happened, Barlow was unduly pessimistic. The EFF did, in fact,
+eventually discover exactly how this figure was reached, and by whom--
+but only in 1991, long after the Neidorf trial was over.
+
+Kim Megahee, a Southern Bell security manager,
+had arrived at the document's value by simply adding up
+the "costs associated with the production" of the E911 Document.
+Those "costs" were as follows:
+
+1. A technical writer had been hired to research and write the E911 Document.
+ 200 hours of work, at $35 an hour, cost : $7,000. A Project Manager had
+ overseen the technical writer. 200 hours, at $31 an hour, made: $6,200.
+
+2. A week of typing had cost $721 dollars. A week of formatting had
+ cost $721. A week of graphics formatting had cost $742.
+
+3. Two days of editing cost $367.
+
+4. A box of order labels cost five dollars.
+
+5. Preparing a purchase order for the Document, including typing
+ and the obtaining of an authorizing signature from within the
+ BellSouth bureaucracy, cost $129.
+
+6. Printing cost $313. Mailing the Document to fifty people
+ took fifty hours by a clerk, and cost $858.
+
+7. Placing the Document in an index took two clerks an hour each,
+ totalling $43.
+
+Bureaucratic overhead alone, therefore, was alleged to have cost
+a whopping $17,099. According to Mr. Megahee, the typing
+of a twelve-page document had taken a full week. Writing it
+had taken five weeks, including an overseer who apparently
+did nothing else but watch the author for five weeks.
+Editing twelve pages had taken two days. Printing and mailing
+an electronic document (which was already available on the
+Southern Bell Data Network to any telco employee who needed it),
+had cost over a thousand dollars.
+
+But this was just the beginning. There were also the HARDWARE EXPENSES.
+Eight hundred fifty dollars for a VT220 computer monitor.
+THIRTY-ONE THOUSAND DOLLARS for a sophisticated VAXstation II computer.
+Six thousand dollars for a computer printer. TWENTY-TWO THOUSAND DOLLARS
+for a copy of "Interleaf" software. Two thousand five hundred dollars
+for VMS software. All this to create the twelve-page Document.
+
+Plus ten percent of the cost of the software and the hardware, for maintenance.
+(Actually, the ten percent maintenance costs, though mentioned, had been left
+off the final $79,449 total, apparently through a merciful oversight).
+
+Mr. Megahee's letter had been mailed directly to William Cook himself,
+at the office of the Chicago federal attorneys. The United States Government
+accepted these telco figures without question.
+
+As incredulity mounted, the value of the E911 Document was officially
+revised downward. This time, Robert Kibler of BellSouth Security
+estimated the value of the twelve pages as a mere $24,639.05--based,
+purportedly, on "R&D costs." But this specific estimate,
+right down to the nickel, did not move the skeptics at all;
+in fact it provoked open scorn and a torrent of sarcasm.
+
+The financial issues concerning theft of proprietary information
+have always been peculiar. It could be argued that BellSouth
+had not "lost" its E911 Document at all in the first place,
+and therefore had not suffered any monetary damage from this "theft."
+And Sheldon Zenner did in fact argue this at Neidorf's trial--
+that Prophet's raid had not been "theft," but was better understood
+as illicit copying.
+
+The money, however, was not central to anyone's true purposes in this trial.
+It was not Cook's strategy to convince the jury that the E911 Document
+was a major act of theft and should be punished for that reason alone.
+His strategy was to argue that the E911 Document was DANGEROUS.
+It was his intention to establish that the E911 Document was "a road-map"
+to the Enhanced 911 System. Neidorf had deliberately and recklessly
+distributed a dangerous weapon. Neidorf and the Prophet did not care
+(or perhaps even gloated at the sinister idea) that the E911 Document
+could be used by hackers to disrupt 911 service, "a life line for every
+person certainly in the Southern Bell region of the United States,
+and indeed, in many communities throughout the United States,"
+in Cook's own words. Neidorf had put people's lives in danger.
+
+In pre-trial maneuverings, Cook had established that the E911 Document
+was too hot to appear in the public proceedings of the Neidorf trial.
+The JURY ITSELF would not be allowed to ever see this Document,
+lest it slip into the official court records, and thus into the hands
+of the general public, and, thus, somehow, to malicious hackers
+who might lethally abuse it.
+
+Hiding the E911 Document from the jury may have been a
+clever legal maneuver, but it had a severe flaw. There were,
+in point of fact, hundreds, perhaps thousands, of people,
+already in possession of the E911 Document, just as Phrack
+had published it. Its true nature was already obvious
+to a wide section of the interested public (all of whom,
+by the way, were, at least theoretically, party to
+a gigantic wire-fraud conspiracy). Most everyone
+in the electronic community who had a modem and any
+interest in the Neidorf case already had a copy of the Document.
+It had already been available in Phrack for over a year.
+
+People, even quite normal people without any particular
+prurient interest in forbidden knowledge, did not shut their eyes
+in terror at the thought of beholding a "dangerous" document
+from a telephone company. On the contrary, they tended to trust
+their own judgement and simply read the Document for themselves.
+And they were not impressed.
+
+One such person was John Nagle. Nagle was a forty-one-year-old
+professional programmer with a masters' degree in computer science
+from Stanford. He had worked for Ford Aerospace, where he had invented
+a computer-networking technique known as the "Nagle Algorithm,"
+and for the prominent Californian computer-graphics firm "Autodesk,"
+where he was a major stockholder.
+
+Nagle was also a prominent figure on the Well, much respected
+for his technical knowledgeability.
+
+Nagle had followed the civil-liberties debate closely,
+for he was an ardent telecommunicator. He was no particular friend
+of computer intruders, but he believed electronic publishing
+had a great deal to offer society at large, and attempts
+to restrain its growth, or to censor free electronic expression,
+strongly roused his ire.
+
+The Neidorf case, and the E911 Document, were both being discussed
+in detail on the Internet, in an electronic publication called Telecom Digest.
+Nagle, a longtime Internet maven, was a regular reader of Telecom Digest.
+Nagle had never seen a copy of Phrack, but the implications of the case
+disturbed him.
+
+While in a Stanford bookstore hunting books on robotics,
+Nagle happened across a book called The Intelligent Network.
+Thumbing through it at random, Nagle came across an entire chapter
+meticulously detailing the workings of E911 police emergency systems.
+This extensive text was being sold openly, and yet in Illinois
+a young man was in danger of going to prison for publishing
+a thin six-page document about 911 service.
+
+Nagle made an ironic comment to this effect in Telecom Digest.
+From there, Nagle was put in touch with Mitch Kapor,
+and then with Neidorf's lawyers.
+
+Sheldon Zenner was delighted to find a computer telecommunications expert
+willing to speak up for Neidorf, one who was not a wacky teenage "hacker."
+Nagle was fluent, mature, and respectable; he'd once had a federal
+security clearance.
+
+Nagle was asked to fly to Illinois to join the defense team.
+
+Having joined the defense as an expert witness, Nagle read the entire
+E911 Document for himself. He made his own judgement about its potential
+for menace.
+
+The time has now come for you yourself, the reader, to have a look
+at the E911 Document. This six-page piece of work was the pretext
+for a federal prosecution that could have sent an electronic publisher
+to prison for thirty, or even sixty, years. It was the pretext
+for the search and seizure of Steve Jackson Games, a legitimate publisher
+of printed books. It was also the formal pretext for the search
+and seizure of the Mentor's bulletin board, "Phoenix Project,"
+and for the raid on the home of Erik Bloodaxe. It also had much
+to do with the seizure of Richard Andrews' Jolnet node
+and the shutdown of Charles Boykin's AT&T node.
+The E911 Document was the single most important piece
+of evidence in the Hacker Crackdown. There can be no real
+and legitimate substitute for the Document itself.
+
+
+==Phrack Inc.==
+
+Volume Two, Issue 24, File 5 of 13
+
+Control Office Administration
+Of Enhanced 911 Services For
+Special Services and Account Centers
+
+by the Eavesdropper
+
+March, 1988
+
+
+Description of Service
+~~~~~~~~~~~~~~~~~~~~~
+The control office for Emergency 911 service is assigned in
+accordance with the existing standard guidelines to one of
+the following centers:
+
+o Special Services Center (SSC)
+o Major Accounts Center (MAC)
+o Serving Test Center (STC)
+o Toll Control Center (TCC)
+
+The SSC/MAC designation is used in this document interchangeably
+for any of these four centers. The Special Services Centers (SSCs)
+or Major Account Centers (MACs) have been designated as the trouble
+reporting contact for all E911 customer (PSAP) reported troubles.
+Subscribers who have trouble on an E911 call will continue
+to contact local repair service (CRSAB) who will refer the
+trouble to the SSC/MAC, when appropriate.
+
+Due to the critical nature of E911 service, the control
+and timely repair of troubles is demanded. As the primary
+E911 customer contact, the SSC/MAC is in the unique position
+to monitor the status of the trouble and insure its resolution.
+
+System Overview
+~~~~~~~~~~~~~~
+The number 911 is intended as a nationwide universal
+telephone number which provides the public with direct
+access to a Public Safety Answering Point (PSAP). A PSAP
+is also referred to as an Emergency Service Bureau (ESB).
+A PSAP is an agency or facility which is authorized by a
+municipality to receive and respond to police, fire and/or
+ambulance services. One or more attendants are located
+at the PSAP facilities to receive and handle calls of an
+emergency nature in accordance with the local municipal
+requirements.
+
+An important advantage of E911 emergency service is
+improved (reduced) response times for emergency
+services. Also close coordination among agencies
+providing various emergency services is a valuable
+capability provided by E911 service.
+
+1A ESS is used as the tandem office for the E911 network to
+route all 911 calls to the correct (primary) PSAP designated
+to serve the calling station. The E911 feature was
+developed primarily to provide routing to the correct PSAP
+for all 911 calls. Selective routing allows a 911 call
+originated from a particular station located in a particular
+district, zone, or town, to be routed to the primary PSAP
+designated to serve that customer station regardless of
+wire center boundaries. Thus, selective routing eliminates
+the problem of wire center boundaries not coinciding with
+district or other political boundaries.
+
+The services available with the E911 feature include:
+
+Forced Disconnect Default Routing
+Alternative Routing Night Service
+Selective Routing Automatic Number
+Identification (ANI)
+Selective Transfer Automatic Location
+Identification (ALI)
+
+
+Preservice/Installation Guidelines
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+When a contract for an E911 system has been signed, it is
+the responsibility of Network Marketing to establish an
+implementation/cutover committee which should include
+a representative from the SSC/MAC. Duties of the E911
+Implementation Team include coordination of all phases
+of the E911 system deployment and the formation of an
+on-going E911 maintenance subcommittee.
+
+Marketing is responsible for providing the following
+customer specific information to the SSC/MAC prior to
+the start of call through testing:
+
+o All PSAP's (name, address, local contact)
+o All PSAP circuit ID's
+o 1004 911 service request including PSAP details on each PSAP
+ (1004 Section K, L, M)
+o Network configuration
+o Any vendor information (name, telephone number, equipment)
+
+The SSC/MAC needs to know if the equipment and sets
+at the PSAP are maintained by the BOCs, an independent
+company, or an outside vendor, or any combination.
+This information is then entered on the PSAP profile sheets
+and reviewed quarterly for changes, additions and deletions.
+
+Marketing will secure the Major Account Number (MAN)
+and provide this number to Corporate Communications
+so that the initial issue of the service orders carry
+the MAN and can be tracked by the SSC/MAC via CORDNET.
+PSAP circuits are official services by definition.
+
+All service orders required for the installation of the E911
+system should include the MAN assigned to the city/county
+which has purchased the system.
+
+In accordance with the basic SSC/MAC strategy for provisioning,
+the SSC/MAC will be Overall Control Office (OCO) for all Node
+to PSAP circuits (official services) and any other services
+for this customer. Training must be scheduled for all SSC/MAC
+involved personnel during the pre-service stage of the project.
+
+The E911 Implementation Team will form the on-going
+maintenance subcommittee prior to the initial
+implementation of the E911 system. This sub-committee
+will establish post implementation quality assurance
+procedures to ensure that the E911 system continues to
+provide quality service to the customer.
+Customer/Company training, trouble reporting interfaces
+for the customer, telephone company and any involved
+independent telephone companies needs to be addressed
+and implemented prior to E911 cutover. These functions
+can be best addressed by the formation of a sub-
+committee of the E911 Implementation Team to set up
+guidelines for and to secure service commitments of
+interfacing organizations. A SSC/MAC supervisor should
+chair this subcommittee and include the following
+organizations:
+
+1) Switching Control Center
+ - E911 translations
+ - Trunking
+ - End office and Tandem office hardware/software
+2) Recent Change Memory Administration Center
+ - Daily RC update activity for TN/ESN translations
+ - Processes validity errors and rejects
+3) Line and Number Administration
+ - Verification of TN/ESN translations
+4) Special Service Center/Major Account Center
+ - Single point of contact for all PSAP and Node to host troubles
+ - Logs, tracks & statusing of all trouble reports
+ - Trouble referral, follow up, and escalation
+ - Customer notification of status and restoration
+ - Analyzation of "chronic" troubles
+ - Testing, installation and maintenance of E911 circuits
+5) Installation and Maintenance (SSIM/I&M)
+ - Repair and maintenance of PSAP equipment and Telco owned sets
+6) Minicomputer Maintenance Operations Center
+ - E911 circuit maintenance (where applicable)
+7) Area Maintenance Engineer
+ - Technical assistance on voice (CO-PSAP) network related E911 troubles
+
+
+Maintenance Guidelines
+~~~~~~~~~~~~~~~~~~~~~
+The CCNC will test the Node circuit from the 202T at the
+Host site to the 202T at the Node site. Since Host to Node
+(CCNC to MMOC) circuits are official company services,
+the CCNC will refer all Node circuit troubles to the
+SSC/MAC. The SSC/MAC is responsible for the testing
+and follow up to restoration of these circuit troubles.
+
+Although Node to PSAP circuit are official services, the
+MMOC will refer PSAP circuit troubles to the appropriate
+SSC/MAC. The SSC/MAC is responsible for testing and
+follow up to restoration of PSAP circuit troubles.
+
+The SSC/MAC will also receive reports from
+CRSAB/IMC(s) on subscriber 911 troubles when they are
+not line troubles. The SSC/MAC is responsible for testing
+and restoration of these troubles.
+
+Maintenance responsibilities are as follows:
+
+SCC@ Voice Network (ANI to PSAP)
+@SCC responsible for tandem switch
+
+SSIM/I&M PSAP Equipment (Modems, CIU's, sets)
+Vendor PSAP Equipment (when CPE)
+SSC/MAC PSAP to Node circuits, and tandem to
+ PSAP voice circuits (EMNT)
+MMOC Node site (Modems, cables, etc)
+
+Note: All above work groups are required to resolve troubles
+by interfacing with appropriate work groups for resolution.
+
+The Switching Control Center (SCC) is responsible for
+E911/1AESS translations in tandem central offices.
+These translations route E911 calls, selective transfer,
+default routing, speed calling, etc., for each PSAP.
+The SCC is also responsible for troubleshooting on
+the voice network (call originating to end office tandem equipment).
+
+For example, ANI failures in the originating offices would
+be a responsibility of the SCC.
+
+Recent Change Memory Administration Center (RCMAC) performs
+the daily tandem translation updates (recent change)
+for routing of individual telephone numbers.
+
+Recent changes are generated from service order activity
+(new service, address changes, etc.) and compiled into
+a daily file by the E911 Center (ALI/DMS E911 Computer).
+
+SSIM/I&M is responsible for the installation and repair of
+PSAP equipment. PSAP equipment includes ANI Controller,
+ALI Controller, data sets, cables, sets, and other peripheral
+equipment that is not vendor owned. SSIM/I&M is responsible
+for establishing maintenance test kits, complete with spare parts
+for PSAP maintenance. This includes test gear, data sets,
+and ANI/ALI Controller parts.
+
+Special Services Center (SSC) or Major Account Center
+(MAC) serves as the trouble reporting contact for all
+(PSAP) troubles reported by customer. The SSC/MAC
+refers troubles to proper organizations for handling and
+tracks status of troubles, escalating when necessary.
+The SSC/MAC will close out troubles with customer.
+The SSC/MAC will analyze all troubles and tracks "chronic"
+PSAP troubles.
+
+Corporate Communications Network Center (CCNC) will
+test and refer troubles on all node to host circuits.
+All E911 circuits are classified as official company property.
+
+The Minicomputer Maintenance Operations Center
+(MMOC) maintains the E911 (ALI/DMS) computer
+hardware at the Host site. This MMOC is also responsible
+for monitoring the system and reporting certain PSAP
+and system problems to the local MMOC's, SCC's or
+SSC/MAC's. The MMOC personnel also operate software
+programs that maintain the TN data base under the
+direction of the E911 Center. The maintenance of the
+NODE computer (the interface between the PSAP and the
+ALI/DMS computer) is a function of the MMOC at the
+NODE site. The MMOC's at the NODE sites may also be
+involved in the testing of NODE to Host circuits.
+The MMOC will also assist on Host to PSAP and data network
+related troubles not resolved through standard trouble
+clearing procedures.
+
+Installation And Maintenance Center (IMC) is responsible
+for referral of E911 subscriber troubles that are not subscriber
+line problems.
+
+E911 Center - Performs the role of System Administration
+and is responsible for overall operation of the E911
+computer software. The E911 Center does A-Z trouble
+analysis and provides statistical information on the
+performance of the system.
+
+This analysis includes processing PSAP inquiries (trouble
+reports) and referral of network troubles. The E911 Center
+also performs daily processing of tandem recent change
+and provides information to the RCMAC for tandem input.
+The E911 Center is responsible for daily processing
+of the ALI/DMS computer data base and provides error files,
+etc. to the Customer Services department for investigation and correction.
+The E911 Center participates in all system implementations and on-going
+maintenance effort and assists in the development of procedures,
+training and education of information to all groups.
+
+Any group receiving a 911 trouble from the SSC/MAC should
+close out the trouble with the SSC/MAC or provide a status
+if the trouble has been referred to another group.
+This will allow the SSC/MAC to provide a status back
+to the customer or escalate as appropriate.
+
+Any group receiving a trouble from the Host site (MMOC
+or CCNC) should close the trouble back to that group.
+
+The MMOC should notify the appropriate SSC/MAC
+when the Host, Node, or all Node circuits are down so that
+the SSC/MAC can reply to customer reports that may be
+called in by the PSAPs. This will eliminate duplicate
+reporting of troubles. On complete outages the MMOC
+will follow escalation procedures for a Node after two (2)
+hours and for a PSAP after four (4) hours. Additionally the
+MMOC will notify the appropriate SSC/MAC when the
+Host, Node, or all Node circuits are down.
+
+The PSAP will call the SSC/MAC to report E911 troubles.
+The person reporting the E911 trouble may not have a
+circuit I.D. and will therefore report the PSAP name and
+address. Many PSAP troubles are not circuit specific. In
+those instances where the caller cannot provide a circuit
+I.D., the SSC/MAC will be required to determine the
+circuit I.D. using the PSAP profile. Under no circumstances
+will the SSC/MAC Center refuse to take the trouble.
+The E911 trouble should be handled as quickly as possible,
+with the SSC/MAC providing as much assistance as
+possible while taking the trouble report from the caller.
+
+The SSC/MAC will screen/test the trouble to determine the
+appropriate handoff organization based on the following criteria:
+
+PSAP equipment problem: SSIM/I&M
+Circuit problem: SSC/MAC
+Voice network problem: SCC (report trunk group number)
+Problem affecting multiple PSAPs (No ALI report from
+all PSAPs): Contact the MMOC to check for NODE or
+Host computer problems before further testing.
+
+The SSC/MAC will track the status of reported troubles
+and escalate as appropriate. The SSC/MAC will close out
+customer/company reports with the initiating contact.
+Groups with specific maintenance responsibilities,
+defined above, will investigate "chronic" troubles upon
+request from the SSC/MAC and the ongoing maintenance subcommittee.
+
+All "out of service" E911 troubles are priority one type reports.
+One link down to a PSAP is considered a priority one trouble
+and should be handled as if the PSAP was isolated.
+
+The PSAP will report troubles with the ANI controller, ALI
+controller or set equipment to the SSC/MAC.
+
+NO ANI: Where the PSAP reports NO ANI (digital
+display screen is blank) ask if this condition exists on all
+screens and on all calls. It is important to differentiate
+between blank screens and screens displaying 911-00XX,
+or all zeroes.
+
+When the PSAP reports all screens on all calls, ask if there
+is any voice contact with callers. If there is no voice
+contact the trouble should be referred to the SCC
+immediately since 911 calls are not getting through which
+may require alternate routing of calls to another PSAP.
+
+When the PSAP reports this condition on all screens
+but not all calls and has voice contact with callers,
+the report should be referred to SSIM/I&M for dispatch.
+The SSC/MAC should verify with the SCC that ANI
+is pulsing before dispatching SSIM.
+
+When the PSAP reports this condition on one screen for
+all calls (others work fine) the trouble should be referred
+to SSIM/I&M for dispatch, because the trouble is isolated to
+one piece of equipment at the customer premise.
+
+An ANI failure (i.e. all zeroes) indicates that the ANI has
+not been received by the PSAP from the tandem office or
+was lost by the PSAP ANI controller. The PSAP may
+receive "02" alarms which can be caused by the ANI
+controller logging more than three all zero failures on the
+same trunk. The PSAP has been instructed to report this
+condition to the SSC/MAC since it could indicate an
+equipment trouble at the PSAP which might be affecting
+all subscribers calling into the PSAP. When all zeroes are
+being received on all calls or "02" alarms continue, a tester
+should analyze the condition to determine the appropriate
+action to be taken. The tester must perform cooperative
+testing with the SCC when there appears to be a problem
+on the Tandem-PSAP trunks before requesting dispatch.
+
+When an occasional all zero condition is reported,
+the SSC/MAC should dispatch SSIM/I&M to routine
+equipment on a "chronic" troublesweep.
+
+The PSAPs are instructed to report incidental ANI failures
+to the BOC on a PSAP inquiry trouble ticket (paper) that
+is sent to the Customer Services E911 group and forwarded
+to E911 center when required. This usually involves only a
+particular telephone number and is not a condition that
+would require a report to the SSC/MAC. Multiple ANI
+failures which our from the same end office (XX denotes
+end office), indicate a hard trouble condition may exist
+in the end office or end office tandem trunks. The PSAP will
+report this type of condition to the SSC/MAC and the
+SSC/MAC should refer the report to the SCC responsible
+for the tandem office. NOTE: XX is the ESCO (Emergency
+Service Number) associated with the incoming 911 trunks
+into the tandem. It is important that the C/MAC tell the
+SCC what is displayed at the PSAP (i.e. 911-0011) which
+indicates to the SCC which end office is in trouble.
+
+Note: It is essential that the PSAP fill out inquiry form
+on every ANI failure.
+
+The PSAP will report a trouble any time an address is not
+received on an address display (screen blank) E911 call.
+(If a record is not in the 911 data base or an ANI failure
+is encountered, the screen will provide a display noticing
+such condition). The SSC/MAC should verify with the PSAP
+whether the NO ALI condition is on one screen or all screens.
+
+When the condition is on one screen (other screens
+receive ALI information) the SSC/MAC will request
+SSIM/I&M to dispatch.
+
+If no screens are receiving ALI information, there is usually
+a circuit trouble between the PSAP and the Host computer.
+The SSC/MAC should test the trouble and refer for restoral.
+
+Note: If the SSC/MAC receives calls from multiple
+PSAP's, all of which are receiving NO ALI, there is a
+problem with the Node or Node to Host circuits or the
+Host computer itself. Before referring the trouble the
+SSC/MAC should call the MMOC to inquire if the Node
+or Host is in trouble.
+
+Alarm conditions on the ANI controller digital display at
+the PSAP are to be reported by the PSAP's. These alarms
+can indicate various trouble conditions so the SSC/MAC
+should ask the PSAP if any portion of the E911 system
+is not functioning properly.
+
+The SSC/MAC should verify with the PSAP attendant that
+the equipment's primary function is answering E911 calls.
+If it is, the SSC/MAC should request a dispatch SSIM/I&M.
+If the equipment is not primarily used for E911,
+then the SSC/MAC should advise PSAP to contact their CPE vendor.
+
+Note: These troubles can be quite confusing when the
+PSAP has vendor equipment mixed in with equipment
+that the BOC maintains. The Marketing representative
+should provide the SSC/MAC information concerning any
+unusual or exception items where the PSAP should
+contact their vendor. This information should be included
+in the PSAP profile sheets.
+
+ANI or ALI controller down: When the host computer sees
+the PSAP equipment down and it does not come back up,
+the MMOC will report the trouble to the SSC/MAC;
+the equipment is down at the PSAP, a dispatch will be required.
+
+PSAP link (circuit) down: The MMOC will provide the
+SSC/MAC with the circuit ID that the Host computer
+indicates in trouble. Although each PSAP has two circuits,
+when either circuit is down the condition must be treated
+as an emergency since failure of the second circuit will
+cause the PSAP to be isolated.
+
+Any problems that the MMOC identifies from the Node
+location to the Host computer will be handled directly
+with the appropriate MMOC(s)/CCNC.
+
+Note: The customer will call only when a problem is
+apparent to the PSAP. When only one circuit is down to
+the PSAP, the customer may not be aware there is a
+trouble, even though there is one link down,
+notification should appear on the PSAP screen.
+Troubles called into the SSC/MAC from the MMOC
+or other company employee should not be closed out
+by calling the PSAP since it may result in the
+customer responding that they do not have a trouble.
+These reports can only be closed out by receiving
+information that the trouble was fixed and by checking
+with the company employee that reported the trouble.
+The MMOC personnel will be able to verify that the
+trouble has cleared by reviewing a printout from the host.
+
+When the CRSAB receives a subscriber complaint
+(i.e., cannot dial 911) the RSA should obtain as much
+information as possible while the customer is on the line.
+
+For example, what happened when the subscriber dialed 911?
+The report is automatically directed to the IMC for subscriber line testing.
+When no line trouble is found, the IMC will refer the trouble condition
+to the SSC/MAC. The SSC/MAC will contact Customer Services E911 Group
+and verify that the subscriber should be able to call 911 and obtain the ESN.
+The SSC/MAC will verify the ESN via 2SCCS. When both verifications match,
+the SSC/MAC will refer the report to the SCC responsible for the 911 tandem
+office for investigation and resolution. The MAC is responsible for tracking
+the trouble and informing the IMC when it is resolved.
+
+
+For more information, please refer to E911 Glossary of Terms.
+End of Phrack File
+_____________________________________
+
+
+The reader is forgiven if he or she was entirely unable to read
+this document. John Perry Barlow had a great deal of fun at its expense,
+in "Crime and Puzzlement:" "Bureaucrat-ese of surpassing opacity. . . .
+To read the whole thing straight through without entering coma requires
+either a machine or a human who has too much practice thinking like one.
+Anyone who can understand it fully and fluidly had altered his consciousness
+beyond the ability to ever again read Blake, Whitman, or Tolstoy. . . .
+the document contains little of interest to anyone who is not a student
+of advanced organizational sclerosis."
+
+With the Document itself to hand, however, exactly as it was published
+(in its six-page edited form) in Phrack, the reader may be able to verify
+a few statements of fact about its nature. First, there is no software,
+no computer code, in the Document. It is not computer-programming language
+like FORTRAN or C++, it is English; all the sentences have nouns and verbs
+and punctuation. It does not explain how to break into the E911 system.
+It does not suggest ways to destroy or damage the E911 system.
+
+There are no access codes in the Document. There are no computer passwords.
+It does not explain how to steal long distance service. It does not explain
+how to break in to telco switching stations. There is nothing in it about
+using a personal computer or a modem for any purpose at all, good or bad.
+
+Close study will reveal that this document is not about machinery.
+The E911 Document is about ADMINISTRATION. It describes how one creates
+and administers certain units of telco bureaucracy:
+Special Service Centers and Major Account Centers (SSC/MAC).
+It describes how these centers should distribute responsibility
+for the E911 service, to other units of telco bureaucracy,
+in a chain of command, a formal hierarchy. It describes
+who answers customer complaints, who screens calls,
+who reports equipment failures, who answers those reports,
+who handles maintenance, who chairs subcommittees,
+who gives orders, who follows orders, WHO tells WHOM what to do.
+The Document is not a "roadmap" to computers.
+The Document is a roadmap to PEOPLE.
+
+As an aid to breaking into computer systems, the Document is USELESS.
+As an aid to harassing and deceiving telco people, however, the Document
+might prove handy (especially with its Glossary, which I have not included).
+An intense and protracted study of this Document and its Glossary,
+combined with many other such documents, might teach one to speak like
+a telco employee. And telco people live by SPEECH--they live by phone
+communication. If you can mimic their language over the phone,
+you can "social-engineer" them. If you can con telco people, you can
+wreak havoc among them. You can force them to no longer trust one another;
+you can break the telephonic ties that bind their community; you can make
+them paranoid. And people will fight harder to defend their community
+than they will fight to defend their individual selves.
+
+This was the genuine, gut-level threat posed by Phrack magazine.
+The real struggle was over the control of telco language,
+the control of telco knowledge. It was a struggle to defend the social
+"membrane of differentiation" that forms the walls of the telco
+community's ivory tower --the special jargon that allows telco
+professionals to recognize one another, and to exclude charlatans,
+thieves, and upstarts. And the prosecution brought out this fact.
+They repeatedly made reference to the threat posed to telco professionals
+by hackers using "social engineering."
+
+However, Craig Neidorf was not on trial for learning to speak like
+a professional telecommunications expert. Craig Neidorf was on trial
+for access device fraud and transportation of stolen property.
+He was on trial for stealing a document that was purportedly
+highly sensitive and purportedly worth tens of thousands of dollars.
+
+#
+
+John Nagle read the E911 Document. He drew his own conclusions.
+And he presented Zenner and his defense team with an overflowing box
+of similar material, drawn mostly from Stanford University's
+engineering libraries. During the trial, the defense team--Zenner,
+half-a-dozen other attorneys, Nagle, Neidorf, and computer-security
+expert Dorothy Denning, all pored over the E911 Document line-by-line.
+
+On the afternoon of July 25, 1990, Zenner began to cross-examine
+a woman named Billie Williams, a service manager for Southern Bell
+in Atlanta. Ms. Williams had been responsible for the E911 Document.
+(She was not its author--its original "author" was a Southern Bell
+staff manager named Richard Helms. However, Mr. Helms should not bear
+the entire blame; many telco staff people and maintenance personnel
+had amended the Document. It had not been so much "written" by a
+single author, as built by committee out of concrete-blocks of jargon.)
+
+Ms. Williams had been called as a witness for the prosecution,
+and had gamely tried to explain the basic technical structure
+of the E911 system, aided by charts.
+
+Now it was Zenner's turn. He first established that the
+"proprietary stamp" that BellSouth had used on the E911 Document
+was stamped on EVERY SINGLE DOCUMENT that BellSouth wrote--
+THOUSANDS of documents. "We do not publish anything other
+than for our own company," Ms. Williams explained.
+"Any company document of this nature is considered proprietary."
+Nobody was in charge of singling out special high-security publications
+for special high-security protection. They were ALL special,
+no matter how trivial, no matter what their subject matter--
+the stamp was put on as soon as any document was written,
+and the stamp was never removed.
+
+Zenner now asked whether the charts she had been using to explain
+the mechanics of E911 system were "proprietary," too.
+Were they PUBLIC INFORMATION, these charts, all about PSAPs,
+ALIs, nodes, local end switches? Could he take the charts out
+in the street and show them to anybody, "without violating
+some proprietary notion that BellSouth has?"
+
+Ms Williams showed some confusion, but finally areed that the charts were,
+in fact, public.
+
+"But isn't this what you said was basically what appeared in Phrack?"
+
+Ms. Williams denied this.
+
+Zenner now pointed out that the E911 Document as published in Phrack
+was only half the size of the original E911 Document (as Prophet
+had purloined it). Half of it had been deleted--edited by Neidorf.
+
+Ms. Williams countered that "Most of the information that is
+in the text file is redundant."
+
+Zenner continued to probe. Exactly what bits of knowledge in the Document
+were, in fact, unknown to the public? Locations of E911 computers?
+Phone numbers for telco personnel? Ongoing maintenance subcommittees?
+Hadn't Neidorf removed much of this?
+
+Then he pounced. "Are you familiar with Bellcore Technical Reference
+Document TR-TSY-000350?" It was, Zenner explained, officially titled
+"E911 Public Safety Answering Point Interface Between 1-1AESS Switch
+and Customer Premises Equipment." It contained highly detailed
+and specific technical information about the E911 System.
+It was published by Bellcore and publicly available for about $20.
+
+He showed the witness a Bellcore catalog which listed thousands
+of documents from Bellcore and from all the Baby Bells, BellSouth included.
+The catalog, Zenner pointed out, was free. Anyone with a credit card
+could call the Bellcore toll-free 800 number and simply order any
+of these documents, which would be shipped to any customer without question.
+Including, for instance, "BellSouth E911 Service Interfaces to
+Customer Premises Equipment at a Public Safety Answering Point."
+
+Zenner gave the witness a copy of "BellSouth E911 Service Interfaces,"
+which cost, as he pointed out, $13, straight from the catalog.
+"Look at it carefully," he urged Ms. Williams, "and tell me
+if it doesn't contain about twice as much detailed information
+about the E911 system of BellSouth than appeared anywhere in Phrack."
+
+"You want me to. . . ." Ms. Williams trailed off. "I don't understand."
+
+"Take a careful look," Zenner persisted. "Take a look at that document,
+and tell me when you're done looking at it if, indeed, it doesn't contain
+much more detailed information about the E911 system than appeared in Phrack."
+
+"Phrack wasn't taken from this," Ms. Williams said.
+
+"Excuse me?" said Zenner.
+
+"Phrack wasn't taken from this."
+
+"I can't hear you," Zenner said.
+
+"Phrack was not taken from this document. I don't understand
+your question to me."
+
+"I guess you don't," Zenner said.
+
+At this point, the prosecution's case had been gutshot.
+Ms. Williams was distressed. Her confusion was quite genuine.
+Phrack had not been taken from any publicly available Bellcore document.
+Phrack's E911 Document had been stolen from her own company's computers,
+from her own company's text files, that her own colleagues had written,
+and revised, with much labor.
+
+But the "value" of the Document had been blown to smithereens.
+It wasn't worth eighty grand. According to Bellcore it was worth
+thirteen bucks. And the looming menace that it supposedly posed
+had been reduced in instants to a scarecrow. Bellcore itself
+was selling material far more detailed and "dangerous,"
+to anybody with a credit card and a phone.
+
+Actually, Bellcore was not giving this information to just anybody.
+They gave it to ANYBODY WHO ASKED, but not many did ask.
+Not many people knew that Bellcore had a free catalog and an 800 number.
+John Nagle knew, but certainly the average teenage phreak didn't know.
+"Tuc," a friend of Neidorf's and sometime Phrack contributor, knew,
+and Tuc had been very helpful to the defense, behind the scenes.
+But the Legion of Doom didn't know--otherwise, they would never
+have wasted so much time raiding dumpsters. Cook didn't know.
+Foley didn't know. Kluepfel didn't know. The right hand
+of Bellcore knew not what the left hand was doing. The right
+hand was battering hackers without mercy, while the left hand
+was distributing Bellcore's intellectual property to anybody
+who was interested in telephone technical trivia--apparently,
+a pathetic few.
+
+The digital underground was so amateurish and poorly organized
+that they had never discovered this heap of unguarded riches.
+The ivory tower of the telcos was so wrapped-up in the fog
+of its own technical obscurity that it had left all the
+windows open and flung open the doors. No one had even noticed.
+
+Zenner sank another nail in the coffin. He produced a printed issue
+of Telephone Engineer & Management, a prominent industry journal
+that comes out twice a month and costs $27 a year. This particular issue
+of TE&M, called "Update on 911," featured a galaxy of technical details
+on 911 service and a glossary far more extensive than Phrack's.
+
+The trial rumbled on, somehow, through its own momentum.
+Tim Foley testified about his interrogations of Neidorf.
+Neidorf's written admission that he had known the E911 Document
+was pilfered was officially read into the court record.
+
+An interesting side issue came up: "Terminus" had once passed Neidorf
+a piece of UNIX AT&T software, a log-in sequence, that had been cunningly
+altered so that it could trap passwords. The UNIX software itself was
+illegally copied AT&T property, and the alterations "Terminus" had made to it,
+had transformed it into a device for facilitating computer break-ins. Terminus
+himself would eventually plead guilty to theft of this piece of software,
+and the Chicago group would send Terminus to prison for it. But it was
+of dubious relevance in the Neidorf case. Neidorf hadn't written the program.
+He wasn't accused of ever having used it. And Neidorf wasn't being charged
+with software theft or owning a password trapper.
+
+On the next day, Zenner took the offensive. The civil libertarians
+now had their own arcane, untried legal weaponry to launch into action--
+the Electronic Communications Privacy Act of 1986, 18 US Code,
+Section 2701 et seq. Section 2701 makes it a crime to intentionally
+access without authorization a facility in which an electronic communication
+service is provided--it is, at heart, an anti-bugging and anti-tapping law,
+intended to carry the traditional protections of telephones into other
+electronic channels of communication. While providing penalties for amateur
+snoops, however, Section 2703 of the ECPA also lays some formal difficulties
+on the bugging and tapping activities of police.
+
+The Secret Service, in the person of Tim Foley, had served Richard Andrews
+with a federal grand jury subpoena, in their pursuit of Prophet,
+the E911 Document, and the Terminus software ring. But according to
+the Electronic Communications Privacy Act, a "provider of remote
+computing service" was legally entitled to "prior notice" from
+the government if a subpoena was used. Richard Andrews and his
+basement UNIX node, Jolnet, had not received any "prior notice."
+Tim Foley had purportedly violated the ECPA and committed
+an electronic crime! Zenner now sought the judge's permission
+to cross-examine Foley on the topic of Foley's own electronic misdeeds.
+
+Cook argued that Richard Andrews' Jolnet was a privately owned
+bulletin board, and not within the purview of ECPA. Judge Bua
+granted the motion of the government to prevent cross-examination
+on that point, and Zenner's offensive fizzled. This, however,
+was the first direct assault on the legality of the actions
+of the Computer Fraud and Abuse Task Force itself--
+the first suggestion that they themselves had broken the law,
+and might, perhaps, be called to account.
+
+Zenner, in any case, did not really need the ECPA.
+Instead, he grilled Foley on the glaring contradictions in
+the supposed value of the E911 Document. He also brought up
+the embarrassing fact that the supposedly red-hot E911 Document
+had been sitting around for months, in Jolnet, with Kluepfel's knowledge,
+while Kluepfel had done nothing about it.
+
+In the afternoon, the Prophet was brought in to testify
+for the prosecution. (The Prophet, it will be recalled,
+had also been indicted in the case as partner in a fraud
+scheme with Neidorf.) In Atlanta, the Prophet had already
+pled guilty to one charge of conspiracy, one charge of wire fraud
+and one charge of interstate transportation of stolen property.
+The wire fraud charge, and the stolen property charge,
+were both directly based on the E911 Document.
+
+The twenty-year-old Prophet proved a sorry customer,
+answering questions politely but in a barely audible mumble,
+his voice trailing off at the ends of sentences.
+He was constantly urged to speak up.
+
+Cook, examining Prophet, forced him to admit that
+he had once had a "drug problem," abusing amphetamines,
+marijuana, cocaine, and LSD. This may have established
+to the jury that "hackers" are, or can be, seedy lowlife characters,
+but it may have damaged Prophet's credibility somewhat.
+Zenner later suggested that drugs might have damaged Prophet's memory.
+The interesting fact also surfaced that Prophet had never
+physically met Craig Neidorf. He didn't even know
+Neidorf's last name--at least, not until the trial.
+
+Prophet confirmed the basic facts of his hacker career.
+He was a member of the Legion of Doom. He had abused codes,
+he had broken into switching stations and re-routed calls,
+he had hung out on pirate bulletin boards. He had raided
+the BellSouth AIMSX computer, copied the E911 Document,
+stored it on Jolnet, mailed it to Neidorf. He and Neidorf
+had edited it, and Neidorf had known where it came from.
+
+Zenner, however, had Prophet confirm that Neidorf was not a member
+of the Legion of Doom, and had not urged Prophet to break into
+BellSouth computers. Neidorf had never urged Prophet to defraud anyone,
+or to steal anything. Prophet also admitted that he had never known Neidorf
+to break in to any computer. Prophet said that no one in the Legion of Doom
+considered Craig Neidorf a "hacker" at all. Neidorf was not a UNIX maven,
+and simply lacked the necessary skill and ability to break into computers.
+Neidorf just published a magazine.
+
+On Friday, July 27, 1990, the case against Neidorf collapsed.
+Cook moved to dismiss the indictment, citing "information currently
+available to us that was not available to us at the inception of the trial."
+Judge Bua praised the prosecution for this action, which he described as
+"very responsible," then dismissed a juror and declared a mistrial.
+
+Neidorf was a free man. His defense, however, had cost himself
+and his family dearly. Months of his life had been consumed in anguish;
+he had seen his closest friends shun him as a federal criminal.
+He owed his lawyers over a hundred thousand dollars, despite
+a generous payment to the defense by Mitch Kapor.
+
+Neidorf was not found innocent. The trial was simply dropped.
+Nevertheless, on September 9, 1991, Judge Bua granted Neidorf's
+motion for the "expungement and sealing" of his indictment record.
+The United States Secret Service was ordered to delete and destroy
+all fingerprints, photographs, and other records of arrest
+or processing relating to Neidorf's indictment, including
+their paper documents and their computer records.
+
+Neidorf went back to school, blazingly determined to become a lawyer.
+Having seen the justice system at work, Neidorf lost much of his enthusiasm
+for merely technical power. At this writing, Craig Neidorf is working
+in Washington as a salaried researcher for the American Civil Liberties Union.
+
+#
+
+The outcome of the Neidorf trial changed the EFF
+from voices-in-the-wilderness to the media darlings
+of the new frontier.
+
+Legally speaking, the Neidorf case was not a sweeping triumph
+for anyone concerned. No constitutional principles had been established.
+The issues of "freedom of the press" for electronic publishers remained
+in legal limbo. There were public misconceptions about the case.
+Many people thought Neidorf had been found innocent and relieved
+of all his legal debts by Kapor. The truth was that the government
+had simply dropped the case, and Neidorf's family had gone deeply
+into hock to support him.
+
+But the Neidorf case did provide a single, devastating, public sound-bite:
+THE FEDS SAID IT WAS WORTH EIGHTY GRAND, AND IT WAS ONLY WORTH THIRTEEN BUCKS.
+
+This is the Neidorf case's single most memorable element. No serious report
+of the case missed this particular element. Even cops could not read this
+without a wince and a shake of the head. It left the public credibility
+of the crackdown agents in tatters.
+
+The crackdown, in fact, continued, however. Those two charges
+against Prophet, which had been based on the E911 Document,
+were quietly forgotten at his sentencing--even though Prophet
+had already pled guilty to them. Georgia federal prosecutors
+strongly argued for jail time for the Atlanta Three, insisting on
+"the need to send a message to the community," "the message that
+hackers around the country need to hear."
+
+There was a great deal in their sentencing memorandum
+about the awful things that various other hackers had done
+(though the Atlanta Three themselves had not, in fact,
+actually committed these crimes). There was also much
+speculation about the awful things that the Atlanta Three
+MIGHT have done and WERE CAPABLE of doing (even though
+they had not, in fact, actually done them).
+The prosecution's argument carried the day.
+The Atlanta Three were sent to prison:
+Urvile and Leftist both got 14 months each,
+while Prophet (a second offender) got 21 months.
+
+The Atlanta Three were also assessed staggering fines as "restitution":
+$233,000 each. BellSouth claimed that the defendants had "stolen"
+"approximately $233,880 worth" of "proprietary computer access information"--
+specifically, $233,880 worth of computer passwords and connect addresses.
+BellSouth's astonishing claim of the extreme value of its own computer
+passwords and addresses was accepted at face value by the Georgia court.
+Furthermore (as if to emphasize its theoretical nature) this enormous sum
+was not divvied up among the Atlanta Three, but each of them had to pay
+all of it.
+
+A striking aspect of the sentence was that the Atlanta Three were
+specifically forbidden to use computers, except for work or under supervision.
+Depriving hackers of home computers and modems makes some sense if one
+considers hackers as "computer addicts," but EFF, filing an amicus brief
+in the case, protested that this punishment was unconstitutional--
+it deprived the Atlanta Three of their rights of free association
+and free expression through electronic media.
+
+Terminus, the "ultimate hacker," was finally sent to prison for a year
+through the dogged efforts of the Chicago Task Force. His crime,
+to which he pled guilty, was the transfer of the UNIX password trapper,
+which was officially valued by AT&T at $77,000, a figure which aroused
+intense skepticism among those familiar with UNIX "login.c" programs.
+
+The jailing of Terminus and the Atlanta Legionnaires of Doom, however,
+did not cause the EFF any sense of embarrassment or defeat.
+On the contrary, the civil libertarians were rapidly gathering strength.
+
+An early and potent supporter was Senator Patrick Leahy,
+Democrat from Vermont, who had been a Senate sponsor
+of the Electronic Communications Privacy Act. Even before
+the Neidorf trial, Leahy had spoken out in defense of hacker-power
+and freedom of the keyboard: "We cannot unduly inhibit the inquisitive
+13-year-old who, if left to experiment today, may tomorrow develop
+the telecommunications or computer technology to lead the United States
+into the 21st century. He represents our future and our best hope
+to remain a technologically competitive nation."
+
+It was a handsome statement, rendered perhaps rather more effective
+by the fact that the crackdown raiders DID NOT HAVE any Senators
+speaking out for THEM. On the contrary, their highly secretive
+actions and tactics, all "sealed search warrants" here and
+"confidential ongoing investigations" there, might have won
+them a burst of glamorous publicity at first, but were crippling
+them in the on-going propaganda war. Gail Thackeray was reduced
+to unsupported bluster: "Some of these people who are loudest
+on the bandwagon may just slink into the background,"
+she predicted in Newsweek--when all the facts came out,
+and the cops were vindicated.
+
+But all the facts did not come out. Those facts that did,
+were not very flattering. And the cops were not vindicated.
+And Gail Thackeray lost her job. By the end of 1991,
+William Cook had also left public employment.
+
+1990 had belonged to the crackdown, but by '91 its agents
+were in severe disarray, and the libertarians were on a roll.
+People were flocking to the cause.
+
+A particularly interesting ally had been Mike Godwin of Austin, Texas.
+Godwin was an individual almost as difficult to describe as Barlow;
+he had been editor of the student newspaper of the University of Texas,
+and a computer salesman, and a programmer, and in 1990 was back
+in law school, looking for a law degree.
+
+Godwin was also a bulletin board maven. He was very well-known
+in the Austin board community under his handle "Johnny Mnemonic,"
+which he adopted from a cyberpunk science fiction story by William Gibson.
+Godwin was an ardent cyberpunk science fiction fan. As a fellow Austinite
+of similar age and similar interests, I myself had known Godwin socially
+for many years. When William Gibson and myself had been writing our
+collaborative SF novel, The Difference Engine, Godwin had been our
+technical advisor in our effort to link our Apple word-processors
+from Austin to Vancouver. Gibson and I were so pleased by his generous
+expert help that we named a character in the novel "Michael Godwin"
+in his honor.
+
+The handle "Mnemonic" suited Godwin very well. His erudition
+and his mastery of trivia were impressive to the point of stupor;
+his ardent curiosity seemed insatiable, and his desire to debate
+and argue seemed the central drive of his life. Godwin had even
+started his own Austin debating society, wryly known as the
+"Dull Men's Club." In person, Godwin could be overwhelming;
+a flypaper-brained polymath who could not seem to let any idea go.
+On bulletin boards, however, Godwin's closely reasoned,
+highly grammatical, erudite posts suited the medium well,
+and he became a local board celebrity.
+
+Mike Godwin was the man most responsible for the public national exposure
+of the Steve Jackson case. The Izenberg seizure in Austin had received
+no press coverage at all. The March 1 raids on Mentor, Bloodaxe, and
+Steve Jackson Games had received a brief front-page splash in the
+front page of the Austin American-Statesman, but it was confused
+and ill-informed: the warrants were sealed, and the Secret Service
+wasn't talking. Steve Jackson seemed doomed to obscurity.
+Jackson had not been arrested; he was not charged with any crime;
+he was not on trial. He had lost some computers in an ongoing
+investigation--so what? Jackson tried hard to attract attention
+to the true extent of his plight, but he was drawing a blank;
+no one in a position to help him seemed able to get a mental grip
+on the issues.
+
+Godwin, however, was uniquely, almost magically, qualified
+to carry Jackson's case to the outside world. Godwin was
+a board enthusiast, a science fiction fan, a former journalist,
+a computer salesman, a lawyer-to-be, and an Austinite.
+Through a coincidence yet more amazing, in his last year
+of law school Godwin had specialized in federal prosecutions
+and criminal procedure. Acting entirely on his own, Godwin made
+up a press packet which summarized the issues and provided useful
+contacts for reporters. Godwin's behind-the-scenes effort
+(which he carried out mostly to prove a point in a local board debate)
+broke the story again in the Austin American-Statesman and then in Newsweek.
+
+Life was never the same for Mike Godwin after that. As he joined the growing
+civil liberties debate on the Internet, it was obvious to all parties involved
+that here was one guy who, in the midst of complete murk and confusion,
+GENUINELY UNDERSTOOD EVERYTHING HE WAS TALKING ABOUT. The disparate elements
+of Godwin's dilettantish existence suddenly fell together as neatly as
+the facets of a Rubik's cube.
+
+When the time came to hire a full-time EFF staff attorney,
+Godwin was the obvious choice. He took the Texas bar exam,
+left Austin, moved to Cambridge, became a full-time, professional,
+computer civil libertarian, and was soon touring the nation on behalf
+of EFF, delivering well-received addresses on the issues to crowds
+as disparate as academics, industrialists, science fiction fans,
+and federal cops.
+
+Michael Godwin is currently the chief legal counsel of
+the Electronic Frontier Foundation in Cambridge, Massachusetts.
+
+#
+
+Another early and influential participant in the controversy
+was Dorothy Denning. Dr. Denning was unique among investigators
+of the computer underground in that she did not enter the debate
+with any set of politicized motives. She was a professional
+cryptographer and computer security expert whose primary interest
+in hackers was SCHOLARLY. She had a B.A. and M.A. in mathematics,
+and a Ph.D. in computer science from Purdue. She had worked for SRI
+International, the California think-tank that was also the home of
+computer-security maven Donn Parker, and had authored an influential text
+called Cryptography and Data Security. In 1990, Dr. Denning was working for
+Digital Equipment Corporation in their Systems Reseach Center. Her husband,
+Peter Denning, was also a computer security expert, working for NASA's
+Research Institute for Advanced Computer Science. He had edited the
+well-received Computers Under Attack: Intruders, Worms and Viruses.
+
+Dr. Denning took it upon herself to contact the digital underground,
+more or less with an anthropological interest. There she discovered
+that these computer-intruding hackers, who had been characterized
+as unethical, irresponsible, and a serious danger to society,
+did in fact have their own subculture and their own rules.
+They were not particularly well-considered rules, but they were,
+in fact, rules. Basically, they didn't take money and they
+didn't break anything.
+
+Her dispassionate reports on her researches did a great deal
+to influence serious-minded computer professionals--the sort
+of people who merely rolled their eyes at the cyberspace
+rhapsodies of a John Perry Barlow.
+
+For young hackers of the digital underground, meeting Dorothy Denning
+was a genuinely mind-boggling experience. Here was this neatly coiffed,
+conservatively dressed, dainty little personage, who reminded most
+hackers of their moms or their aunts. And yet she was an IBM systems
+programmer with profound expertise in computer architectures
+and high-security information flow, who had personal friends
+in the FBI and the National Security Agency.
+
+Dorothy Denning was a shining example of the American mathematical
+intelligentsia, a genuinely brilliant person from the central ranks
+of the computer-science elite. And here she was, gently questioning
+twenty-year-old hairy-eyed phone-phreaks over the deeper ethical
+implications of their behavior.
+
+Confronted by this genuinely nice lady, most hackers sat up very straight
+and did their best to keep the anarchy-file stuff down to a faint whiff
+of brimstone. Nevertheless, the hackers WERE in fact prepared to seriously
+discuss serious issues with Dorothy Denning. They were willing to speak
+the unspeakable and defend the indefensible, to blurt out their convictions
+that information cannot be owned, that the databases of governments and large
+corporations were a threat to the rights and privacy of individuals.
+
+Denning's articles made it clear to many that "hacking"
+was not simple vandalism by some evil clique of psychotics.
+"Hacking" was not an aberrant menace that could be charmed away
+by ignoring it, or swept out of existence by jailing a few ringleaders.
+Instead, "hacking" was symptomatic of a growing, primal struggle over
+knowledge and power in the age of information.
+
+Denning pointed out that the attitude of hackers were at least partially
+shared by forward-looking management theorists in the business community:
+people like Peter Drucker and Tom Peters. Peter Drucker, in his book
+The New Realities, had stated that "control of information by the government
+is no longer possible. Indeed, information is now transnational.
+Like money, it has no `fatherland.'"
+
+And management maven Tom Peters had chided large corporations for uptight,
+proprietary attitudes in his bestseller, Thriving on Chaos:
+"Information hoarding, especially by politically motivated,
+power-seeking staffs, had been commonplace throughout American industry,
+service and manufacturing alike. It will be an impossible
+millstone aroung the neck of tomorrow's organizations."
+
+Dorothy Denning had shattered the social membrane of the
+digital underground. She attended the Neidorf trial,
+where she was prepared to testify for the defense as an expert witness.
+She was a behind-the-scenes organizer of two of the most important
+national meetings of the computer civil libertarians. Though not
+a zealot of any description, she brought disparate elements of the
+electronic community into a surprising and fruitful collusion.
+
+Dorothy Denning is currently the Chair of the Computer Science Department
+at Georgetown University in Washington, DC.
+
+#
+
+There were many stellar figures in the civil libertarian community.
+There's no question, however, that its single most influential figure
+was Mitchell D. Kapor. Other people might have formal titles,
+or governmental positions, have more experience with crime,
+or with the law, or with the arcanities of computer security
+or constitutional theory. But by 1991 Kapor had transcended
+any such narrow role. Kapor had become "Mitch."
+
+Mitch had become the central civil-libertarian ad-hocrat.
+Mitch had stood up first, he had spoken out loudly, directly,
+vigorously and angrily, he had put his own reputation,
+and his very considerable personal fortune, on the line.
+By mid-'91 Kapor was the best-known advocate of his cause
+and was known PERSONALLY by almost every single human being in America
+with any direct influence on the question of civil liberties in cyberspace.
+Mitch had built bridges, crossed voids, changed paradigms, forged metaphors,
+made phone-calls and swapped business cards to such spectacular effect
+that it had become impossible for anyone to take any action in the
+"hacker question" without wondering what Mitch might think--
+and say--and tell his friends.
+
+The EFF had simply NETWORKED the situation into an entirely new status quo.
+And in fact this had been EFF's deliberate strategy from the beginning.
+Both Barlow and Kapor loathed bureaucracies and had deliberately
+chosen to work almost entirely through the electronic spiderweb of
+"valuable personal contacts."
+
+After a year of EFF, both Barlow and Kapor had every reason
+to look back with satisfaction. EFF had established its own Internet node,
+"eff.org," with a well-stocked electronic archive of documents on
+electronic civil rights, privacy issues, and academic freedom.
+EFF was also publishing EFFector, a quarterly printed journal,
+as well as EFFector Online, an electronic newsletter with
+over 1,200 subscribers. And EFF was thriving on the Well.
+
+EFF had a national headquarters in Cambridge and a full-time staff.
+It had become a membership organization and was attracting
+grass-roots support. It had also attracted the support
+of some thirty civil-rights lawyers, ready and eager
+to do pro bono work in defense of the Constitution in Cyberspace.
+
+EFF had lobbied successfully in Washington and in Massachusetts
+to change state and federal legislation on computer networking.
+Kapor in particular had become a veteran expert witness,
+and had joined the Computer Science and Telecommunications Board
+of the National Academy of Science and Engineering.
+
+EFF had sponsored meetings such as "Computers, Freedom and Privacy"
+and the CPSR Roundtable. It had carried out a press offensive that,
+in the words of EFFector, "has affected the climate of opinion about
+computer networking and begun to reverse the slide into
+`hacker hysteria' that was beginning to grip the nation."
+
+It had helped Craig Neidorf avoid prison.
+
+And, last but certainly not least, the Electronic Frontier Foundation
+had filed a federal lawsuit in the name of Steve Jackson,
+Steve Jackson Games Inc., and three users of the Illuminati
+bulletin board system. The defendants were, and are,
+the United States Secret Service, William Cook, Tim Foley,
+Barbara Golden and Henry Kleupfel.
+
+The case, which is in pre-trial procedures in an Austin federal court
+as of this writing, is a civil action for damages to redress
+alleged violations of the First and Fourth Amendments to the
+United States Constitution, as well as the Privacy Protection Act
+of 1980 (42 USC 2000aa et seq.), and the Electronic Communications
+Privacy Act (18 USC 2510 et seq and 2701 et seq).
+
+EFF had established that it had credibility. It had also established
+that it had teeth.
+
+In the fall of 1991 I travelled to Massachusetts to speak personally
+with Mitch Kapor. It was my final interview for this book.
+
+#
+
+The city of Boston has always been one of the major intellectual centers
+of the American republic. It is a very old city by American standards,
+a place of skyscrapers overshadowing seventeenth-century graveyards,
+where the high-tech start-up companies of Route 128 co-exist with the
+hand-wrought pre-industrial grace of "Old Ironsides," the USS CONSTITUTION.
+
+The Battle of Bunker Hill, one of the first and bitterest armed clashes
+of the American Revolution, was fought in Boston's environs. Today there is
+a monumental spire on Bunker Hill, visible throughout much of the city.
+The willingness of the republican revolutionaries to take up arms and fire
+on their oppressors has left a cultural legacy that two full centuries
+have not effaced. Bunker Hill is still a potent center of American political
+symbolism, and the Spirit of '76 is still a potent image for those who seek
+to mold public opinion.
+
+Of course, not everyone who wraps himself in the flag is necessarily
+a patriot. When I visited the spire in September 1991, it bore a huge,
+badly-erased, spray-can grafitto around its bottom reading
+"BRITS OUT--IRA PROVOS." Inside this hallowed edifice was
+a glass-cased diorama of thousands of tiny toy soldiers,
+rebels and redcoats, fighting and dying over the green hill,
+the riverside marshes, the rebel trenchworks. Plaques indicated the
+movement of troops, the shiftings of strategy. The Bunker Hill Monument
+is occupied at its very center by the toy soldiers of a military
+war-game simulation.
+
+The Boston metroplex is a place of great universities,
+prominent among the Massachusetts Institute of Technology,
+where the term "computer hacker" was first coined. The Hacker Crackdown
+of 1990 might be interpreted as a political struggle among American cities:
+traditional strongholds of longhair intellectual liberalism,
+such as Boston, San Francisco, and Austin, versus the bare-knuckle
+industrial pragmatism of Chicago and Phoenix (with Atlanta and New York
+wrapped in internal struggle).
+
+The headquarters of the Electronic Frontier Foundation is on
+155 Second Street in Cambridge, a Bostonian suburb north
+of the River Charles. Second Street has weedy sidewalks of dented,
+sagging brick and elderly cracked asphalt; large street-signs warn
+"NO PARKING DURING DECLARED SNOW EMERGENCY." This is an old area
+of modest manufacturing industries; the EFF is catecorner from the
+Greene Rubber Company. EFF's building is two stories of red brick;
+its large wooden windows feature gracefully arched tops and stone sills.
+
+The glass window beside the Second Street entrance bears three sheets
+of neatly laser-printed paper, taped against the glass. They read:
+ON Technology. EFF. KEI.
+
+"ON Technology" is Kapor's software company, which currently specializes
+in "groupware" for the Apple Macintosh computer. "Groupware" is intended
+to promote efficient social interaction among office-workers linked
+by computers. ON Technology's most successful software products to date
+are "Meeting Maker" and "Instant Update."
+
+"KEI" is Kapor Enterprises Inc., Kapor's personal holding company,
+the commercial entity that formally controls his extensive investments
+in other hardware and software corporations.
+
+"EFF" is a political action group--of a special sort.
+
+Inside, someone's bike has been chained to the handrails
+of a modest flight of stairs. A wall of modish glass brick
+separates this anteroom from the offices. Beyond the brick,
+there's an alarm system mounted on the wall, a sleek, complex little
+number that resembles a cross between a thermostat and a CD player.
+Piled against the wall are box after box of a recent special issue
+of Scientific American, "How to Work, Play, and Thrive in Cyberspace,"
+with extensive coverage of electronic networking techniques
+and political issues, including an article by Kapor himself.
+These boxes are addressed to Gerard Van der Leun, EFF's
+Director of Communications, who will shortly mail those magazines
+to every member of the EFF.
+
+The joint headquarters of EFF, KEI, and ON Technology,
+which Kapor currently rents, is a modestly bustling place.
+It's very much the same physical size as Steve Jackson's gaming company.
+It's certainly a far cry from the gigantic gray steel-sided railway
+shipping barn, on the Monsignor O'Brien Highway, that is owned
+by Lotus Development Corporation.
+
+Lotus is, of course, the software giant that Mitchell Kapor founded
+in the late 70s. The software program Kapor co-authored,
+"Lotus 1-2-3," is still that company's most profitable product.
+"Lotus 1-2-3" also bears a singular distinction in the
+digital underground: it's probably the most pirated piece
+of application software in world history.
+
+Kapor greets me cordially in his own office, down a hall.
+Kapor, whose name is pronounced KAY-por, is in his early forties,
+married and the father of two. He has a round face, high forehead,
+straight nose, a slightly tousled mop of black hair peppered with gray.
+His large brown eyes are wideset, reflective, one might almost say soulful.
+He disdains ties, and commonly wears Hawaiian shirts and tropical prints,
+not so much garish as simply cheerful and just that little bit anomalous.
+
+There is just the whiff of hacker brimstone about Mitch Kapor.
+He may not have the hard-riding, hell-for-leather, guitar-strumming
+charisma of his Wyoming colleague John Perry Barlow, but there's
+something about the guy that still stops one short. He has the air
+of the Eastern city dude in the bowler hat, the dreamy,
+Longfellow-quoting poker shark who only HAPPENS to know
+the exact mathematical odds against drawing to an inside straight.
+Even among his computer-community colleagues, who are hardly known
+for mental sluggishness, Kapor strikes one forcefully as a very
+intelligent man. He speaks rapidly, with vigorous gestures,
+his Boston accent sometimes slipping to the sharp nasal tang
+of his youth in Long Island.
+
+Kapor, whose Kapor Family Foundation does much of his philanthropic work,
+is a strong supporter of Boston's Computer Museum. Kapor's interest
+in the history of his industry has brought him some remarkable curios,
+such as the "byte" just outside his office door. This "byte"--
+eight digital bits--has been salvaged from the wreck of an
+electronic computer of the pre-transistor age. It's a standing gunmetal
+rack about the size of a small toaster-oven: with eight slots
+of hand-soldered breadboarding featuring thumb-sized vacuum tubes.
+If it fell off a table it could easily break your foot,
+but it was state-of-the-art computation in the 1940s.
+(It would take exactly 157,184 of these primordial toasters
+to hold the first part of this book.)
+
+There's also a coiling, multicolored, scaly dragon that some
+inspired techno-punk artist has cobbled up entirely out of transistors,
+capacitors, and brightly plastic-coated wiring.
+
+Inside the office, Kapor excuses himself briefly to do a little
+mouse-whizzing housekeeping on his personal Macintosh IIfx.
+If its giant screen were an open window, an agile person
+could climb through it without much trouble at all.
+There's a coffee-cup at Kapor's elbow, a memento of his
+recent trip to Eastern Europe, which has a black-and-white
+stencilled photo and the legend CAPITALIST FOOLS TOUR.
+It's Kapor, Barlow, and two California venture-capitalist luminaries
+of their acquaintance, four windblown, grinning Baby Boomer
+dudes in leather jackets, boots, denim, travel bags,
+standing on airport tarmac somewhere behind the formerly Iron Curtain.
+They look as if they're having the absolute time of their lives.
+
+Kapor is in a reminiscent mood. We talk a bit about his youth--
+high school days as a "math nerd," Saturdays attending Columbia University's
+high-school science honors program, where he had his first experience
+programming computers. IBM 1620s, in 1965 and '66. "I was very interested,"
+says Kapor, "and then I went off to college and got distracted by drugs sex
+and rock and roll, like anybody with half a brain would have then!"
+After college he was a progressive-rock DJ in Hartford, Connecticut,
+for a couple of years.
+
+I ask him if he ever misses his rock and roll days--if he ever wished
+he could go back to radio work.
+
+He shakes his head flatly. "I stopped thinking about going back
+to be a DJ the day after Altamont."
+
+Kapor moved to Boston in 1974 and got a job programming mainframes in COBOL.
+He hated it. He quit and became a teacher of transcendental meditation.
+(It was Kapor's long flirtation with Eastern mysticism that gave the
+world "Lotus.")
+
+In 1976 Kapor went to Switzerland, where the Transcendental Meditation
+movement had rented a gigantic Victorian hotel in St-Moritz. It was
+an all-male group--a hundred and twenty of them--determined upon
+Enlightenment or Bust. Kapor had given the transcendant his best shot.
+He was becoming disenchanted by "the nuttiness in the organization."
+"They were teaching people to levitate," he says, staring at the floor.
+His voice drops an octave, becomes flat. "THEY DON'T LEVITATE."
+
+Kapor chose Bust. He went back to the States and acquired a degree
+in counselling psychology. He worked a while in a hospital,
+couldn't stand that either. "My rep was," he says "a very bright kid
+with a lot of potential who hasn't found himself. Almost thirty.
+Sort of lost."
+
+Kapor was unemployed when he bought his first personal computer--an Apple II.
+He sold his stereo to raise cash and drove to New Hampshire to avoid the
+sales tax.
+
+"The day after I purchased it," Kapor tells me, "I was hanging out
+in a computer store and I saw another guy, a man in his forties,
+well-dressed guy, and eavesdropped on his conversation with the salesman.
+He didn't know anything about computers. I'd had a year programming.
+And I could program in BASIC. I'd taught myself. So I went up to him,
+and I actually sold myself to him as a consultant." He pauses.
+"I don't know where I got the nerve to do this. It was uncharacteristic.
+I just said, `I think I can help you, I've been listening,
+this is what you need to do and I think I can do it for you.'
+And he took me on! He was my first client! I became a computer
+consultant the first day after I bought the Apple II."
+
+Kapor had found his true vocation. He attracted more clients
+for his consultant service, and started an Apple users' group.
+
+A friend of Kapor's, Eric Rosenfeld, a graduate student at MIT,
+had a problem. He was doing a thesis on an arcane form of
+financial statistics, but could not wedge himself into the crowded queue
+for time on MIT's mainframes. (One might note at this point that if
+Mr. Rosenfeld had dishonestly broken into the MIT mainframes,
+Kapor himself might have never invented Lotus 1-2-3 and
+the PC business might have been set back for years!)
+Eric Rosenfeld did have an Apple II, however,
+and he thought it might be possible to scale the problem down.
+Kapor, as favor, wrote a program for him in BASIC that did the job.
+
+It then occurred to the two of them, out of the blue,
+that it might be possible to SELL this program.
+They marketed it themselves, in plastic baggies,
+for about a hundred bucks a pop, mail order.
+"This was a total cottage industry by a marginal consultant,"
+Kapor says proudly. "That's how I got started, honest to God."
+
+Rosenfeld, who later became a very prominent figure on Wall Street,
+urged Kapor to go to MIT's business school for an MBA.
+Kapor did seven months there, but never got his MBA.
+He picked up some useful tools--mainly a firm grasp
+of the principles of accounting--and, in his own words,
+"learned to talk MBA." Then he dropped out and went to Silicon Valley.
+
+The inventors of VisiCalc, the Apple computer's premier business program,
+had shown an interest in Mitch Kapor. Kapor worked diligently for them
+for six months, got tired of California, and went back to Boston
+where they had better bookstores. The VisiCalc group had made
+the critical error of bringing in "professional management."
+"That drove them into the ground," Kapor says.
+
+"Yeah, you don't hear a lot about VisiCalc these days," I muse.
+
+Kapor looks surprised. "Well, Lotus. . . we BOUGHT it."
+
+"Oh. You BOUGHT it?"
+
+"Yeah."
+
+"Sort of like the Bell System buying Western Union?"
+
+Kapor grins. "Yep! Yep! Yeah, exactly!"
+
+Mitch Kapor was not in full command of the destiny of himself
+or his industry. The hottest software commodities of the early 1980s
+were COMPUTER GAMES--the Atari seemed destined to enter every teenage home
+in America. Kapor got into business software simply because he didn't have
+any particular feeling for computer games. But he was supremely fast
+on his feet, open to new ideas and inclined to trust his instincts.
+And his instincts were good. He chose good people to deal with--
+gifted programmer Jonathan Sachs (the co-author of Lotus 1-2-3).
+Financial wizard Eric Rosenfeld, canny Wall Street analyst
+and venture capitalist Ben Rosen. Kapor was the founder and CEO of Lotus,
+one of the most spectacularly successful business ventures of the
+later twentieth century.
+
+He is now an extremely wealthy man. I ask him if he actually
+knows how much money he has.
+
+"Yeah," he says. "Within a percent or two."
+
+How much does he actually have, then?
+
+He shakes his head. "A lot. A lot. Not something I talk about.
+Issues of money and class are things that cut pretty close to the bone."
+
+I don't pry. It's beside the point. One might presume, impolitely,
+that Kapor has at least forty million--that's what he got the year
+he left Lotus. People who ought to know claim Kapor has about
+a hundred and fifty million, give or take a market swing
+in his stock holdings. If Kapor had stuck with Lotus,
+as his colleague friend and rival Bill Gates has stuck
+with his own software start-up, Microsoft, then Kapor
+would likely have much the same fortune Gates has--
+somewhere in the neighborhood of three billion,
+give or take a few hundred million. Mitch Kapor
+has all the money he wants. Money has lost whatever charm
+it ever held for him--probably not much in the first place.
+When Lotus became too uptight, too bureaucratic, too far
+from the true sources of his own satisfaction, Kapor walked.
+He simply severed all connections with the company and went out the door.
+It stunned everyone--except those who knew him best.
+
+Kapor has not had to strain his resources to wreak a thorough
+transformation in cyberspace politics. In its first year,
+EFF's budget was about a quarter of a million dollars.
+Kapor is running EFF out of his pocket change.
+
+Kapor takes pains to tell me that he does not consider himself
+a civil libertarian per se. He has spent quite some time
+with true-blue civil libertarians lately, and there's a
+political-correctness to them that bugs him. They seem
+to him to spend entirely too much time in legal nitpicking
+and not enough vigorously exercising civil rights in the
+everyday real world.
+
+Kapor is an entrepreneur. Like all hackers, he prefers his involvements
+direct, personal, and hands-on. "The fact that EFF has a node on the
+Internet is a great thing. We're a publisher. We're a distributor
+of information." Among the items the eff.org Internet node carries
+is back issues of Phrack. They had an internal debate about that in EFF,
+and finally decided to take the plunge. They might carry other
+digital underground publications--but if they do, he says,
+"we'll certainly carry Donn Parker, and anything Gail Thackeray
+wants to put up. We'll turn it into a public library, that has
+the whole spectrum of use. Evolve in the direction of people making up
+their own minds." He grins. "We'll try to label all the editorials."
+
+Kapor is determined to tackle the technicalities of the Internet
+in the service of the public interest. "The problem with being a node
+on the Net today is that you've got to have a captive technical specialist.
+We have Chris Davis around, for the care and feeding of the balky beast!
+We couldn't do it ourselves!"
+
+He pauses. "So one direction in which technology has to evolve
+is much more standardized units, that a non-technical person
+can feel comfortable with. It's the same shift as from minicomputers to PCs.
+I can see a future in which any person can have a Node on the Net.
+Any person can be a publisher. It's better than the media we now have.
+It's possible. We're working actively."
+
+Kapor is in his element now, fluent, thoroughly in command in his material.
+"You go tell a hardware Internet hacker that everyone should have a node
+on the Net," he says, "and the first thing they're going to say is,
+`IP doesn't scale!'" ("IP" is the interface protocol for the Internet.
+As it currently exists, the IP software is simply not capable of
+indefinite expansion; it will run out of usable addresses, it will saturate.)
+"The answer," Kapor says, "is: evolve the protocol! Get the smart people
+together and figure out what to do. Do we add ID? Do we add new protocol?
+Don't just say, WE CAN'T DO IT."
+
+Getting smart people together to figure out what to do is a skill
+at which Kapor clearly excels. I counter that people on the Internet
+rather enjoy their elite technical status, and don't seem particularly
+anxious to democratize the Net.
+
+Kapor agrees, with a show of scorn. "I tell them that this is the snobbery
+of the people on the Mayflower looking down their noses at the people
+who came over ON THE SECOND BOAT! Just because they got here a year,
+or five years, or ten years before everybody else, that doesn't give
+them ownership of cyberspace! By what right?"
+
+I remark that the telcos are an electronic network, too,
+and they seem to guard their specialized knowledge pretty closely.
+
+Kapor ripostes that the telcos and the Internet are entirely
+different animals. "The Internet is an open system,
+everything is published, everything gets argued about,
+basically by anybody who can get in. Mostly, it's exclusive
+and elitist just because it's so difficult. Let's make it easier to use."
+
+On the other hand, he allows with a swift change of emphasis,
+the so-called elitists do have a point as well. "Before people start coming in,
+who are new, who want to make suggestions, and criticize the Net as
+`all screwed up'. . . . They should at least take the time to understand
+the culture on its own terms. It has its own history--show some respect
+for it. I'm a conservative, to that extent."
+
+The Internet is Kapor's paradigm for the future of telecommunications.
+The Internet is decentralized, non-hierarchical, almost anarchic.
+There are no bosses, no chain of command, no secret data.
+If each node obeys the general interface standards,
+there's simply no need for any central network authority.
+
+Wouldn't that spell the doom of AT&T as an institution? I ask.
+
+That prospect doesn't faze Kapor for a moment. "Their big advantage,
+that they have now, is that they have all of the wiring.
+But two things are happening. Anyone with right-of-way
+is putting down fiber--Southern Pacific Railroad,
+people like that--there's enormous `dark fiber' laid in."
+("Dark Fiber" is fiber-optic cable, whose enormous capacity
+so exceeds the demands of current usage that much of the
+fiber still has no light-signals on it--it's still `dark,'
+awaiting future use.)
+
+"The other thing that's happening is the local-loop stuff
+is going to go wireless. Everyone from Bellcore to the cable TV
+companies to AT&T wants to put in these things called
+`personal communication systems.' So you could have local competition--
+you could have multiplicity of people, a bunch of neighborhoods,
+sticking stuff up on poles. And a bunch of other people laying in dark fiber.
+So what happens to the telephone companies? There's enormous pressure
+on them from both sides.
+
+"The more I look at this, the more I believe that in a post-industrial,
+digital world, the idea of regulated monopolies is bad. People will
+look back on it and say that in the 19th and 20th centuries
+the idea of public utilities was an okay compromise.
+You needed one set of wires in the ground. It was too economically
+inefficient, otherwise. And that meant one entity running it.
+But now, with pieces being wireless--the connections are going
+to be via high-level interfaces, not via wires. I mean, ULTIMATELY
+there are going to be wires--but the wires are just a commodity.
+Fiber, wireless. You no longer NEED a utility."
+
+Water utilities? Gas utilities?
+
+Of course we still need those, he agrees. "But when what you're moving
+is information, instead of physical substances, then you can play by
+a different set of rules. We're evolving those rules now!
+Hopefully you can have a much more decentralized system,
+and one in which there's more competition in the marketplace.
+
+"The role of government will be to make sure that nobody cheats.
+The proverbial `level playing field.' A policy that prevents monopolization.
+It should result in better service, lower prices, more choices,
+and local empowerment." He smiles. "I'm very big on local empowerment."
+
+Kapor is a man with a vision. It's a very novel vision which he
+and his allies are working out in considerable detail and with great energy.
+Dark, cynical, morbid cyberpunk that I am, I cannot avoid considering
+some of the darker implications of "decentralized, nonhierarchical,
+locally empowered" networking.
+
+I remark that some pundits have suggested that electronic networking--faxes,
+phones, small-scale photocopiers--played a strong role in dissolving
+the power of centralized communism and causing the collapse of the Warsaw Pact.
+
+Socialism is totally discredited, says Kapor, fresh back from
+the Eastern Bloc. The idea that faxes did it, all by themselves,
+is rather wishful thinking.
+
+Has it occurred to him that electronic networking might corrode
+America's industrial and political infrastructure to the point
+where the whole thing becomes untenable, unworkable--and the old order
+just collapses headlong, like in Eastern Europe?
+
+"No," Kapor says flatly. "I think that's extraordinarily unlikely.
+In part, because ten or fifteen years ago, I had similar hopes
+about personal computers--which utterly failed to materialize."
+He grins wryly, then his eyes narrow. "I'm VERY opposed to techno-utopias.
+Every time I see one, I either run away, or try to kill it."
+
+It dawns on me then that Mitch Kapor is not trying to
+make the world safe for democracy. He certainly is not
+trying to make it safe for anarchists or utopians--
+least of all for computer intruders or electronic rip-off artists.
+What he really hopes to do is make the world safe for
+future Mitch Kapors. This world of decentralized, small-scale nodes,
+with instant global access for the best and brightest,
+would be a perfect milieu for the shoestring attic capitalism
+that made Mitch Kapor what he is today.
+
+Kapor is a very bright man. He has a rare combination
+of visionary intensity with a strong practical streak.
+The Board of the EFF: John Barlow, Jerry Berman of the ACLU,
+Stewart Brand, John Gilmore, Steve Wozniak, and Esther Dyson,
+the doyenne of East-West computer entrepreneurism--share his gift,
+his vision, and his formidable networking talents.
+They are people of the 1960s, winnowed-out by its turbulence
+and rewarded with wealth and influence. They are some of the best
+and the brightest that the electronic community has to offer.
+But can they do it, in the real world? Or are they only dreaming?
+They are so few. And there is so much against them.
+
+I leave Kapor and his networking employees struggling cheerfully
+with the promising intricacies of their newly installed Macintosh
+System 7 software. The next day is Saturday. EFF is closed.
+I pay a few visits to points of interest downtown.
+
+One of them is the birthplace of the telephone.
+
+It's marked by a bronze plaque in a plinth of black-and-white speckled granite. It sits in the
+plaza of the John F. Kennedy Federal Building, the very place where Kapor was
+once fingerprinted by the FBI.
+
+The plaque has a bas-relief picture of Bell's original telephone.
+"BIRTHPLACE OF THE TELEPHONE," it reads. "Here, on June 2, 1875,
+Alexander Graham Bell and Thomas A. Watson first transmitted sound over wires.
+
+"This successful experiment was completed in a fifth floor garret
+at what was then 109 Court Street and marked the beginning of
+world-wide telephone service."
+
+109 Court Street is long gone. Within sight of Bell's plaque,
+across a street, is one of the central offices of NYNEX,
+the local Bell RBOC, on 6 Bowdoin Square.
+
+I cross the street and circle the telco building, slowly,
+hands in my jacket pockets. It's a bright, windy, New England
+autumn day. The central office is a handsome 1940s-era megalith
+in late Art Deco, eight stories high.
+
+Parked outside the back is a power-generation truck.
+The generator strikes me as rather anomalous. Don't they
+already have their own generators in this eight-story monster?
+Then the suspicion strikes me that NYNEX must have heard
+of the September 17 AT&T power-outage which crashed New York City.
+Belt-and-suspenders, this generator. Very telco.
+
+Over the glass doors of the front entrance is a handsome bronze
+bas-relief of Art Deco vines, sunflowers, and birds, entwining
+the Bell logo and the legend NEW ENGLAND TELEPHONE AND TELEGRAPH COMPANY
+--an entity which no longer officially exists.
+
+The doors are locked securely. I peer through the shadowed glass.
+Inside is an official poster reading:
+
+
+"New England Telephone a NYNEX Company
+
+ATTENTION
+
+"All persons while on New England Telephone
+Company premises are required to visibly wear their
+identification cards (C.C.P. Section 2, Page 1).
+
+"Visitors, vendors, contractors, and all others are
+required to visibly wear a daily pass.
+
+"Thank you.
+
+Kevin C. Stanton.
+Building Security Coordinator."
+
+
+Outside, around the corner, is a pull-down ribbed metal security door,
+a locked delivery entrance. Some passing stranger has grafitti-tagged
+this door, with a single word in red spray-painted cursive:
+
+Fury
+
+#
+
+My book on the Hacker Crackdown is almost over now.
+I have deliberately saved the best for last.
+
+In February 1991, I attended the CPSR Public Policy Roundtable,
+in Washington, DC. CPSR, Computer Professionals for Social Responsibility,
+was a sister organization of EFF, or perhaps its aunt, being older
+and perhaps somewhat wiser in the ways of the world of politics.
+
+Computer Professionals for Social Responsibility began in 1981
+in Palo Alto, as an informal discussion group of Californian
+computer scientists and technicians, united by nothing more
+than an electronic mailing list. This typical high-tech
+ad-hocracy received the dignity of its own acronym in 1982,
+and was formally incorporated in 1983.
+
+CPSR lobbied government and public alike with an educational
+outreach effort, sternly warning against any foolish
+and unthinking trust in complex computer systems.
+CPSR insisted that mere computers should never be
+considered a magic panacea for humanity's social,
+ethical or political problems. CPSR members were especially
+troubled about the stability, safety, and dependability
+of military computer systems, and very especially troubled
+by those systems controlling nuclear arsenals. CPSR was
+best-known for its persistent and well-publicized attacks on the
+scientific credibility of the Strategic Defense Initiative ("Star Wars").
+
+In 1990, CPSR was the nation's veteran cyber-political activist group,
+with over two thousand members in twenty- one local chapters across the US.
+It was especially active in Boston, Silicon Valley, and Washington DC,
+where its Washington office sponsored the Public Policy Roundtable.
+
+The Roundtable, however, had been funded by EFF, which had passed CPSR
+an extensive grant for operations. This was the first large-scale,
+official meeting of what was to become the electronic civil
+libertarian community.
+
+Sixty people attended, myself included--in this instance, not so much
+as a journalist as a cyberpunk author. Many of the luminaries
+of the field took part: Kapor and Godwin as a matter of course.
+Richard Civille and Marc Rotenberg of CPSR. Jerry Berman of the ACLU.
+John Quarterman, author of The Matrix. Steven Levy, author of Hackers.
+George Perry and Sandy Weiss of Prodigy Services, there to network
+about the civil-liberties troubles their young commercial
+network was experiencing. Dr. Dorothy Denning. Cliff Figallo,
+manager of the Well. Steve Jackson was there, having finally
+found his ideal target audience, and so was Craig Neidorf,
+"Knight Lightning" himself, with his attorney, Sheldon Zenner.
+Katie Hafner, science journalist, and co-author of Cyberpunk:
+Outlaws and Hackers on the Computer Frontier. Dave Farber,
+ARPAnet pioneer and fabled Internet guru. Janlori Goldman
+of the ACLU's Project on Privacy and Technology. John Nagle
+of Autodesk and the Well. Don Goldberg of the House Judiciary Committee.
+Tom Guidoboni, the defense attorney in the Internet Worm case.
+Lance Hoffman, computer-science professor at The George Washington
+University. Eli Noam of Columbia. And a host of others no less distinguished.
+
+Senator Patrick Leahy delivered the keynote address,
+expressing his determination to keep ahead of the curve
+on the issue of electronic free speech. The address was
+well-received, and the sense of excitement was palpable.
+Every panel discussion was interesting--some were entirely
+compelling. People networked with an almost frantic interest.
+
+I myself had a most interesting and cordial lunch discussion with
+Noel and Jeanne Gayler, Admiral Gayler being a former director
+of the National Security Agency. As this was the first known encounter
+between an actual no-kidding cyberpunk and a chief executive of
+America's largest and best-financed electronic espionage apparat,
+there was naturally a bit of eyebrow-raising on both sides.
+
+Unfortunately, our discussion was off-the-record. In fact
+all the discussions at the CPSR were officially off-the-record,
+the idea being to do some serious networking in an atmosphere
+of complete frankness, rather than to stage a media circus.
+
+In any case, CPSR Roundtable, though interesting and intensely valuable,
+was as nothing compared to the truly mind-boggling event that transpired
+a mere month later.
+
+#
+
+"Computers, Freedom and Privacy." Four hundred people from
+every conceivable corner of America's electronic community.
+As a science fiction writer, I have been to some weird gigs in my day,
+but this thing is truly BEYOND THE PALE. Even "Cyberthon,"
+Point Foundation's "Woodstock of Cyberspace" where Bay Area
+psychedelia collided headlong with the emergent world
+of computerized virtual reality, was like a Kiwanis Club gig
+compared to this astonishing do.
+
+The "electronic community" had reached an apogee.
+Almost every principal in this book is in attendance.
+Civil Libertarians. Computer Cops. The Digital Underground.
+Even a few discreet telco people. Colorcoded dots
+for lapel tags are distributed. Free Expression issues.
+Law Enforcement. Computer Security. Privacy. Journalists.
+Lawyers. Educators. Librarians. Programmers.
+Stylish punk-black dots for the hackers and phone phreaks.
+Almost everyone here seems to wear eight or nine dots,
+to have six or seven professional hats.
+
+It is a community. Something like Lebanon perhaps,
+but a digital nation. People who had feuded all year
+in the national press, people who entertained the deepest
+suspicions of one another's motives and ethics, are now
+in each others' laps. "Computers, Freedom and Privacy"
+had every reason in the world to turn ugly, and yet except
+for small irruptions of puzzling nonsense from the
+convention's token lunatic, a surprising bonhomie reigned.
+CFP was like a wedding-party in which two lovers,
+unstable bride and charlatan groom, tie the knot
+in a clearly disastrous matrimony.
+
+It is clear to both families--even to neighbors and random guests--
+that this is not a workable relationship, and yet the young couple's
+desperate attraction can brook no further delay. They simply cannot
+help themselves. Crockery will fly, shrieks from their newlywed home
+will wake the city block, divorce waits in the wings like a vulture
+over the Kalahari, and yet this is a wedding, and there is going
+to be a child from it. Tragedies end in death; comedies in marriage.
+The Hacker Crackdown is ending in marriage. And there will be a child.
+
+From the beginning, anomalies reign. John Perry Barlow,
+cyberspace ranger, is here. His color photo in
+The New York Times Magazine, Barlow scowling
+in a grim Wyoming snowscape, with long black coat,
+dark hat, a Macintosh SE30 propped on a fencepost
+and an awesome frontier rifle tucked under one arm,
+will be the single most striking visual image
+of the Hacker Crackdown. And he is CFP's guest of honor--
+along with Gail Thackeray of the FCIC! What on earth do
+they expect these dual guests to do with each other? Waltz?
+
+Barlow delivers the first address. Uncharacteristically,
+he is hoarse--the sheer volume of roadwork has worn him down.
+He speaks briefly, congenially, in a plea for conciliation,
+and takes his leave to a storm of applause.
+
+Then Gail Thackeray takes the stage. She's visibly nervous.
+She's been on the Well a lot lately. Reading those Barlow posts.
+Following Barlow is a challenge to anyone. In honor of the famous
+lyricist for the Grateful Dead, she announces reedily, she is going to read--
+A POEM. A poem she has composed herself.
+
+It's an awful poem, doggerel in the rollicking meter of Robert W. Service's
+The Cremation of Sam McGee, but it is in fact, a poem. It's the Ballad
+of the Electronic Frontier! A poem about the Hacker Crackdown and the
+sheer unlikelihood of CFP. It's full of in-jokes. The score or so cops
+in the audience, who are sitting together in a nervous claque,
+are absolutely cracking-up. Gail's poem is the funniest goddamn thing
+they've ever heard. The hackers and civil-libs, who had this woman figured
+for Ilsa She-Wolf of the SS, are staring with their jaws hanging loosely.
+Never in the wildest reaches of their imagination had they figured
+Gail Thackeray was capable of such a totally off-the-wall move.
+You can see them punching their mental CONTROL-RESET buttons.
+Jesus! This woman's a hacker weirdo! She's JUST LIKE US!
+God, this changes everything!
+
+Al Bayse, computer technician for the FBI, had been the only cop
+at the CPSR Roundtable, dragged there with his arm bent by
+Dorothy Denning. He was guarded and tightlipped at CPSR Roundtable;
+a "lion thrown to the Christians."
+
+At CFP, backed by a claque of cops, Bayse suddenly waxes eloquent
+and even droll, describing the FBI's "NCIC 2000", a gigantic digital catalog
+of criminal records, as if he has suddenly become some weird hybrid
+of George Orwell and George Gobel. Tentatively, he makes an arcane
+joke about statistical analysis. At least a third of the crowd laughs aloud.
+
+"They didn't laugh at that at my last speech," Bayse observes.
+He had been addressing cops--STRAIGHT cops, not computer people.
+It had been a worthy meeting, useful one supposes, but nothing like THIS.
+There has never been ANYTHING like this. Without any prodding,
+without any preparation, people in the audience simply begin to ask questions.
+Longhairs, freaky people, mathematicians. Bayse is answering, politely,
+frankly, fully, like a man walking on air. The ballroom's atmosphere
+crackles with surreality. A female lawyer behind me breaks into a sweat
+and a hot waft of surprisingly potent and musky perfume flows off
+her pulse-points.
+
+People are giddy with laughter. People are interested,
+fascinated, their eyes so wide and dark that they seem eroticized.
+Unlikely daisy-chains form in the halls, around the bar, on the escalators:
+cops with hackers, civil rights with FBI, Secret Service with phone phreaks.
+
+Gail Thackeray is at her crispest in a white wool sweater with a
+tiny Secret Service logo. "I found Phiber Optik at the payphones,
+and when he saw my sweater, he turned into a PILLAR OF SALT!" she chortles.
+
+Phiber discusses his case at much length with his arresting officer,
+Don Delaney of the New York State Police. After an hour's chat,
+the two of them look ready to begin singing "Auld Lang Syne."
+Phiber finally finds the courage to get his worst complaint off his chest.
+It isn't so much the arrest. It was the CHARGE. Pirating service
+off 900 numbers. I'm a PROGRAMMER, Phiber insists. This lame charge
+is going to hurt my reputation. It would have been cool to be busted
+for something happening, like Section 1030 computer intrusion.
+Maybe some kind of crime that's scarcely been invented yet.
+Not lousy phone fraud. Phooey.
+
+Delaney seems regretful. He had a mountain of possible criminal charges
+against Phiber Optik. The kid's gonna plead guilty anyway. He's a
+first timer, they always plead. Coulda charged the kid with most anything,
+and gotten the same result in the end. Delaney seems genuinely sorry
+not to have gratified Phiber in this harmless fashion. Too late now.
+Phiber's pled already. All water under the bridge. Whaddya gonna do?
+
+Delaney's got a good grasp on the hacker mentality.
+He held a press conference after he busted a bunch of
+Masters of Deception kids. Some journo had asked him:
+"Would you describe these people as GENIUSES?"
+Delaney's deadpan answer, perfect: "No, I would describe
+these people as DEFENDANTS." Delaney busts a kid for
+hacking codes with repeated random dialling. Tells the
+press that NYNEX can track this stuff in no time flat nowadays,
+and a kid has to be STUPID to do something so easy to catch.
+Dead on again: hackers don't mind being thought of as Genghis Khan
+by the straights, but if there's anything that really gets 'em
+where they live, it's being called DUMB.
+
+Won't be as much fun for Phiber next time around.
+As a second offender he's gonna see prison.
+Hackers break the law. They're not geniuses, either.
+They're gonna be defendants. And yet, Delaney muses over
+a drink in the hotel bar, he has found it impossible to treat
+them as common criminals. Delaney knows criminals. These kids,
+by comparison, are clueless--there is just no crook vibe off of them,
+they don't smell right, they're just not BAD.
+
+Delaney has seen a lot of action. He did Vietnam.
+He's been shot at, he has shot people. He's a homicide
+cop from New York. He has the appearance of a man who
+has not only seen the shit hit the fan but has seen it splattered
+across whole city blocks and left to ferment for years.
+This guy has been around.
+
+He listens to Steve Jackson tell his story. The dreamy
+game strategist has been dealt a bad hand. He has played
+it for all he is worth. Under his nerdish SF-fan exterior
+is a core of iron. Friends of his say Steve Jackson believes
+in the rules, believes in fair play. He will never compromise
+his principles, never give up. "Steve," Delaney says to
+Steve Jackson, "they had some balls, whoever busted you.
+You're all right!" Jackson, stunned, falls silent and
+actually blushes with pleasure.
+
+Neidorf has grown up a lot in the past year. The kid is
+a quick study, you gotta give him that. Dressed by his mom,
+the fashion manager for a national clothing chain,
+Missouri college techie-frat Craig Neidorf out-dappers
+everyone at this gig but the toniest East Coast lawyers.
+The iron jaws of prison clanged shut without him and now
+law school beckons for Neidorf. He looks like a larval Congressman.
+
+Not a "hacker," our Mr. Neidorf. He's not interested
+in computer science. Why should he be? He's not
+interested in writing C code the rest of his life,
+and besides, he's seen where the chips fall.
+To the world of computer science he and Phrack
+were just a curiosity. But to the world of law. . . .
+The kid has learned where the bodies are buried.
+He carries his notebook of press clippings wherever he goes.
+
+Phiber Optik makes fun of Neidorf for a Midwestern geek,
+for believing that "Acid Phreak" does acid and listens to acid rock.
+Hell no. Acid's never done ACID! Acid's into ACID HOUSE MUSIC.
+Jesus. The very idea of doing LSD. Our PARENTS did LSD, ya clown.
+
+Thackeray suddenly turns upon Craig Neidorf the full lighthouse
+glare of her attention and begins a determined half-hour attempt
+to WIN THE BOY OVER. The Joan of Arc of Computer Crime is
+GIVING CAREER ADVICE TO KNIGHT LIGHTNING! "Your experience
+would be very valuable--a real asset," she tells him with
+unmistakeable sixty-thousand-watt sincerity. Neidorf is fascinated.
+He listens with unfeigned attention. He's nodding and saying yes ma'am.
+Yes, Craig, you too can forget all about money and enter the glamorous
+and horribly underpaid world of PROSECUTING COMPUTER CRIME!
+You can put your former friends in prison--ooops. . . .
+
+You cannot go on dueling at modem's length indefinitely.
+You cannot beat one another senseless with rolled-up press-clippings.
+Sooner or later you have to come directly to grips.
+And yet the very act of assembling here has changed
+the entire situation drastically. John Quarterman,
+author of The Matrix, explains the Internet at his symposium.
+It is the largest news network in the world, it is growing
+by leaps and bounds, and yet you cannot measure Internet because
+you cannot stop it in place. It cannot stop, because there
+is no one anywhere in the world with the authority to stop Internet.
+It changes, yes, it grows, it embeds itself across the post-industrial,
+postmodern world and it generates community wherever it
+touches, and it is doing this all by itself.
+
+Phiber is different. A very fin de siecle kid, Phiber Optik.
+Barlow says he looks like an Edwardian dandy. He does rather.
+Shaven neck, the sides of his skull cropped hip-hop close,
+unruly tangle of black hair on top that looks pomaded,
+he stays up till four a.m. and misses all the sessions,
+then hangs out in payphone booths with his acoustic coupler
+gutsily CRACKING SYSTEMS RIGHT IN THE MIDST OF THE HEAVIEST
+LAW ENFORCEMENT DUDES IN THE U.S., or at least PRETENDING to. . . .
+Unlike "Frank Drake." Drake, who wrote Dorothy Denning out
+of nowhere, and asked for an interview for his cheapo
+cyberpunk fanzine, and then started grilling her on her ethics.
+She was squirmin', too. . . . Drake, scarecrow-tall with his
+floppy blond mohawk, rotting tennis shoes and black leather jacket
+lettered ILLUMINATI in red, gives off an unmistakeable air
+of the bohemian literatus. Drake is the kind of guy
+who reads British industrial design magazines and appreciates
+William Gibson because the quality of the prose is so tasty.
+Drake could never touch a phone or a keyboard again,
+and he'd still have the nose-ring and the blurry photocopied
+fanzines and the sampled industrial music. He's a radical punk
+with a desktop-publishing rig and an Internet address.
+Standing next to Drake, the diminutive Phiber looks like he's
+been physically coagulated out of phone-lines. Born to phreak.
+
+Dorothy Denning approaches Phiber suddenly. The two of them
+are about the same height and body-build. Denning's blue eyes
+flash behind the round window-frames of her glasses.
+"Why did you say I was `quaint?'" she asks Phiber, quaintly.
+
+It's a perfect description but Phiber is nonplussed. . .
+"Well, I uh, you know. . . ."
+
+"I also think you're quaint, Dorothy," I say, novelist to the rescue,
+the journo gift of gab. . . . She is neat and dapper and yet there's
+an arcane quality to her, something like a Pilgrim Maiden behind
+leaded glass; if she were six inches high Dorothy Denning would look
+great inside a china cabinet. . .The Cryptographeress. . .
+The Cryptographrix. . .whatever. . . . Weirdly, Peter Denning looks
+just like his wife, you could pick this gentleman out of a thousand guys
+as the soulmate of Dorothy Denning. Wearing tailored slacks,
+a spotless fuzzy varsity sweater, and a neatly knotted academician's tie. . . .
+This fineboned, exquisitely polite, utterly civilized and hyperintelligent
+couple seem to have emerged from some cleaner and finer parallel universe,
+where humanity exists to do the Brain Teasers column in Scientific American.
+Why does this Nice Lady hang out with these unsavory characters?
+
+Because the time has come for it, that's why.
+Because she's the best there is at what she does.
+
+Donn Parker is here, the Great Bald Eagle of Computer Crime. . . .
+With his bald dome, great height, and enormous Lincoln-like hands,
+the great visionary pioneer of the field plows through the lesser mortals
+like an icebreaker. . . . His eyes are fixed on the future with the
+rigidity of a bronze statue. . . . Eventually, he tells his audience,
+all business crime will be computer crime, because businesses will do
+everything through computers. "Computer crime" as a category will vanish.
+
+In the meantime, passing fads will flourish and fail and evaporate. . . .
+Parker's commanding, resonant voice is sphinxlike, everything is viewed
+from some eldritch valley of deep historical abstraction. . . .
+Yes, they've come and they've gone, these passing flaps in the world
+of digital computation. . . . The radio-frequency emanation scandal. . .
+KGB and MI5 and CIA do it every day, it's easy, but nobody else ever has. . . .
+The salami-slice fraud, mostly mythical. . . . "Crimoids," he calls them. . . .
+Computer viruses are the current crimoid champ, a lot less dangerous than
+most people let on, but the novelty is fading and there's a crimoid vacuum at
+the moment, the press is visibly hungering for something more outrageous. . . .
+The Great Man shares with us a few speculations on the coming crimoids. . . .
+Desktop Forgery! Wow. . . . Computers stolen just for the sake of the
+information within them--data-napping! Happened in Britain a while ago,
+could be the coming thing. . . . Phantom nodes in the Internet!
+
+Parker handles his overhead projector sheets with an ecclesiastical air. . . .
+He wears a grey double-breasted suit, a light blue shirt, and a
+very quiet tie of understated maroon and blue paisley. . . .
+Aphorisms emerge from him with slow, leaden emphasis. . . .
+There is no such thing as an adequately secure computer
+when one faces a sufficiently powerful adversary. . . .
+Deterrence is the most socially useful aspect of security. . . .
+People are the primary weakness in all information systems. . . .
+The entire baseline of computer security must be shifted upward. . . .
+Don't ever violate your security by publicly describing
+your security measures. . . .
+
+People in the audience are beginning to squirm, and yet
+there is something about the elemental purity of this guy's
+philosophy that compels uneasy respect. . . . Parker sounds
+like the only sane guy left in the lifeboat, sometimes.
+The guy who can prove rigorously, from deep moral principles,
+that Harvey there, the one with the broken leg and the checkered past,
+is the one who has to be, err. . .that is, Mr. Harvey is best placed
+to make the necessary sacrifice for the security and indeed
+the very survival of the rest of this lifeboat's crew. . . .
+Computer security, Parker informs us mournfully, is a
+nasty topic, and we wish we didn't have to have it. . . .
+The security expert, armed with method and logic, must think--imagine--
+everything that the adversary might do before the adversary might
+actually do it. It is as if the criminal's dark brain were an
+extensive subprogram within the shining cranium of Donn Parker.
+He is a Holmes whose Moriarty does not quite yet exist
+and so must be perfectly simulated.
+
+CFP is a stellar gathering, with the giddiness of a wedding.
+It is a happy time, a happy ending, they know their world
+is changing forever tonight, and they're proud to have been there
+to see it happen, to talk, to think, to help.
+
+And yet as night falls, a certain elegiac quality manifests itself,
+as the crowd gathers beneath the chandeliers with their wineglasses
+and dessert plates. Something is ending here, gone forever,
+and it takes a while to pinpoint it.
+
+It is the End of the Amateurs.
+
+
+
+
+
+
+
+
+
+End of the Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling
+
+*** END OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN ***
+
+***** This file should be named 101.txt or 101.zip *****
+This and all associated files of various formats will be found in:
+ http://www.gutenberg.org/1/0/101/
+
+
+
+Updated editions will replace the previous one--the old editions will be
+renamed.
+
+Creating the works from public domain print editions means that no one
+owns a United States copyright in these works, so the Foundation (and
+you!) can copy and distribute it in the United States without permission
+and without paying copyright royalties. Special rules, set forth in the
+General Terms of Use part of this license, apply to copying and
+distributing Project Gutenberg-tm electronic works to protect the
+PROJECT GUTENBERG-tm concept and trademark. Project Gutenberg is a
+registered trademark, and may not be used if you charge for the eBooks,
+unless you receive specific permission. If you do not charge anything
+for copies of this eBook, complying with the rules is very easy. You may
+use this eBook for nearly any purpose such as creation of derivative
+works, reports, performances and research. They may be modified and
+printed and given away--you may do practically ANYTHING with public
+domain eBooks. Redistribution is subject to the trademark license,
+especially commercial redistribution.
+
+
+
+*** START: FULL LICENSE ***
+
+THE FULL PROJECT GUTENBERG LICENSE
+PLEASE READ THIS BEFORE YOU DISTRIBUTE OR USE THIS WORK
+
+To protect the Project Gutenberg-tm mission of promoting the free
+distribution of electronic works, by using or distributing this work
+(or any other work associated in any way with the phrase "Project
+Gutenberg"), you agree to comply with all the terms of the Full Project
+Gutenberg-tm License (available with this file or online at
+http://www.gutenberg.org/license).
+
+
+Section 1. General Terms of Use and Redistributing Project Gutenberg-tm
+electronic works
+
+1.A. By reading or using any part of this Project Gutenberg-tm
+electronic work, you indicate that you have read, understand, agree to
+and accept all the terms of this license and intellectual property
+(trademark/copyright) agreement. If you do not agree to abide by all
+the terms of this agreement, you must cease using and return or destroy
+all copies of Project Gutenberg-tm electronic works in your possession.
+If you paid a fee for obtaining a copy of or access to a Project
+Gutenberg-tm electronic work and you do not agree to be bound by the
+terms of this agreement, you may obtain a refund from the person or
+entity to whom you paid the fee as set forth in paragraph 1.E.8.
+
+1.B. "Project Gutenberg" is a registered trademark. It may only be
+used on or associated in any way with an electronic work by people who
+agree to be bound by the terms of this agreement. There are a few
+things that you can do with most Project Gutenberg-tm electronic works
+even without complying with the full terms of this agreement. See
+paragraph 1.C below. There are a lot of things you can do with Project
+Gutenberg-tm electronic works if you follow the terms of this agreement
+and help preserve free future access to Project Gutenberg-tm electronic
+works. See paragraph 1.E below.
+
+1.C. The Project Gutenberg Literary Archive Foundation ("the Foundation"
+or PGLAF), owns a compilation copyright in the collection of Project
+Gutenberg-tm electronic works. Nearly all the individual works in the
+collection are in the public domain in the United States. If an
+individual work is in the public domain in the United States and you are
+located in the United States, we do not claim a right to prevent you from
+copying, distributing, performing, displaying or creating derivative
+works based on the work as long as all references to Project Gutenberg
+are removed. Of course, we hope that you will support the Project
+Gutenberg-tm mission of promoting free access to electronic works by
+freely sharing Project Gutenberg-tm works in compliance with the terms of
+this agreement for keeping the Project Gutenberg-tm name associated with
+the work. You can easily comply with the terms of this agreement by
+keeping this work in the same format with its attached full Project
+Gutenberg-tm License when you share it without charge with others.
+This particular work is one of the few copyrighted individual works
+included with the permission of the copyright holder. Information on
+the copyright owner for this particular work and the terms of use
+imposed by the copyright holder on this work are set forth at the
+beginning of this work.
+
+1.D. The copyright laws of the place where you are located also govern
+what you can do with this work. Copyright laws in most countries are in
+a constant state of change. If you are outside the United States, check
+the laws of your country in addition to the terms of this agreement
+before downloading, copying, displaying, performing, distributing or
+creating derivative works based on this work or any other Project
+Gutenberg-tm work. The Foundation makes no representations concerning
+the copyright status of any work in any country outside the United
+States.
+
+1.E. Unless you have removed all references to Project Gutenberg:
+
+1.E.1. The following sentence, with active links to, or other immediate
+access to, the full Project Gutenberg-tm License must appear prominently
+whenever any copy of a Project Gutenberg-tm work (any work on which the
+phrase "Project Gutenberg" appears, or with which the phrase "Project
+Gutenberg" is associated) is accessed, displayed, performed, viewed,
+copied or distributed:
+
+This eBook is for the use of anyone anywhere at no cost and with
+almost no restrictions whatsoever. You may copy it, give it away or
+re-use it under the terms of the Project Gutenberg License included
+with this eBook or online at www.gutenberg.org
+
+1.E.2. If an individual Project Gutenberg-tm electronic work is derived
+from the public domain (does not contain a notice indicating that it is
+posted with permission of the copyright holder), the work can be copied
+and distributed to anyone in the United States without paying any fees
+or charges. If you are redistributing or providing access to a work
+with the phrase "Project Gutenberg" associated with or appearing on the
+work, you must comply either with the requirements of paragraphs 1.E.1
+through 1.E.7 or obtain permission for the use of the work and the
+Project Gutenberg-tm trademark as set forth in paragraphs 1.E.8 or
+1.E.9.
+
+1.E.3. If an individual Project Gutenberg-tm electronic work is posted
+with the permission of the copyright holder, your use and distribution
+must comply with both paragraphs 1.E.1 through 1.E.7 and any additional
+terms imposed by the copyright holder. Additional terms will be linked
+to the Project Gutenberg-tm License for all works posted with the
+permission of the copyright holder found at the beginning of this work.
+
+1.E.4. Do not unlink or detach or remove the full Project Gutenberg-tm
+License terms from this work, or any files containing a part of this
+work or any other work associated with Project Gutenberg-tm.
+
+1.E.5. Do not copy, display, perform, distribute or redistribute this
+electronic work, or any part of this electronic work, without
+prominently displaying the sentence set forth in paragraph 1.E.1 with
+active links or immediate access to the full terms of the Project
+Gutenberg-tm License.
+
+1.E.6. You may convert to and distribute this work in any binary,
+compressed, marked up, nonproprietary or proprietary form, including any
+word processing or hypertext form. However, if you provide access to or
+distribute copies of a Project Gutenberg-tm work in a format other than
+"Plain Vanilla ASCII" or other format used in the official version
+posted on the official Project Gutenberg-tm web site (www.gutenberg.org),
+you must, at no additional cost, fee or expense to the user, provide a
+copy, a means of exporting a copy, or a means of obtaining a copy upon
+request, of the work in its original "Plain Vanilla ASCII" or other
+form. Any alternate format must include the full Project Gutenberg-tm
+License as specified in paragraph 1.E.1.
+
+1.E.7. Do not charge a fee for access to, viewing, displaying,
+performing, copying or distributing any Project Gutenberg-tm works
+unless you comply with paragraph 1.E.8 or 1.E.9.
+
+1.E.8. You may charge a reasonable fee for copies of or providing
+access to or distributing Project Gutenberg-tm electronic works provided
+that
+
+- You pay a royalty fee of 20% of the gross profits you derive from
+ the use of Project Gutenberg-tm works calculated using the method
+ you already use to calculate your applicable taxes. The fee is
+ owed to the owner of the Project Gutenberg-tm trademark, but he
+ has agreed to donate royalties under this paragraph to the
+ Project Gutenberg Literary Archive Foundation. Royalty payments
+ must be paid within 60 days following each date on which you
+ prepare (or are legally required to prepare) your periodic tax
+ returns. Royalty payments should be clearly marked as such and
+ sent to the Project Gutenberg Literary Archive Foundation at the
+ address specified in Section 4, "Information about donations to
+ the Project Gutenberg Literary Archive Foundation."
+
+- You provide a full refund of any money paid by a user who notifies
+ you in writing (or by e-mail) within 30 days of receipt that s/he
+ does not agree to the terms of the full Project Gutenberg-tm
+ License. You must require such a user to return or
+ destroy all copies of the works possessed in a physical medium
+ and discontinue all use of and all access to other copies of
+ Project Gutenberg-tm works.
+
+- You provide, in accordance with paragraph 1.F.3, a full refund of any
+ money paid for a work or a replacement copy, if a defect in the
+ electronic work is discovered and reported to you within 90 days
+ of receipt of the work.
+
+- You comply with all other terms of this agreement for free
+ distribution of Project Gutenberg-tm works.
+
+1.E.9. If you wish to charge a fee or distribute a Project Gutenberg-tm
+electronic work or group of works on different terms than are set
+forth in this agreement, you must obtain permission in writing from
+both the Project Gutenberg Literary Archive Foundation and Michael
+Hart, the owner of the Project Gutenberg-tm trademark. Contact the
+Foundation as set forth in Section 3 below.
+
+1.F.
+
+1.F.1. Project Gutenberg volunteers and employees expend considerable
+effort to identify, do copyright research on, transcribe and proofread
+public domain works in creating the Project Gutenberg-tm
+collection. Despite these efforts, Project Gutenberg-tm electronic
+works, and the medium on which they may be stored, may contain
+"Defects," such as, but not limited to, incomplete, inaccurate or
+corrupt data, transcription errors, a copyright or other intellectual
+property infringement, a defective or damaged disk or other medium, a
+computer virus, or computer codes that damage or cannot be read by
+your equipment.
+
+1.F.2. LIMITED WARRANTY, DISCLAIMER OF DAMAGES - Except for the "Right
+of Replacement or Refund" described in paragraph 1.F.3, the Project
+Gutenberg Literary Archive Foundation, the owner of the Project
+Gutenberg-tm trademark, and any other party distributing a Project
+Gutenberg-tm electronic work under this agreement, disclaim all
+liability to you for damages, costs and expenses, including legal
+fees. YOU AGREE THAT YOU HAVE NO REMEDIES FOR NEGLIGENCE, STRICT
+LIABILITY, BREACH OF WARRANTY OR BREACH OF CONTRACT EXCEPT THOSE
+PROVIDED IN PARAGRAPH 1.F.3. YOU AGREE THAT THE FOUNDATION, THE
+TRADEMARK OWNER, AND ANY DISTRIBUTOR UNDER THIS AGREEMENT WILL NOT BE
+LIABLE TO YOU FOR ACTUAL, DIRECT, INDIRECT, CONSEQUENTIAL, PUNITIVE OR
+INCIDENTAL DAMAGES EVEN IF YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+1.F.3. LIMITED RIGHT OF REPLACEMENT OR REFUND - If you discover a
+defect in this electronic work within 90 days of receiving it, you can
+receive a refund of the money (if any) you paid for it by sending a
+written explanation to the person you received the work from. If you
+received the work on a physical medium, you must return the medium with
+your written explanation. The person or entity that provided you with
+the defective work may elect to provide a replacement copy in lieu of a
+refund. If you received the work electronically, the person or entity
+providing it to you may choose to give you a second opportunity to
+receive the work electronically in lieu of a refund. If the second copy
+is also defective, you may demand a refund in writing without further
+opportunities to fix the problem.
+
+1.F.4. Except for the limited right of replacement or refund set forth
+in paragraph 1.F.3, this work is provided to you 'AS-IS,' WITH NO OTHER
+WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+WARRANTIES OF MERCHANTIBILITY OR FITNESS FOR ANY PURPOSE.
+
+1.F.5. Some states do not allow disclaimers of certain implied
+warranties or the exclusion or limitation of certain types of damages.
+If any disclaimer or limitation set forth in this agreement violates the
+law of the state applicable to this agreement, the agreement shall be
+interpreted to make the maximum disclaimer or limitation permitted by
+the applicable state law. The invalidity or unenforceability of any
+provision of this agreement shall not void the remaining provisions.
+
+1.F.6. INDEMNITY - You agree to indemnify and hold the Foundation, the
+trademark owner, any agent or employee of the Foundation, anyone
+providing copies of Project Gutenberg-tm electronic works in accordance
+with this agreement, and any volunteers associated with the production,
+promotion and distribution of Project Gutenberg-tm electronic works,
+harmless from all liability, costs and expenses, including legal fees,
+that arise directly or indirectly from any of the following which you do
+or cause to occur: (a) distribution of this or any Project Gutenberg-tm
+work, (b) alteration, modification, or additions or deletions to any
+Project Gutenberg-tm work, and (c) any Defect you cause.
+
+
+Section 2. Information about the Mission of Project Gutenberg-tm
+
+Project Gutenberg-tm is synonymous with the free distribution of
+electronic works in formats readable by the widest variety of computers
+including obsolete, old, middle-aged and new computers. It exists
+because of the efforts of hundreds of volunteers and donations from
+people in all walks of life.
+
+Volunteers and financial support to provide volunteers with the
+assistance they need are critical to reaching Project Gutenberg-tm's
+goals and ensuring that the Project Gutenberg-tm collection will
+remain freely available for generations to come. In 2001, the Project
+Gutenberg Literary Archive Foundation was created to provide a secure
+and permanent future for Project Gutenberg-tm and future generations.
+To learn more about the Project Gutenberg Literary Archive Foundation
+and how your efforts and donations can help, see Sections 3 and 4
+and the Foundation web page at http://www.pglaf.org.
+
+
+Section 3. Information about the Project Gutenberg Literary Archive
+Foundation
+
+The Project Gutenberg Literary Archive Foundation is a non profit
+501(c)(3) educational corporation organized under the laws of the
+state of Mississippi and granted tax exempt status by the Internal
+Revenue Service. The Foundation's EIN or federal tax identification
+number is 64-6221541. Its 501(c)(3) letter is posted at
+http://pglaf.org/fundraising. Contributions to the Project Gutenberg
+Literary Archive Foundation are tax deductible to the full extent
+permitted by U.S. federal laws and your state's laws.
+
+The Foundation's principal office is located at 4557 Melan Dr. S.
+Fairbanks, AK, 99712., but its volunteers and employees are scattered
+throughout numerous locations. Its business office is located at
+809 North 1500 West, Salt Lake City, UT 84116, (801) 596-1887, email
+business@pglaf.org. Email contact links and up to date contact
+information can be found at the Foundation's web site and official
+page at http://pglaf.org
+
+For additional contact information:
+ Dr. Gregory B. Newby
+ Chief Executive and Director
+ gbnewby@pglaf.org
+
+Section 4. Information about Donations to the Project Gutenberg
+Literary Archive Foundation
+
+Project Gutenberg-tm depends upon and cannot survive without wide
+spread public support and donations to carry out its mission of
+increasing the number of public domain and licensed works that can be
+freely distributed in machine readable form accessible by the widest
+array of equipment including outdated equipment. Many small donations
+($1 to $5,000) are particularly important to maintaining tax exempt
+status with the IRS.
+
+The Foundation is committed to complying with the laws regulating
+charities and charitable donations in all 50 states of the United
+States. Compliance requirements are not uniform and it takes a
+considerable effort, much paperwork and many fees to meet and keep up
+with these requirements. We do not solicit donations in locations
+where we have not received written confirmation of compliance. To
+SEND DONATIONS or determine the status of compliance for any
+particular state visit http://pglaf.org
+
+While we cannot and do not solicit contributions from states where we
+have not met the solicitation requirements, we know of no prohibition
+against accepting unsolicited donations from donors in such states who
+approach us with offers to donate.
+
+International donations are gratefully accepted, but we cannot make
+any statements concerning tax treatment of donations received from
+outside the United States. U.S. laws alone swamp our small staff.
+
+Please check the Project Gutenberg Web pages for current donation
+methods and addresses. Donations are accepted in a number of other
+ways including checks, online payments and credit card donations.
+To donate, please visit: http://pglaf.org/donate
+
+
+Section 5. General Information About Project Gutenberg-tm electronic
+works.
+
+Professor Michael S. Hart is the originator of the Project Gutenberg-tm
+concept of a library of electronic works that could be freely shared
+with anyone. For thirty years, he produced and distributed Project
+Gutenberg-tm eBooks with only a loose network of volunteer support.
+
+Project Gutenberg-tm eBooks are often created from several printed
+editions, all of which are confirmed as Public Domain in the U.S.
+unless a copyright notice is included. Thus, we do not necessarily
+keep eBooks in compliance with any particular paper edition.
+
+Each eBook is in a subdirectory of the same number as the eBook's
+eBook number, often in several formats including plain vanilla ASCII,
+compressed (zipped), HTML and others.
+
+Corrected EDITIONS of our eBooks replace the old file and take over
+the old filename and etext number. The replaced older file is renamed.
+VERSIONS based on separate sources are treated as new eBooks receiving
+new filenames and etext numbers.
+
+Most people start at our Web site which has the main PG search facility:
+
+http://www.gutenberg.org
+
+This Web site includes information about Project Gutenberg-tm,
+including how to make donations to the Project Gutenberg Literary
+Archive Foundation, how to help produce our new eBooks, and how to
+subscribe to our email newsletter to hear about new eBooks.
+
+EBooks posted prior to November 2003, with eBook numbers BELOW #10000,
+are filed in directories based on their release date. If you want to
+download any of these eBooks directly, rather than using the regular
+search system you may utilize the following addresses and just
+download by the etext year.
+
+http://www.ibiblio.org/gutenberg/etext06
+
+ (Or /etext 05, 04, 03, 02, 01, 00, 99,
+ 98, 97, 96, 95, 94, 93, 92, 92, 91 or 90)
+
+EBooks posted since November 2003, with etext numbers OVER #10000, are
+filed in a different way. The year of a release date is no longer part
+of the directory path. The path is based on the etext number (which is
+identical to the filename). The path to the file is made up of single
+digits corresponding to all but the last digit in the filename. For
+example an eBook of filename 10234 would be found at:
+
+http://www.gutenberg.org/1/0/2/3/10234
+
+or filename 24689 would be found at:
+http://www.gutenberg.org/2/4/6/8/24689
+
+An alternative method of locating eBooks:
+http://www.gutenberg.org/GUTINDEX.ALL
+
+*** END: FULL LICENSE ***
diff --git a/common/src/leap/soledad/common/tests/test_async.py b/common/src/leap/soledad/common/tests/test_async.py new file mode 100644 index 00000000..03b8c553 --- /dev/null +++ b/common/src/leap/soledad/common/tests/test_async.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# test_async.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import os +import hashlib + +from twisted.internet import defer + +from leap.soledad.common.tests.util import BaseSoledadTest +from leap.soledad.client import adbapi +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +class ASyncSQLCipherRetryTestCase(BaseSoledadTest): + """ + Test asynchronous SQLCipher operation. + """ + + NUM_DOCS = 5000 + + def _get_dbpool(self): + tmpdb = os.path.join(self.tempdir, "test.soledad") + opts = SQLCipherOptions(tmpdb, "secret", create=True) + return adbapi.getConnectionPool(opts) + + def _get_sample(self): + if not getattr(self, "_sample", None): + dirname = os.path.dirname(os.path.realpath(__file__)) + sample_file = os.path.join(dirname, "hacker_crackdown.txt") + with open(sample_file) as f: + self._sample = f.readlines() + return self._sample + + def test_concurrent_puts_fail_with_few_retries_and_small_timeout(self): + """ + Test if concurrent updates to the database with small timeout and + small number of retries fail with "database is locked" error. + + Many concurrent write attempts to the same sqlcipher database may fail + when the timeout is small and there are no retries. This test will + pass if any of the attempts to write the database fail. + + This test is much dependent on the environment and its result intends + to contrast with the test for the workaround for the "database is + locked" problem, which is addressed by the "test_concurrent_puts" test + below. + + If this test ever fails, it means that either (1) the platform where + you are running is it very powerful and you should try with an even + lower timeout value, or (2) the bug has been solved by a better + implementation of the underlying database pool, and thus this test + should be removed from the test suite. + """ + + old_timeout = adbapi.SQLCIPHER_CONNECTION_TIMEOUT + old_max_retries = adbapi.SQLCIPHER_MAX_RETRIES + + adbapi.SQLCIPHER_CONNECTION_TIMEOUT = 1 + adbapi.SQLCIPHER_MAX_RETRIES = 1 + + dbpool = self._get_dbpool() + + def _create_doc(doc): + return dbpool.runU1DBQuery("create_doc", doc) + + def _insert_docs(): + deferreds = [] + for i in range(self.NUM_DOCS): + payload = self._get_sample()[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = _create_doc(doc) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def _errback(e): + if e.value[0].getErrorMessage() == "database is locked": + adbapi.SQLCIPHER_CONNECTION_TIMEOUT = old_timeout + adbapi.SQLCIPHER_MAX_RETRIES = old_max_retries + return defer.succeed("") + raise Exception + + d = _insert_docs() + d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) + d.addErrback(_errback) + return d + + def test_concurrent_puts(self): + """ + Test that many concurrent puts succeed. + + Currently, there's a known problem with the concurrent database pool + which is that many concurrent attempts to write to the database may + fail when the lock timeout is small and when there are no (or few) + retries. We currently workaround this problem by increasing the + timeout and the number of retries. + + Should this test ever fail, it probably means that the timeout and/or + number of retries should be increased for the platform you're running + the test. If the underlying database pool is ever fixed, then the test + above will fail and we should remove this comment from here. + """ + + dbpool = self._get_dbpool() + + def _create_doc(doc): + return dbpool.runU1DBQuery("create_doc", doc) + + def _insert_docs(): + deferreds = [] + for i in range(self.NUM_DOCS): + payload = self._get_sample()[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = _create_doc(doc) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + + def _count_docs(results): + _, docs = results + if self.NUM_DOCS == len(docs): + return defer.succeed("") + raise Exception + + d = _insert_docs() + d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) + d.addCallback(_count_docs) + return d diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index 10d6c136..d2aef9bb 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -20,134 +20,21 @@ Test ObjectStore and Couch backend bits. """ -import re -import copy -import shutil -from base64 import b64decode -from mock import Mock -from urlparse import urljoin +import simplejson as json + +from urlparse import urljoin from u1db import errors as u1db_errors from couchdb.client import Server -from leap.common.files import mkdir_p +from testscenarios import TestWithScenarios + +from leap.soledad.common import couch, errors from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_backends from leap.soledad.common.tests.u1db_tests import test_sync -from leap.soledad.common import couch, errors -import simplejson as json - - -#----------------------------------------------------------------------------- -# A wrapper for running couchdb locally. -#----------------------------------------------------------------------------- - -import re -import os -import tempfile -import subprocess -import time -import unittest - - -# from: https://github.com/smcq/paisley/blob/master/paisley/test/util.py -# TODO: include license of above project. -class CouchDBWrapper(object): - """ - Wrapper for external CouchDB instance which is started and stopped for - testing. - """ - - def start(self): - """ - Start a CouchDB instance for a test. - """ - self.tempdir = tempfile.mkdtemp(suffix='.couch.test') - - path = os.path.join(os.path.dirname(__file__), - 'couchdb.ini.template') - handle = open(path) - conf = handle.read() % { - 'tempdir': self.tempdir, - } - handle.close() - - confPath = os.path.join(self.tempdir, 'test.ini') - handle = open(confPath, 'w') - handle.write(conf) - handle.close() - - # create the dirs from the template - mkdir_p(os.path.join(self.tempdir, 'lib')) - mkdir_p(os.path.join(self.tempdir, 'log')) - args = ['couchdb', '-n', '-a', confPath] - null = open('/dev/null', 'w') - - self.process = subprocess.Popen( - args, env=None, stdout=null.fileno(), stderr=null.fileno(), - close_fds=True) - # find port - logPath = os.path.join(self.tempdir, 'log', 'couch.log') - while not os.path.exists(logPath): - if self.process.poll() is not None: - got_stdout, got_stderr = "", "" - if self.process.stdout is not None: - got_stdout = self.process.stdout.read() - - if self.process.stderr is not None: - got_stderr = self.process.stderr.read() - raise Exception(""" -couchdb exited with code %d. -stdout: -%s -stderr: -%s""" % ( - self.process.returncode, got_stdout, got_stderr)) - time.sleep(0.01) - while os.stat(logPath).st_size == 0: - time.sleep(0.01) - PORT_RE = re.compile( - 'Apache CouchDB has started on http://127.0.0.1:(?P<port>\d+)') - - handle = open(logPath) - line = handle.read() - handle.close() - m = PORT_RE.search(line) - if not m: - self.stop() - raise Exception("Cannot find port in line %s" % line) - self.port = int(m.group('port')) - - def stop(self): - """ - Terminate the CouchDB instance. - """ - self.process.terminate() - self.process.communicate() - shutil.rmtree(self.tempdir) - - -class CouchDBTestCase(unittest.TestCase): - """ - TestCase base class for tests against a real CouchDB server. - """ - - @classmethod - def setUpClass(cls): - """ - Make sure we have a CouchDB instance for a test. - """ - cls.wrapper = CouchDBWrapper() - cls.wrapper.start() - #self.db = self.wrapper.db - - @classmethod - def tearDownClass(cls): - """ - Stop CouchDB instance for test. - """ - cls.wrapper.stop() +from leap.soledad.common.tests.util import CouchDBTestCase #----------------------------------------------------------------------------- @@ -239,7 +126,8 @@ COUCH_SCENARIOS = [ ] -class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase): +class CouchTests( + TestWithScenarios, test_backends.AllDatabaseTests, CouchDBTestCase): scenarios = COUCH_SCENARIOS @@ -262,7 +150,8 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase): test_backends.AllDatabaseTests.tearDown(self) -class CouchDatabaseTests(test_backends.LocalDatabaseTests, CouchDBTestCase): +class CouchDatabaseTests( + TestWithScenarios, test_backends.LocalDatabaseTests, CouchDBTestCase): scenarios = COUCH_SCENARIOS @@ -271,7 +160,7 @@ class CouchDatabaseTests(test_backends.LocalDatabaseTests, CouchDBTestCase): test_backends.LocalDatabaseTests.tearDown(self) -class CouchValidateGenNTransIdTests( +class CouchValidateGenNTransIdTests(TestWithScenarios, test_backends.LocalDatabaseValidateGenNTransIdTests, CouchDBTestCase): scenarios = COUCH_SCENARIOS @@ -281,7 +170,7 @@ class CouchValidateGenNTransIdTests( test_backends.LocalDatabaseValidateGenNTransIdTests.tearDown(self) -class CouchValidateSourceGenTests( +class CouchValidateSourceGenTests(TestWithScenarios, test_backends.LocalDatabaseValidateSourceGenTests, CouchDBTestCase): scenarios = COUCH_SCENARIOS @@ -291,7 +180,7 @@ class CouchValidateSourceGenTests( test_backends.LocalDatabaseValidateSourceGenTests.tearDown(self) -class CouchWithConflictsTests( +class CouchWithConflictsTests(TestWithScenarios, test_backends.LocalDatabaseWithConflictsTests, CouchDBTestCase): scenarios = COUCH_SCENARIOS @@ -325,23 +214,11 @@ simple_doc = tests.simple_doc nested_doc = tests.nested_doc -class CouchDatabaseSyncTargetTests(test_sync.DatabaseSyncTargetTests, - CouchDBTestCase): +class CouchDatabaseSyncTargetTests( + TestWithScenarios, test_sync.DatabaseSyncTargetTests, CouchDBTestCase): scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios)) - def setUp(self): - # we implement parents' setUp methods here to prevent from launching - # more couch instances then needed. - tests.TestCase.setUp(self) - self.server = self.server_thread = None - self.db, self.st = self.create_db_and_target(self) - self.other_changes = [] - - def tearDown(self): - self.db.delete_database() - test_sync.DatabaseSyncTargetTests.tearDown(self) - def test_sync_exchange_returns_many_new_docs(self): # This test was replicated to allow dictionaries to be compared after # JSON expansion (because one dictionary may have many different @@ -372,7 +249,7 @@ from u1db.backends.inmemory import InMemoryIndex class IndexedCouchDatabase(couch.CouchDatabase): def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True): - old_class.__init__(self, url, dbname, replica_uid=replica_uid, + old_class.__init__(self, url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs) self._indexes = {} @@ -458,7 +335,8 @@ for name, scenario in COUCH_SCENARIOS: scenario = dict(scenario) -class CouchDatabaseSyncTests(test_sync.DatabaseSyncTests, CouchDBTestCase): +class CouchDatabaseSyncTests( + TestWithScenarios, test_sync.DatabaseSyncTests, CouchDBTestCase): scenarios = sync_scenarios @@ -498,6 +376,7 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): def tearDown(self): self.db.delete_database() self.db.close() + CouchDBTestCase.tearDown(self) def test_missing_design_doc_raises(self): """ @@ -670,6 +549,3 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): self.assertRaises( errors.MissingDesignDocDeletedError, self.db._do_set_replica_gen_and_trans_id, 1, 2, 3) - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py index 6465eb80..3de4da1c 100644 --- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py +++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py @@ -15,26 +15,25 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Test atomocity for couch operations. +Test atomicity of couch operations. """ import os -import mock import tempfile import threading - from urlparse import urljoin - +from twisted.internet import defer from leap.soledad.client import Soledad from leap.soledad.common.couch import CouchDatabase, CouchServerState -from leap.soledad.common.tests.test_couch import CouchDBTestCase -from leap.soledad.common.tests.u1db_tests import TestCaseWithServer -from leap.soledad.common.tests.test_sync_target import ( + +from leap.soledad.common.tests.util import ( make_token_soledad_app, - make_leap_document_for_test, - token_leap_sync_target, + make_soledad_document_for_test, + token_soledad_sync_target, ) +from leap.soledad.common.tests.test_couch import CouchDBTestCase +from leap.soledad.common.tests.u1db_tests import TestCaseWithServer from leap.soledad.common.tests.test_server import _couch_ensure_database @@ -52,15 +51,15 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): def make_app_after_state(state): return make_token_soledad_app(state) - make_document_for_test = make_leap_document_for_test + make_document_for_test = make_soledad_document_for_test - sync_target = token_leap_sync_target + sync_target = token_soledad_sync_target def _soledad_instance(self, user='user-uuid', passphrase=u'123', prefix='', - secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME, + secrets_path='secrets.json', local_db_path='soledad.u1db', server_url='', - cert_file=None, auth_token=None, secret_id=None): + cert_file=None, auth_token=None): """ Instantiate Soledad. """ @@ -70,19 +69,6 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): def _put_doc_side_effect(doc): self._doc_put = doc - # we need a mocked shared db or else Soledad will try to access the - # network to find if there are uploaded secrets. - class MockSharedDB(object): - - get_doc = mock.Mock(return_value=None) - put_doc = mock.Mock(side_effect=_put_doc_side_effect) - lock = mock.Mock(return_value=('atoken', 300)) - unlock = mock.Mock() - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() return Soledad( user, passphrase, @@ -92,11 +78,10 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): server_url=server_url, cert_file=cert_file, auth_token=auth_token, - secret_id=secret_id) + shared_db=self.get_default_shared_mock(_put_doc_side_effect)) def make_app(self): - self.request_state = CouchServerState(self._couch_url, 'shared', - 'tokens') + self.request_state = CouchServerState(self._couch_url) return self.make_app_after_state(self.request_state) def setUp(self): @@ -126,7 +111,6 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): puts. """ doc = self.db.create_doc({'ops': 0}) - ops = 1 docs = [doc.doc_id] for i in range(0, REPEAT_TIMES): self.assertEqual( @@ -183,24 +167,27 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): auth_token='auth-token', server_url=self.getURL()) - def _create_docs_and_sync(sol, syncs): - # create a lot of documents - for i in range(0, REPEAT_TIMES): - sol.create_doc({}) + def _create_docs(results): + deferreds = [] + for i in xrange(0, REPEAT_TIMES): + deferreds.append(sol.create_doc({})) + return defer.DeferredList(deferreds) + + def _assert_transaction_and_sync_logs(results, sync_idx): # assert sizes of transaction and sync logs self.assertEqual( - syncs*REPEAT_TIMES, + sync_idx*REPEAT_TIMES, len(self.db._get_transaction_log())) self.assertEqual( - 1 if syncs > 0 else 0, + 1 if sync_idx > 0 else 0, len(self.db._database.view('syncs/log').rows)) - # sync to the remote db - sol.sync() - gen, docs = self.db.get_all_docs() - self.assertEqual((syncs+1)*REPEAT_TIMES, gen) - self.assertEqual((syncs+1)*REPEAT_TIMES, len(docs)) + + def _assert_sync(results, sync_idx): + gen, docs = results + self.assertEqual((sync_idx+1)*REPEAT_TIMES, gen) + self.assertEqual((sync_idx+1)*REPEAT_TIMES, len(docs)) # assert sizes of transaction and sync logs - self.assertEqual((syncs+1)*REPEAT_TIMES, + self.assertEqual((sync_idx+1)*REPEAT_TIMES, len(self.db._get_transaction_log())) sync_log_rows = self.db._database.view('syncs/log').rows sync_log = sync_log_rows[0].value @@ -210,14 +197,32 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): # assert sync_log has exactly 1 row self.assertEqual(1, len(sync_log_rows)) # assert it has the correct replica_uid, gen and trans_id - self.assertEqual(sol._db._replica_uid, replica_uid) - sol_gen, sol_trans_id = sol._db._get_generation_info() + self.assertEqual(sol._dbpool.replica_uid, replica_uid) + conn_key = sol._dbpool._u1dbconnections.keys().pop() + conn = sol._dbpool._u1dbconnections[conn_key] + sol_gen, sol_trans_id = conn._get_generation_info() self.assertEqual(sol_gen, known_gen) self.assertEqual(sol_trans_id, known_trans_id) + + # create some documents + d = _create_docs(None) - _create_docs_and_sync(sol, 0) - _create_docs_and_sync(sol, 1) - sol.close() + # sync first time and assert success + d.addCallback(_assert_transaction_and_sync_logs, 0) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: sol.get_all_docs()) + d.addCallback(_assert_sync, 0) + + # create more docs, sync second time and assert success + d.addCallback(_create_docs) + d.addCallback(_assert_transaction_and_sync_logs, 1) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: sol.get_all_docs()) + d.addCallback(_assert_sync, 1) + + d.addCallback(lambda _: sol.close()) + + return d # # Concurrency tests @@ -313,86 +318,76 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): """ Assert that the sync_log is correct after concurrent syncs. """ - threads = [] docs = [] - pool = threading.BoundedSemaphore(value=1) + self.startServer() + sol = self._soledad_instance( auth_token='auth-token', server_url=self.getURL()) - def _run_method(self): - # create a lot of documents - doc = self._params['sol'].create_doc({}) - pool.acquire() - docs.append(doc.doc_id) - pool.release() + def _save_doc_ids(results): + for doc in results: + docs.append(doc.doc_id) - # launch threads to create documents in parallel + # create documents in parallel + deferreds = [] for i in range(0, REPEAT_TIMES): - thread = self._WorkerThread( - {'sol': sol, 'syncs': i}, - _run_method) - thread.start() - threads.append(thread) + d = sol.create_doc({}) + deferreds.append(d) - # wait for threads to finish - for thread in threads: - thread.join() + # wait for documents creation and sync + d = defer.gatherResults(deferreds) + d.addCallback(_save_doc_ids) + d.addCallback(lambda _: sol.sync()) - # do the sync! - sol.sync() + def _assert_logs(results): + transaction_log = self.db._get_transaction_log() + self.assertEqual(REPEAT_TIMES, len(transaction_log)) + # assert all documents are in the remote log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) - transaction_log = self.db._get_transaction_log() - self.assertEqual(REPEAT_TIMES, len(transaction_log)) - # assert all documents are in the remote log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - sol.close() + d.addCallback(_assert_logs) + d.addCallback(lambda _: sol.close()) + + return d def test_concurrent_syncs_do_not_fail(self): """ Assert that concurrent attempts to sync end up being executed sequentially and do not fail. """ - threads = [] docs = [] - pool = threading.BoundedSemaphore(value=1) + self.startServer() + sol = self._soledad_instance( auth_token='auth-token', server_url=self.getURL()) - def _run_method(self): - # create a lot of documents - doc = self._params['sol'].create_doc({}) - # do the sync! - sol.sync() - pool.acquire() - docs.append(doc.doc_id) - pool.release() - - # launch threads to create documents in parallel - for i in range(0, REPEAT_TIMES): - thread = self._WorkerThread( - {'sol': sol, 'syncs': i}, - _run_method) - thread.start() - threads.append(thread) - - # wait for threads to finish - for thread in threads: - thread.join() - - transaction_log = self.db._get_transaction_log() - self.assertEqual(REPEAT_TIMES, len(transaction_log)) - # assert all documents are in the remote log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - sol.close() + deferreds = [] + for i in xrange(0, REPEAT_TIMES): + d = sol.create_doc({}) + d.addCallback(lambda doc: docs.append(doc.doc_id)) + d.addCallback(lambda _: sol.sync()) + deferreds.append(d) + + def _assert_logs(results): + transaction_log = self.db._get_transaction_log() + self.assertEqual(REPEAT_TIMES, len(transaction_log)) + # assert all documents are in the remote log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + d = defer.gatherResults(deferreds) + d.addCallback(_assert_logs) + d.addCallback(lambda _: sol.close()) + + return d diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index 1071af14..fdad8aac 100644 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -23,8 +23,14 @@ import binascii from leap.soledad.client import crypto from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.crypto import WrongMac, UnknownMacMethod +from leap.soledad.common.tests.util import BaseSoledadTest +from leap.soledad.common.crypto import WrongMacError +from leap.soledad.common.crypto import UnknownMacMethodError +from leap.soledad.common.crypto import EncryptionMethods +from leap.soledad.common.crypto import ENC_JSON_KEY +from leap.soledad.common.crypto import ENC_SCHEME_KEY +from leap.soledad.common.crypto import MAC_KEY +from leap.soledad.common.crypto import MAC_METHOD_KEY class EncryptedSyncTestCase(BaseSoledadTest): @@ -46,8 +52,8 @@ class EncryptedSyncTestCase(BaseSoledadTest): self.assertNotEqual( simpledoc, doc1.content, 'incorrect document encryption') - self.assertTrue(crypto.ENC_JSON_KEY in doc1.content) - self.assertTrue(crypto.ENC_SCHEME_KEY in doc1.content) + self.assertTrue(ENC_JSON_KEY in doc1.content) + self.assertTrue(ENC_SCHEME_KEY in doc1.content) # decrypt doc doc1.set_json(crypto.decrypt_doc(self._soledad._crypto, doc1)) self.assertEqual( @@ -57,23 +63,28 @@ class EncryptedSyncTestCase(BaseSoledadTest): class RecoveryDocumentTestCase(BaseSoledadTest): def test_export_recovery_document_raw(self): - rd = self._soledad.export_recovery_document() - secret_id = rd[self._soledad.STORAGE_SECRETS_KEY].items()[0][0] - secret = rd[self._soledad.STORAGE_SECRETS_KEY][secret_id] - self.assertEqual(secret_id, self._soledad._secret_id) - self.assertEqual(secret, self._soledad._secrets[secret_id]) - self.assertTrue(self._soledad.CIPHER_KEY in secret) - self.assertTrue(secret[self._soledad.CIPHER_KEY] == 'aes256') - self.assertTrue(self._soledad.LENGTH_KEY in secret) - self.assertTrue(self._soledad.SECRET_KEY in secret) + rd = self._soledad.secrets._export_recovery_document() + secret_id = rd[self._soledad.secrets.STORAGE_SECRETS_KEY].items()[0][0] + # assert exported secret is the same + secret = self._soledad.secrets._decrypt_storage_secret( + rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id]) + self.assertEqual(secret_id, self._soledad.secrets._secret_id) + self.assertEqual(secret, self._soledad.secrets._secrets[secret_id]) + # assert recovery document structure + encrypted_secret = rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id] + self.assertTrue(self._soledad.secrets.CIPHER_KEY in encrypted_secret) + self.assertTrue( + encrypted_secret[self._soledad.secrets.CIPHER_KEY] == 'aes256') + self.assertTrue(self._soledad.secrets.LENGTH_KEY in encrypted_secret) + self.assertTrue(self._soledad.secrets.SECRET_KEY in encrypted_secret) def test_import_recovery_document(self): - rd = self._soledad.export_recovery_document() + rd = self._soledad.secrets._export_recovery_document() s = self._soledad_instance() - s.import_recovery_document(rd) - s._set_secret_id(self._soledad._secret_id) - self.assertEqual(self._soledad._get_storage_secret(), - s._get_storage_secret(), + s.secrets._import_recovery_document(rd) + s.secrets.set_secret_id(self._soledad.secrets._secret_id) + self.assertEqual(self._soledad.storage_secret, + s.storage_secret, 'Failed settinng secret for symmetric encryption.') s.close() @@ -83,32 +94,32 @@ class SoledadSecretsTestCase(BaseSoledadTest): def test__gen_secret(self): # instantiate and save secret_id sol = self._soledad_instance(user='user@leap.se') - self.assertTrue(len(sol._secrets) == 1) - secret_id_1 = sol.secret_id + self.assertTrue(len(sol.secrets._secrets) == 1) + secret_id_1 = sol.secrets.secret_id # assert id is hash of secret self.assertTrue( secret_id_1 == hashlib.sha256(sol.storage_secret).hexdigest()) # generate new secret - secret_id_2 = sol._gen_secret() + secret_id_2 = sol.secrets._gen_secret() self.assertTrue(secret_id_1 != secret_id_2) sol.close() # re-instantiate - sol = self._soledad_instance( - user='user@leap.se', - secret_id=secret_id_1) + sol = self._soledad_instance(user='user@leap.se') + sol.secrets.set_secret_id(secret_id_1) # assert ids are valid - self.assertTrue(len(sol._secrets) == 2) - self.assertTrue(secret_id_1 in sol._secrets) - self.assertTrue(secret_id_2 in sol._secrets) + self.assertTrue(len(sol.secrets._secrets) == 2) + self.assertTrue(secret_id_1 in sol.secrets._secrets) + self.assertTrue(secret_id_2 in sol.secrets._secrets) # assert format of secret 1 self.assertTrue(sol.storage_secret is not None) self.assertIsInstance(sol.storage_secret, str) - self.assertTrue(len(sol.storage_secret) == sol.GENERATED_SECRET_LENGTH) + secret_length = sol.secrets.GEN_SECRET_LENGTH + self.assertTrue(len(sol.storage_secret) == secret_length) # assert format of secret 2 - sol._set_secret_id(secret_id_2) + sol.secrets.set_secret_id(secret_id_2) self.assertTrue(sol.storage_secret is not None) self.assertIsInstance(sol.storage_secret, str) - self.assertTrue(len(sol.storage_secret) == sol.GENERATED_SECRET_LENGTH) + self.assertTrue(len(sol.storage_secret) == secret_length) # assert id is hash of new secret self.assertTrue( secret_id_2 == hashlib.sha256(sol.storage_secret).hexdigest()) @@ -117,16 +128,18 @@ class SoledadSecretsTestCase(BaseSoledadTest): def test__has_secret(self): sol = self._soledad_instance( user='user@leap.se', prefix=self.rand_prefix) - self.assertTrue(sol._has_secret(), "Should have a secret at " - "this point") + self.assertTrue( + sol.secrets._has_secret(), + "Should have a secret at this point") # setting secret id to None should not interfere in the fact we have a # secret. - sol._set_secret_id(None) - self.assertTrue(sol._has_secret(), "Should have a secret at " - "this point") + sol.secrets.set_secret_id(None) + self.assertTrue( + sol.secrets._has_secret(), + "Should have a secret at this point") # but not being able to decrypt correctly should - sol._secrets[sol.secret_id][sol.SECRET_KEY] = None - self.assertFalse(sol._has_secret()) + sol.secrets._secrets[sol.secrets.secret_id] = None + self.assertFalse(sol.secrets._has_secret()) sol.close() @@ -141,13 +154,13 @@ class MacAuthTestCase(BaseSoledadTest): doc.content = simpledoc # encrypt doc doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc)) - self.assertTrue(crypto.MAC_KEY in doc.content) - self.assertTrue(crypto.MAC_METHOD_KEY in doc.content) + self.assertTrue(MAC_KEY in doc.content) + self.assertTrue(MAC_METHOD_KEY in doc.content) # mess with MAC - doc.content[crypto.MAC_KEY] = '1234567890ABCDEF' + doc.content[MAC_KEY] = '1234567890ABCDEF' # try to decrypt doc self.assertRaises( - WrongMac, + WrongMacError, crypto.decrypt_doc, self._soledad._crypto, doc) def test_decrypt_with_unknown_mac_method_raises(self): @@ -159,13 +172,13 @@ class MacAuthTestCase(BaseSoledadTest): doc.content = simpledoc # encrypt doc doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc)) - self.assertTrue(crypto.MAC_KEY in doc.content) - self.assertTrue(crypto.MAC_METHOD_KEY in doc.content) + self.assertTrue(MAC_KEY in doc.content) + self.assertTrue(MAC_METHOD_KEY in doc.content) # mess with MAC method - doc.content[crypto.MAC_METHOD_KEY] = 'mymac' + doc.content[MAC_METHOD_KEY] = 'mymac' # try to decrypt doc self.assertRaises( - UnknownMacMethod, + UnknownMacMethodError, crypto.decrypt_doc, self._soledad._crypto, doc) @@ -176,20 +189,20 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') plaintext = self._soledad._crypto.decrypt_sym( cyphertext, key, iv=iv, - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertEqual('data', plaintext) def test_decrypt_with_wrong_iv_fails(self): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -200,14 +213,14 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): wrongiv = os.urandom(1) + rawiv[1:] plaintext = self._soledad._crypto.decrypt_sym( cyphertext, key, iv=binascii.b2a_base64(wrongiv), - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertNotEqual('data', plaintext) def test_decrypt_with_wrong_key_fails(self): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -217,7 +230,7 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): wrongkey = os.urandom(32) plaintext = self._soledad._crypto.decrypt_sym( cyphertext, wrongkey, iv=iv, - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertNotEqual('data', plaintext) @@ -228,20 +241,20 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') plaintext = self._soledad._crypto.decrypt_sym( cyphertext, key, iv=iv, - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertEqual('data', plaintext) def test_decrypt_with_wrong_iv_fails(self): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -252,14 +265,14 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest): wrongiv = os.urandom(1) + rawiv[1:] plaintext = self._soledad._crypto.decrypt_sym( cyphertext, key, iv=binascii.b2a_base64(wrongiv), - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertNotEqual('data', plaintext) def test_decrypt_with_wrong_key_fails(self): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -269,5 +282,5 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest): wrongkey = os.urandom(32) plaintext = self._soledad._crypto.decrypt_sym( cyphertext, wrongkey, iv=iv, - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertNotEqual('data', plaintext) diff --git a/common/src/leap/soledad/common/tests/test_http.py b/common/src/leap/soledad/common/tests/test_http.py index d21470e0..1f661b77 100644 --- a/common/src/leap/soledad/common/tests/test_http.py +++ b/common/src/leap/soledad/common/tests/test_http.py @@ -20,8 +20,6 @@ Test Leap backend bits: test http database from u1db.remote import http_database from leap.soledad.client import auth - -from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_http_database @@ -59,6 +57,3 @@ class TestHTTPDatabaseWithCreds( 'token': 'auth-token', }}) self.assertIn('token', db1._creds) - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_http_client.py b/common/src/leap/soledad/common/tests/test_http_client.py index 3169398b..db731c32 100644 --- a/common/src/leap/soledad/common/tests/test_http_client.py +++ b/common/src/leap/soledad/common/tests/test_http_client.py @@ -21,8 +21,9 @@ import json from u1db.remote import http_client +from testscenarios import TestWithScenarios + from leap.soledad.client import auth -from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_http_client from leap.soledad.server.auth import SoledadTokenAuthMiddleware @@ -31,7 +32,9 @@ from leap.soledad.server.auth import SoledadTokenAuthMiddleware # The following tests come from `u1db.tests.test_http_client`. #----------------------------------------------------------------------------- -class TestSoledadClientBase(test_http_client.TestHTTPClientBase): +class TestSoledadClientBase( + TestWithScenarios, + test_http_client.TestHTTPClientBase): """ This class should be used to test Token auth. """ @@ -90,7 +93,7 @@ class TestSoledadClientBase(test_http_client.TestHTTPClientBase): "message": e.message})] uuid, token = encoded.decode('base64').split(':', 1) if uuid != 'user-uuid' and token != 'auth-token': - return unauth_err("Incorrect address or token.") + return Exception("Incorrect address or token.") start_response("200 OK", [('Content-Type', 'application/json')]) return [json.dumps([environ['PATH_INFO'], uuid, token])] @@ -112,5 +115,3 @@ class TestSoledadClientBase(test_http_client.TestHTTPClientBase): res, headers = cli._request('GET', ['doc', 'token']) self.assertEqual( ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_https.py b/common/src/leap/soledad/common/tests/test_https.py index b6288188..4dd55754 100644 --- a/common/src/leap/soledad/common/tests/test_https.py +++ b/common/src/leap/soledad/common/tests/test_https.py @@ -14,30 +14,35 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + + """ Test Leap backend bits: https """ -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.tests import test_sync_target as test_st -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import test_backends -from leap.soledad.common.tests.u1db_tests import test_https -from leap.soledad import client -from leap.soledad.server import SoledadApp from u1db.remote import http_client +from leap.soledad import client + +from testscenarios import TestWithScenarios + +from leap.soledad.common.tests.u1db_tests import test_backends +from leap.soledad.common.tests.u1db_tests import test_https +from leap.soledad.common.tests.util import ( + BaseSoledadTest, + make_soledad_document_for_test, + make_soledad_app, + make_token_soledad_app, +) -def make_soledad_app(state): - return SoledadApp(state) LEAP_SCENARIOS = [ ('http', { 'make_database_for_test': test_backends.make_http_database_for_test, 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': test_st.make_leap_document_for_test, - 'make_app_with_state': test_st.make_soledad_app}), + 'make_document_for_test': make_soledad_document_for_test, + 'make_app_with_state': make_soledad_app}), ] @@ -55,14 +60,15 @@ def token_leap_https_sync_target(test, host, path): class TestSoledadSyncTargetHttpsSupport( + TestWithScenarios, test_https.TestHttpSyncTargetHttpsSupport, BaseSoledadTest): scenarios = [ ('token_soledad_https', {'server_def': test_https.https_server_def, - 'make_app_with_state': test_st.make_token_soledad_app, - 'make_document_for_test': test_st.make_leap_document_for_test, + 'make_app_with_state': make_token_soledad_app, + 'make_document_for_test': make_soledad_document_for_test, 'sync_target': token_leap_https_sync_target}), ] @@ -71,8 +77,8 @@ class TestSoledadSyncTargetHttpsSupport( # run smoothly with standard u1db. test_https.TestHttpSyncTargetHttpsSupport.setUp(self) # so here monkey patch again to test our functionality. - http_client._VerifiedHTTPSConnection = client.VerifiedHTTPSConnection - client.SOLEDAD_CERT = http_client.CA_CERTS + http_client._VerifiedHTTPSConnection = client.api.VerifiedHTTPSConnection + client.api.SOLEDAD_CERT = http_client.CA_CERTS def test_working(self): """ @@ -83,7 +89,7 @@ class TestSoledadSyncTargetHttpsSupport( """ self.startServer() db = self.request_state._create_database('test') - self.patch(client, 'SOLEDAD_CERT', self.cacert_pem) + self.patch(client.api, 'SOLEDAD_CERT', self.cacert_pem) remote_target = self.getSyncTarget('localhost', 'test') remote_target.record_sync_info('other-id', 2, 'T-id') self.assertEqual( @@ -99,10 +105,8 @@ class TestSoledadSyncTargetHttpsSupport( """ self.startServer() self.request_state._create_database('test') - self.patch(client, 'SOLEDAD_CERT', self.cacert_pem) + self.patch(client.api, 'SOLEDAD_CERT', self.cacert_pem) remote_target = self.getSyncTarget('127.0.0.1', 'test') self.assertRaises( http_client.CertificateError, remote_target.record_sync_info, 'other-id', 2, 'T-id') - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index cb5348b4..2b653a1c 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -19,29 +19,28 @@ Tests for server-related functionality. """ import os import tempfile -import simplejson as json import mock import time import binascii from urlparse import urljoin +from twisted.internet import defer -from leap.common.testing.basetest import BaseLeapTest from leap.soledad.common.couch import ( CouchServerState, CouchDatabase, ) -from leap.soledad.common.tests.u1db_tests import ( - TestCaseWithServer, - simple_doc, -) +from leap.soledad.common.tests.u1db_tests import TestCaseWithServer from leap.soledad.common.tests.test_couch import CouchDBTestCase -from leap.soledad.common.tests.test_target_soledad import ( +from leap.soledad.common.tests.util import ( make_token_soledad_app, - make_leap_document_for_test, + make_soledad_document_for_test, + token_soledad_sync_target, + BaseSoledadTest, ) -from leap.soledad.common.tests.test_sync_target import token_leap_sync_target -from leap.soledad.client import Soledad, crypto + +from leap.soledad.common import crypto +from leap.soledad.client import Soledad from leap.soledad.server import LockResource from leap.soledad.server.auth import URLToAuthorization @@ -58,7 +57,7 @@ def _couch_ensure_database(self, dbname): CouchServerState.ensure_database = _couch_ensure_database -class ServerAuthorizationTestCase(BaseLeapTest): +class ServerAuthorizationTestCase(BaseSoledadTest): """ Tests related to Soledad server authorization. """ @@ -272,19 +271,24 @@ class EncryptedSyncTestCase( Tests for encrypted sync using Soledad server backed by a couch database. """ + # increase twisted.trial's timeout because large files syncing might take + # some time to finish. + timeout = 500 + @staticmethod def make_app_with_state(state): return make_token_soledad_app(state) - make_document_for_test = make_leap_document_for_test + make_document_for_test = make_soledad_document_for_test - sync_target = token_leap_sync_target + sync_target = token_soledad_sync_target def _soledad_instance(self, user='user-uuid', passphrase=u'123', prefix='', - secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME, - local_db_path='soledad.u1db', server_url='', - cert_file=None, auth_token=None, secret_id=None): + secrets_path='secrets.json', + local_db_path='soledad.u1db', + server_url='', + cert_file=None, auth_token=None): """ Instantiate Soledad. """ @@ -294,19 +298,15 @@ class EncryptedSyncTestCase( def _put_doc_side_effect(doc): self._doc_put = doc - # we need a mocked shared db or else Soledad will try to access the - # network to find if there are uploaded secrets. - class MockSharedDB(object): - - get_doc = mock.Mock(return_value=None) - put_doc = mock.Mock(side_effect=_put_doc_side_effect) - lock = mock.Mock(return_value=('atoken', 300)) - unlock = mock.Mock() - - def __call__(self): - return self + if not server_url: + # attempt to find the soledad server url + server_address = None + server = getattr(self, 'server', None) + if server: + server_address = getattr(self.server, 'server_address', None) + if server_address: + server_url = 'http://%s:%d' % (server_address) - Soledad._shared_db = MockSharedDB() return Soledad( user, passphrase, @@ -316,78 +316,129 @@ class EncryptedSyncTestCase( server_url=server_url, cert_file=cert_file, auth_token=auth_token, - secret_id=secret_id) + shared_db=self.get_default_shared_mock(_put_doc_side_effect)) def make_app(self): - self.request_state = CouchServerState(self._couch_url, 'shared', - 'tokens') + self.request_state = CouchServerState(self._couch_url) return self.make_app_with_state(self.request_state) def setUp(self): - TestCaseWithServer.setUp(self) + # the order of the following initializations is crucial because of + # dependencies. + # XXX explain better CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") self._couch_url = 'http://localhost:' + str(self.wrapper.port) + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + TestCaseWithServer.setUp(self) def tearDown(self): CouchDBTestCase.tearDown(self) TestCaseWithServer.tearDown(self) - def test_encrypted_sym_sync(self): + def _test_encrypted_sym_sync(self, passphrase=u'123', doc_size=2, + number_of_docs=1): """ Test the complete syncing chain between two soledad dbs using a Soledad server backed by a couch database. """ self.startServer() + # instantiate soledad and create a document sol1 = self._soledad_instance( # token is verified in test_target.make_token_soledad_app - auth_token='auth-token' - ) - _, doclist = sol1.get_all_docs() - self.assertEqual([], doclist) - doc1 = sol1.create_doc(json.loads(simple_doc)) + auth_token='auth-token', + passphrase=passphrase) + + # instantiate another soledad using the same secret as the previous + # one (so we can correctly verify the mac of the synced document) + sol2 = self._soledad_instance( + prefix='x', + auth_token='auth-token', + secrets_path=sol1._secrets_path, + passphrase=passphrase) + # ensure remote db exists before syncing db = CouchDatabase.open_database( urljoin(self._couch_url, 'user-user-uuid'), create=True, ensure_ddocs=True) - # sync with server - sol1._server_url = self.getURL() - sol1.sync() - # assert doc was sent to couch db - _, doclist = db.get_all_docs() - self.assertEqual(1, len(doclist)) - couchdoc = doclist[0] - # assert document structure in couch server - self.assertEqual(doc1.doc_id, couchdoc.doc_id) - self.assertEqual(doc1.rev, couchdoc.rev) - self.assertEqual(6, len(couchdoc.content)) - self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content) - # instantiate soledad with empty db, but with same secrets path - sol2 = self._soledad_instance(prefix='x', auth_token='auth-token') - _, doclist = sol2.get_all_docs() - self.assertEqual([], doclist) - sol2._secrets_path = sol1.secrets_path - sol2._load_secrets() - sol2._set_secret_id(sol1._secret_id) - # sync the new instance - sol2._server_url = self.getURL() - sol2.sync() - _, doclist = sol2.get_all_docs() - self.assertEqual(1, len(doclist)) - doc2 = doclist[0] - # assert incoming doc is equal to the first sent doc - self.assertEqual(doc1, doc2) - db.delete_database() - db.close() - sol1.close() - sol2.close() + + def _db1AssertEmptyDocList(results): + _, doclist = results + self.assertEqual([], doclist) + + def _db1CreateDocs(results): + deferreds = [] + for i in xrange(number_of_docs): + content = binascii.hexlify(os.urandom(doc_size/2)) + deferreds.append(sol1.create_doc({'data': content})) + return defer.DeferredList(deferreds) + + def _db1AssertDocsSyncedToServer(results): + _, sol_doclist = results + self.assertEqual(number_of_docs, len(sol_doclist)) + # assert doc was sent to couch db + _, couch_doclist = db.get_all_docs() + self.assertEqual(number_of_docs, len(couch_doclist)) + for i in xrange(number_of_docs): + soldoc = sol_doclist.pop() + couchdoc = couch_doclist.pop() + # assert document structure in couch server + self.assertEqual(soldoc.doc_id, couchdoc.doc_id) + self.assertEqual(soldoc.rev, couchdoc.rev) + self.assertEqual(6, len(couchdoc.content)) + self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content) + self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content) + self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content) + self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content) + self.assertTrue(crypto.MAC_KEY in couchdoc.content) + self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content) + + d = sol1.get_all_docs() + d.addCallback(_db1AssertEmptyDocList) + d.addCallback(_db1CreateDocs) + d.addCallback(lambda _: sol1.sync()) + d.addCallback(lambda _: sol1.get_all_docs()) + d.addCallback(_db1AssertDocsSyncedToServer) + + def _db2AssertEmptyDocList(results): + _, doclist = results + self.assertEqual([], doclist) + + def _getAllDocsFromBothDbs(results): + d1 = sol1.get_all_docs() + d2 = sol2.get_all_docs() + return defer.DeferredList([d1, d2]) + + d.addCallback(lambda _: sol2.get_all_docs()) + d.addCallback(_db2AssertEmptyDocList) + d.addCallback(lambda _: sol2.sync()) + d.addCallback(_getAllDocsFromBothDbs) + + def _assertDocSyncedFromDb1ToDb2(results): + r1, r2 = results + _, (gen1, doclist1) = r1 + _, (gen2, doclist2) = r2 + self.assertEqual(number_of_docs, gen1) + self.assertEqual(number_of_docs, gen2) + self.assertEqual(number_of_docs, len(doclist1)) + self.assertEqual(number_of_docs, len(doclist2)) + self.assertEqual(doclist1[0], doclist2[0]) + + d.addCallback(_assertDocSyncedFromDb1ToDb2) + + def _cleanUp(results): + db.delete_database() + db.close() + sol1.close() + sol2.close() + + d.addCallback(_cleanUp) + + return d + + def test_encrypted_sym_sync(self): + return self._test_encrypted_sym_sync() def test_encrypted_sym_sync_with_unicode_passphrase(self): """ @@ -395,152 +446,20 @@ class EncryptedSyncTestCase( Soledad server backed by a couch database, using an unicode passphrase. """ - self.startServer() - # instantiate soledad and create a document - sol1 = self._soledad_instance( - # token is verified in test_target.make_token_soledad_app - auth_token='auth-token', - passphrase=u'ãáà äéà ëÃìïóòöõúùüñç', - ) - _, doclist = sol1.get_all_docs() - self.assertEqual([], doclist) - doc1 = sol1.create_doc(json.loads(simple_doc)) - # ensure remote db exists before syncing - db = CouchDatabase.open_database( - urljoin(self._couch_url, 'user-user-uuid'), - create=True, - ensure_ddocs=True) - # sync with server - sol1._server_url = self.getURL() - sol1.sync() - # assert doc was sent to couch db - _, doclist = db.get_all_docs() - self.assertEqual(1, len(doclist)) - couchdoc = doclist[0] - # assert document structure in couch server - self.assertEqual(doc1.doc_id, couchdoc.doc_id) - self.assertEqual(doc1.rev, couchdoc.rev) - self.assertEqual(6, len(couchdoc.content)) - self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content) - # instantiate soledad with empty db, but with same secrets path - sol2 = self._soledad_instance( - prefix='x', - auth_token='auth-token', - passphrase=u'ãáà äéà ëÃìïóòöõúùüñç', - ) - _, doclist = sol2.get_all_docs() - self.assertEqual([], doclist) - sol2._secrets_path = sol1.secrets_path - sol2._load_secrets() - sol2._set_secret_id(sol1._secret_id) - # sync the new instance - sol2._server_url = self.getURL() - sol2.sync() - _, doclist = sol2.get_all_docs() - self.assertEqual(1, len(doclist)) - doc2 = doclist[0] - # assert incoming doc is equal to the first sent doc - self.assertEqual(doc1, doc2) - db.delete_database() - db.close() - sol1.close() - sol2.close() + return self._test_encrypted_sym_sync(passphrase=u'ãáà äéà ëÃìïóòöõúùüñç') def test_sync_very_large_files(self): """ Test if Soledad can sync very large files. """ - # define the size of the "very large file" length = 100*(10**6) # 100 MB - self.startServer() - # instantiate soledad and create a document - sol1 = self._soledad_instance( - # token is verified in test_target.make_token_soledad_app - auth_token='auth-token' - ) - _, doclist = sol1.get_all_docs() - self.assertEqual([], doclist) - content = binascii.hexlify(os.urandom(length/2)) # len() == length - doc1 = sol1.create_doc({'data': content}) - # ensure remote db exists before syncing - db = CouchDatabase.open_database( - urljoin(self._couch_url, 'user-user-uuid'), - create=True, - ensure_ddocs=True) - # sync with server - sol1._server_url = self.getURL() - sol1.sync() - # instantiate soledad with empty db, but with same secrets path - sol2 = self._soledad_instance(prefix='x', auth_token='auth-token') - _, doclist = sol2.get_all_docs() - self.assertEqual([], doclist) - sol2._secrets_path = sol1.secrets_path - sol2._load_secrets() - sol2._set_secret_id(sol1._secret_id) - # sync the new instance - sol2._server_url = self.getURL() - sol2.sync() - _, doclist = sol2.get_all_docs() - self.assertEqual(1, len(doclist)) - doc2 = doclist[0] - # assert incoming doc is equal to the first sent doc - self.assertEqual(doc1, doc2) - # delete remote database - db.delete_database() - db.close() - sol1.close() - sol2.close() + return self._test_encrypted_sym_sync(doc_size=length, number_of_docs=1) def test_sync_many_small_files(self): """ Test if Soledad can sync many smallfiles. """ - number_of_docs = 100 - self.startServer() - # instantiate soledad and create a document - sol1 = self._soledad_instance( - # token is verified in test_target.make_token_soledad_app - auth_token='auth-token' - ) - _, doclist = sol1.get_all_docs() - self.assertEqual([], doclist) - # create many small files - for i in range(0, number_of_docs): - sol1.create_doc(json.loads(simple_doc)) - # ensure remote db exists before syncing - db = CouchDatabase.open_database( - urljoin(self._couch_url, 'user-user-uuid'), - create=True, - ensure_ddocs=True) - # sync with server - sol1._server_url = self.getURL() - sol1.sync() - # instantiate soledad with empty db, but with same secrets path - sol2 = self._soledad_instance(prefix='x', auth_token='auth-token') - _, doclist = sol2.get_all_docs() - self.assertEqual([], doclist) - sol2._secrets_path = sol1.secrets_path - sol2._load_secrets() - sol2._set_secret_id(sol1._secret_id) - # sync the new instance - sol2._server_url = self.getURL() - sol2.sync() - _, doclist = sol2.get_all_docs() - self.assertEqual(number_of_docs, len(doclist)) - # assert incoming docs are equal to sent docs - for doc in doclist: - self.assertEqual(sol1.get_doc(doc.doc_id), doc) - # delete remote database - db.delete_database() - db.close() - sol1.close() - sol2.close() - + return self._test_encrypted_sym_sync(doc_size=2, number_of_docs=100) class LockResourceTestCase( CouchDBTestCase, TestCaseWithServer): @@ -552,15 +471,18 @@ class LockResourceTestCase( def make_app_with_state(state): return make_token_soledad_app(state) - make_document_for_test = make_leap_document_for_test + make_document_for_test = make_soledad_document_for_test - sync_target = token_leap_sync_target + sync_target = token_soledad_sync_target def setUp(self): - TestCaseWithServer.setUp(self) + # the order of the following initializations is crucial because of + # dependencies. + # XXX explain better CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") self._couch_url = 'http://localhost:' + str(self.wrapper.port) + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + TestCaseWithServer.setUp(self) # create the databases CouchDatabase.open_database( urljoin(self._couch_url, 'shared'), @@ -570,18 +492,17 @@ class LockResourceTestCase( urljoin(self._couch_url, 'tokens'), create=True, ensure_ddocs=True) - self._state = CouchServerState( - self._couch_url, 'shared', 'tokens') + self._state = CouchServerState(self._couch_url) def tearDown(self): - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) # delete remote database db = CouchDatabase.open_database( urljoin(self._couch_url, 'shared'), create=True, ensure_ddocs=True) db.delete_database() + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) def test__try_obtain_filesystem_lock(self): responder = mock.Mock() diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index 11e43423..1cd74dad 100644 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -20,46 +20,43 @@ Tests for general Soledad functionality. import os from mock import Mock - -from leap.common.events import events_pb2 as proto -from leap.soledad.common.tests import ( +from leap.common.events import catalog +from leap.soledad.common.tests.util import ( BaseSoledadTest, ADDRESS, ) from leap import soledad from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.crypto import WrongMac -from leap.soledad.client import Soledad, PassphraseTooShort -from leap.soledad.client.crypto import SoledadCrypto +from leap.soledad.common.crypto import WrongMacError +from leap.soledad.client import Soledad +from leap.soledad.client.adbapi import U1DBConnectionPool +from leap.soledad.client.secrets import PassphraseTooShort from leap.soledad.client.shared_db import SoledadSharedDatabase -from leap.soledad.client.target import SoledadSyncTarget class AuxMethodsTestCase(BaseSoledadTest): def test__init_dirs(self): sol = self._soledad_instance(prefix='_init_dirs') - sol._init_dirs() local_db_dir = os.path.dirname(sol.local_db_path) - secrets_path = os.path.dirname(sol.secrets_path) + secrets_path = os.path.dirname(sol.secrets.secrets_path) self.assertTrue(os.path.isdir(local_db_dir)) self.assertTrue(os.path.isdir(secrets_path)) - sol.close() - def test__init_db(self): - sol = self._soledad_instance() - sol._init_dirs() - sol._crypto = SoledadCrypto(sol) - #self._soledad._gpg.import_keys(PUBLIC_KEY) - if not sol._has_secret(): - sol._gen_secret() - sol._load_secrets() - sol._init_db() - from leap.soledad.client.sqlcipher import SQLCipherDatabase - self.assertIsInstance(sol._db, SQLCipherDatabase) + def _close_soledad(results): + sol.close() + + d = sol.create_doc({}) + d.addCallback(_close_soledad) + return d + + def test__init_u1db_sqlcipher_backend(self): + sol = self._soledad_instance(prefix='_init_db') + self.assertIsInstance(sol._dbpool, U1DBConnectionPool) + self.assertTrue(os.path.isfile(sol.local_db_path)) sol.close() - def test__init_config_defaults(self): + def test__init_config_with_defaults(self): """ Test if configuration defaults point to the correct place. """ @@ -69,17 +66,15 @@ class AuxMethodsTestCase(BaseSoledadTest): def __init__(self): pass - # instantiate without initializing so we just test _init_config() + # instantiate without initializing so we just test + # _init_config_with_defaults() sol = SoledadMock() - Soledad._init_config(sol, None, None, '') - # assert value of secrets_path - self.assertEquals( - os.path.join( - sol.DEFAULT_PREFIX, Soledad.STORAGE_SECRETS_FILE_NAME), - sol.secrets_path) + sol._passphrase = u'' + sol._server_url = '' + sol._init_config_with_defaults() # assert value of local_db_path self.assertEquals( - os.path.join(sol.DEFAULT_PREFIX, 'soledad.u1db'), + os.path.join(sol.default_prefix, 'soledad.u1db'), sol.local_db_path) def test__init_config_from_params(self): @@ -95,43 +90,56 @@ class AuxMethodsTestCase(BaseSoledadTest): cert_file=None) self.assertEqual( os.path.join(self.tempdir, 'value_3'), - sol.secrets_path) + sol.secrets.secrets_path) self.assertEqual( os.path.join(self.tempdir, 'value_2'), sol.local_db_path) - self.assertEqual('value_1', sol.server_url) + self.assertEqual('value_1', sol._server_url) sol.close() def test_change_passphrase(self): """ Test if passphrase can be changed. """ + prefix = '_change_passphrase' sol = self._soledad_instance( 'leap@leap.se', passphrase=u'123', - prefix=self.rand_prefix, + prefix=prefix, ) - doc = sol.create_doc({'simple': 'doc'}) - doc_id = doc.doc_id - # change the passphrase - sol.change_passphrase(u'654321') - sol.close() - - self.assertRaises( - WrongMac, - self._soledad_instance, 'leap@leap.se', - passphrase=u'123', - prefix=self.rand_prefix) - - # use new passphrase and retrieve doc - sol2 = self._soledad_instance( - 'leap@leap.se', - passphrase=u'654321', - prefix=self.rand_prefix) - doc2 = sol2.get_doc(doc_id) - self.assertEqual(doc, doc2) - sol2.close() + def _change_passphrase(doc1): + self._doc1 = doc1 + sol.change_passphrase(u'654321') + sol.close() + + def _assert_wrong_password_raises(results): + self.assertRaises( + WrongMacError, + self._soledad_instance, 'leap@leap.se', + passphrase=u'123', + prefix=prefix) + + def _instantiate_with_new_passphrase(results): + sol2 = self._soledad_instance( + 'leap@leap.se', + passphrase=u'654321', + prefix=prefix) + self._sol2 = sol2 + return sol2.get_doc(self._doc1.doc_id) + + def _assert_docs_are_equal(doc2): + self.assertEqual(self._doc1, doc2) + self._sol2.close() + + d = sol.create_doc({'simple': 'doc'}) + d.addCallback(_change_passphrase) + d.addCallback(_assert_wrong_password_raises) + d.addCallback(_instantiate_with_new_passphrase) + d.addCallback(_assert_docs_are_equal) + d.addCallback(lambda _: sol.close()) + + return d def test_change_passphrase_with_short_passphrase_raises(self): """ @@ -152,7 +160,7 @@ class AuxMethodsTestCase(BaseSoledadTest): Assert passphrase getter works fine. """ sol = self._soledad_instance() - self.assertEqual('123', sol.passphrase) + self.assertEqual('123', sol._passphrase) sol.close() @@ -174,10 +182,10 @@ class SoledadSharedDBTestCase(BaseSoledadTest): """ Ensure the shared db is queried with the correct doc_id. """ - doc_id = self._soledad._shared_db_doc_id() - self._soledad._get_secrets_from_shared_db() + doc_id = self._soledad.secrets._shared_db_doc_id() + self._soledad.secrets._get_secrets_from_shared_db() self.assertTrue( - self._soledad._shared_db().get_doc.assert_called_with( + self._soledad.shared_db.get_doc.assert_called_with( doc_id) is None, 'Wrong doc_id when fetching recovery document.') @@ -185,14 +193,14 @@ class SoledadSharedDBTestCase(BaseSoledadTest): """ Ensure recovery document is put into shared recover db. """ - doc_id = self._soledad._shared_db_doc_id() - self._soledad._put_secrets_in_shared_db() + doc_id = self._soledad.secrets._shared_db_doc_id() + self._soledad.secrets._put_secrets_in_shared_db() self.assertTrue( - self._soledad._shared_db().get_doc.assert_called_with( + self._soledad.shared_db.get_doc.assert_called_with( doc_id) is None, 'Wrong doc_id when fetching recovery document.') self.assertTrue( - self._soledad._shared_db.put_doc.assert_called_with( + self._soledad.shared_db.put_doc.assert_called_with( self._doc_put) is None, 'Wrong document when putting recovery document.') self.assertTrue( @@ -210,6 +218,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): def setUp(self): # mock signaling soledad.client.signal = Mock() + soledad.client.secrets.events.emit = Mock() # run parent's setUp BaseSoledadTest.setUp(self) @@ -231,63 +240,63 @@ class SoledadSignalingTestCase(BaseSoledadTest): - downloading keys / done downloading keys. - uploading keys / done uploading keys. """ - soledad.client.signal.reset_mock() + soledad.client.secrets.events.emit.reset_mock() # get a fresh instance so it emits all bootstrap signals sol = self._soledad_instance( secrets_path='alternative_stage3.json', local_db_path='alternative_stage3.u1db') # reverse call order so we can verify in the order the signals were # expected - soledad.client.signal.mock_calls.reverse() - soledad.client.signal.call_args = \ - soledad.client.signal.call_args_list[0] - soledad.client.signal.call_args_list.reverse() + soledad.client.secrets.events.emit.mock_calls.reverse() + soledad.client.secrets.events.emit.call_args = \ + soledad.client.secrets.events.emit.call_args_list[0] + soledad.client.secrets.events.emit.call_args_list.reverse() # downloading keys signals - soledad.client.signal.assert_called_with( - proto.SOLEDAD_DOWNLOADING_KEYS, + soledad.client.secrets.events.emit.assert_called_with( + catalog.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( - proto.SOLEDAD_DONE_DOWNLOADING_KEYS, + self._pop_mock_call(soledad.client.secrets.events.emit) + soledad.client.secrets.events.emit.assert_called_with( + catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) # creating keys signals - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( - proto.SOLEDAD_CREATING_KEYS, + self._pop_mock_call(soledad.client.secrets.events.emit) + soledad.client.secrets.events.emit.assert_called_with( + catalog.SOLEDAD_CREATING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( - proto.SOLEDAD_DONE_CREATING_KEYS, + self._pop_mock_call(soledad.client.secrets.events.emit) + soledad.client.secrets.events.emit.assert_called_with( + catalog.SOLEDAD_DONE_CREATING_KEYS, ADDRESS, ) # downloading once more (inside _put_keys_in_shared_db) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( - proto.SOLEDAD_DOWNLOADING_KEYS, + self._pop_mock_call(soledad.client.secrets.events.emit) + soledad.client.secrets.events.emit.assert_called_with( + catalog.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( - proto.SOLEDAD_DONE_DOWNLOADING_KEYS, + self._pop_mock_call(soledad.client.secrets.events.emit) + soledad.client.secrets.events.emit.assert_called_with( + catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) # uploading keys signals - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( - proto.SOLEDAD_UPLOADING_KEYS, + self._pop_mock_call(soledad.client.secrets.events.emit) + soledad.client.secrets.events.emit.assert_called_with( + catalog.SOLEDAD_UPLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( - proto.SOLEDAD_DONE_UPLOADING_KEYS, + self._pop_mock_call(soledad.client.secrets.events.emit) + soledad.client.secrets.events.emit.assert_called_with( + catalog.SOLEDAD_DONE_UPLOADING_KEYS, ADDRESS, ) # assert db was locked and unlocked - sol._shared_db.lock.assert_called_with() - sol._shared_db.unlock.assert_called_with('atoken') + sol.shared_db.lock.assert_called_with() + sol.shared_db.unlock.assert_called_with('atoken') sol.close() def test_stage2_bootstrap_signals(self): @@ -298,41 +307,31 @@ class SoledadSignalingTestCase(BaseSoledadTest): # get existing instance so we have access to keys sol = self._soledad_instance() # create a document with secrets - doc = SoledadDocument(doc_id=sol._shared_db_doc_id()) - doc.content = sol.export_recovery_document() - - class Stage2MockSharedDB(object): - - get_doc = Mock(return_value=doc) - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock() - - def __call__(self): - return self - + doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id()) + doc.content = sol.secrets._export_recovery_document() sol.close() # reset mock - soledad.client.signal.reset_mock() + soledad.client.secrets.events.emit.reset_mock() # get a fresh instance so it emits all bootstrap signals + shared_db = self.get_default_shared_mock(get_doc_return_value=doc) sol = self._soledad_instance( secrets_path='alternative_stage2.json', local_db_path='alternative_stage2.u1db', - shared_db_class=Stage2MockSharedDB) + shared_db_class=shared_db) # reverse call order so we can verify in the order the signals were # expected - soledad.client.signal.mock_calls.reverse() - soledad.client.signal.call_args = \ - soledad.client.signal.call_args_list[0] - soledad.client.signal.call_args_list.reverse() + soledad.client.secrets.events.emit.mock_calls.reverse() + soledad.client.secrets.events.emit.call_args = \ + soledad.client.secrets.events.emit.call_args_list[0] + soledad.client.secrets.events.emit.call_args_list.reverse() # assert download keys signals - soledad.client.signal.assert_called_with( - proto.SOLEDAD_DOWNLOADING_KEYS, + soledad.client.secrets.events.emit.assert_called_with( + catalog.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( - proto.SOLEDAD_DONE_DOWNLOADING_KEYS, + self._pop_mock_call(soledad.client.secrets.events.emit) + soledad.client.secrets.events.emit.assert_called_with( + catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) sol.close() @@ -356,33 +355,17 @@ class SoledadSignalingTestCase(BaseSoledadTest): sol = self._soledad_instance() # mock the actual db sync so soledad does not try to connect to the # server - sol._db.sync = Mock() - # do the sync - sol.sync() - # assert the signal has been emitted - soledad.client.signal.assert_called_with( - proto.SOLEDAD_DONE_DATA_SYNC, - ADDRESS, - ) - sol.close() - - def test_need_sync_signals(self): - """ - Test Soledad emits SOLEDAD_CREATING_KEYS signal. - """ - soledad.client.signal.reset_mock() - sol = self._soledad_instance() - # mock the sync target - old_get_sync_info = SoledadSyncTarget.get_sync_info - SoledadSyncTarget.get_sync_info = Mock(return_value=[0, 0, 0, 0, 2]) - # mock our generation so soledad thinks there's new data to sync - sol._db._get_generation = Mock(return_value=1) - # check for new data to sync - sol.need_sync('http://provider/userdb') - # assert the signal has been emitted - soledad.client.signal.assert_called_with( - proto.SOLEDAD_NEW_DATA_TO_SYNC, - ADDRESS, - ) - SoledadSyncTarget.get_sync_info = old_get_sync_info - sol.close() + sol._dbsyncer.sync = Mock() + + def _assert_done_data_sync_signal_emitted(results): + # assert the signal has been emitted + soledad.client.signal.assert_called_with( + catalog.SOLEDAD_DONE_DATA_SYNC, + ADDRESS, + ) + sol.close() + + # do the sync and assert signal was emitted + d = sol.sync() + d.addCallback(_assert_done_data_sync_signal_emitted) + return d diff --git a/common/src/leap/soledad/common/tests/test_soledad_app.py b/common/src/leap/soledad/common/tests/test_soledad_app.py new file mode 100644 index 00000000..6efae1d6 --- /dev/null +++ b/common/src/leap/soledad/common/tests/test_soledad_app.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# test_soledad_app.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +Test ObjectStore and Couch backend bits. +""" + + +from testscenarios import TestWithScenarios + +from leap.soledad.common.tests.util import BaseSoledadTest +from leap.soledad.common.tests.util import make_soledad_document_for_test +from leap.soledad.common.tests.util import make_soledad_app +from leap.soledad.common.tests.util import make_token_soledad_app +from leap.soledad.common.tests.util import make_token_http_database_for_test +from leap.soledad.common.tests.util import copy_token_http_database_for_test +from leap.soledad.common.tests.u1db_tests import test_backends + + +#----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_backends`. +#----------------------------------------------------------------------------- + +LEAP_SCENARIOS = [ + ('http', { + 'make_database_for_test': test_backends.make_http_database_for_test, + 'copy_database_for_test': test_backends.copy_http_database_for_test, + 'make_document_for_test': make_soledad_document_for_test, + 'make_app_with_state': make_soledad_app}), +] + + +class SoledadTests( + TestWithScenarios, test_backends.AllDatabaseTests, BaseSoledadTest): + + scenarios = LEAP_SCENARIOS + [ + ('token_http', {'make_database_for_test': + make_token_http_database_for_test, + 'copy_database_for_test': + copy_token_http_database_for_test, + 'make_document_for_test': make_soledad_document_for_test, + 'make_app_with_state': make_token_soledad_app, + }) + ] diff --git a/common/src/leap/soledad/common/tests/test_soledad_doc.py b/common/src/leap/soledad/common/tests/test_soledad_doc.py index 0952de6d..4a67f80a 100644 --- a/common/src/leap/soledad/common/tests/test_soledad_doc.py +++ b/common/src/leap/soledad/common/tests/test_soledad_doc.py @@ -17,28 +17,30 @@ """ Test Leap backend bits: soledad docs """ -from leap.soledad.common.tests import BaseSoledadTest +from testscenarios import TestWithScenarios + from leap.soledad.common.tests.u1db_tests import test_document -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests import test_sync_target as st +from leap.soledad.common.tests.util import BaseSoledadTest +from leap.soledad.common.tests.util import make_soledad_document_for_test + #----------------------------------------------------------------------------- # The following tests come from `u1db.tests.test_document`. #----------------------------------------------------------------------------- - -class TestSoledadDocument(test_document.TestDocument, BaseSoledadTest): +class TestSoledadDocument( + TestWithScenarios, + test_document.TestDocument, BaseSoledadTest): scenarios = ([( 'leap', { - 'make_document_for_test': st.make_leap_document_for_test})]) + 'make_document_for_test': make_soledad_document_for_test})]) -class TestSoledadPyDocument(test_document.TestPyDocument, BaseSoledadTest): +class TestSoledadPyDocument( + TestWithScenarios, + test_document.TestPyDocument, BaseSoledadTest): scenarios = ([( 'leap', { - 'make_document_for_test': st.make_leap_document_for_test})]) - - -load_tests = tests.load_with_scenarios + 'make_document_for_test': make_soledad_document_for_test})]) diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py index 595966ec..ceb095b8 100644 --- a/common/src/leap/soledad/common/tests/test_sqlcipher.py +++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py @@ -19,53 +19,49 @@ Test sqlcipher backend internals. """ import os import time -import simplejson as json import threading - +import tempfile +import shutil from pysqlcipher import dbapi2 -from StringIO import StringIO -from urlparse import urljoin - +from testscenarios import TestWithScenarios # u1db stuff. from u1db import ( errors, query_parser, - sync, - vectorclock, ) from u1db.backends.sqlite_backend import SQLitePartialExpandDatabase # soledad stuff. +from leap.soledad.common import soledad_assert from leap.soledad.common.document import SoledadDocument from leap.soledad.client.sqlcipher import ( SQLCipherDatabase, + SQLCipherOptions, DatabaseIsNotEncrypted, - open as u1db_open, ) -from leap.soledad.client.target import SoledadSyncTarget -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.client.crypto import decrypt_doc_dict # u1db tests stuff. -from leap.common.testing.basetest import BaseLeapTest -from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest +from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_sqlite_backend from leap.soledad.common.tests.u1db_tests import test_backends from leap.soledad.common.tests.u1db_tests import test_open -from leap.soledad.common.tests.u1db_tests import test_sync from leap.soledad.common.tests.util import ( make_sqlcipher_database_for_test, copy_sqlcipher_database_for_test, - make_soledad_app, - SoledadWithCouchServerMixin, PASSWORD, + BaseSoledadTest, ) +def sqlcipher_open(path, passphrase, create=True, document_factory=None): + return SQLCipherDatabase( + SQLCipherOptions(path, passphrase, create=create)) + + #----------------------------------------------------------------------------- # The following tests come from `u1db.tests.test_common_backend`. #----------------------------------------------------------------------------- @@ -73,12 +69,13 @@ from leap.soledad.common.tests.util import ( class TestSQLCipherBackendImpl(tests.TestCase): def test__allocate_doc_id(self): - db = SQLCipherDatabase(':memory:', PASSWORD) + db = sqlcipher_open(':memory:', PASSWORD) doc_id1 = db._allocate_doc_id() self.assertTrue(doc_id1.startswith('D-')) self.assertEqual(34, len(doc_id1)) int(doc_id1[len('D-'):], 16) self.assertNotEqual(doc_id1, db._allocate_doc_id()) + db.close() #----------------------------------------------------------------------------- @@ -96,43 +93,46 @@ SQLCIPHER_SCENARIOS = [ ] -class SQLCipherTests(test_backends.AllDatabaseTests): +class SQLCipherTests(TestWithScenarios, test_backends.AllDatabaseTests): scenarios = SQLCIPHER_SCENARIOS -class SQLCipherDatabaseTests(test_backends.LocalDatabaseTests): +class SQLCipherDatabaseTests(TestWithScenarios, test_backends.LocalDatabaseTests): scenarios = SQLCIPHER_SCENARIOS class SQLCipherValidateGenNTransIdTests( + TestWithScenarios, test_backends.LocalDatabaseValidateGenNTransIdTests): scenarios = SQLCIPHER_SCENARIOS class SQLCipherValidateSourceGenTests( + TestWithScenarios, test_backends.LocalDatabaseValidateSourceGenTests): scenarios = SQLCIPHER_SCENARIOS class SQLCipherWithConflictsTests( + TestWithScenarios, test_backends.LocalDatabaseWithConflictsTests): scenarios = SQLCIPHER_SCENARIOS -class SQLCipherIndexTests(test_backends.DatabaseIndexTests): +class SQLCipherIndexTests( + TestWithScenarios, test_backends.DatabaseIndexTests): scenarios = SQLCIPHER_SCENARIOS -load_tests = tests.load_with_scenarios - - #----------------------------------------------------------------------------- # The following tests come from `u1db.tests.test_sqlite_backend`. #----------------------------------------------------------------------------- -class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase): +class TestSQLCipherDatabase(TestWithScenarios, test_sqlite_backend.TestSQLiteDatabase): def test_atomic_initialize(self): + # This test was modified to ensure that db2.close() is called within + # the thread that created the database. tmpdir = self.createTempDir() dbname = os.path.join(tmpdir, 'atomic.db') @@ -144,7 +144,9 @@ class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase): def __init__(self, dbname, ntry): self._try = ntry self._is_initialized_invocations = 0 - SQLCipherDatabase.__init__(self, dbname, PASSWORD) + SQLCipherDatabase.__init__( + self, + SQLCipherOptions(dbname, PASSWORD)) def _is_initialized(self, c): res = \ @@ -157,23 +159,25 @@ class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase): time.sleep(0.05) return res - outcome2 = [] + class SecondTry(threading.Thread): + + outcome2 = [] - def second_try(): - try: - db2 = SQLCipherDatabaseTesting(dbname, 2) - except Exception, e: - outcome2.append(e) - else: - outcome2.append(db2) + def run(self): + try: + db2 = SQLCipherDatabaseTesting(dbname, 2) + except Exception, e: + SecondTry.outcome2.append(e) + else: + SecondTry.outcome2.append(db2) - t2 = threading.Thread(target=second_try) + t2 = SecondTry() db1 = SQLCipherDatabaseTesting(dbname, 1) t2.join() - self.assertIsInstance(outcome2[0], SQLCipherDatabaseTesting) - db2 = outcome2[0] - self.assertTrue(db2._is_initialized(db1._get_sqlite_handle().cursor())) + self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting) + self.assertTrue(db1._is_initialized(db1._get_sqlite_handle().cursor())) + db1.close() class TestAlternativeDocument(SoledadDocument): @@ -189,23 +193,23 @@ class TestSQLCipherPartialExpandDatabase( def setUp(self): test_sqlite_backend.TestSQLitePartialExpandDatabase.setUp(self) - self.db = SQLCipherDatabase(':memory:', PASSWORD) - self.db._set_replica_uid('test') + self.db = sqlcipher_open(':memory:', PASSWORD) + + def tearDown(self): + self.db.close() + test_sqlite_backend.TestSQLitePartialExpandDatabase.tearDown(self) def test_default_replica_uid(self): - self.db = SQLCipherDatabase(':memory:', PASSWORD) self.assertIsNot(None, self.db._replica_uid) self.assertEqual(32, len(self.db._replica_uid)) int(self.db._replica_uid, 16) def test__parse_index(self): - self.db = SQLCipherDatabase(':memory:', PASSWORD) g = self.db._parse_index_definition('fieldname') self.assertIsInstance(g, query_parser.ExtractField) self.assertEqual(['fieldname'], g.field) def test__update_indexes(self): - self.db = SQLCipherDatabase(':memory:', PASSWORD) g = self.db._parse_index_definition('fieldname') c = self.db._get_sqlite_handle().cursor() self.db._update_indexes('doc-id', {'fieldname': 'val'}, @@ -216,7 +220,6 @@ class TestSQLCipherPartialExpandDatabase( def test__set_replica_uid(self): # Start from scratch, so that replica_uid isn't set. - self.db = SQLCipherDatabase(':memory:', PASSWORD) self.assertIsNot(None, self.db._real_replica_uid) self.assertIsNot(None, self.db._replica_uid) self.db._set_replica_uid('foo') @@ -229,98 +232,67 @@ class TestSQLCipherPartialExpandDatabase( self.assertEqual('foo', self.db._replica_uid) def test__open_database(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/test.sqlite' - SQLCipherDatabase(path, PASSWORD) - db2 = SQLCipherDatabase._open_database(path, PASSWORD) - self.assertIsInstance(db2, SQLCipherDatabase) + # SQLCipherDatabase has no _open_database() method, so we just pass + # (and test for the same funcionality on test_open_database_existing() + # below). + pass def test__open_database_with_factory(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/test.sqlite' - SQLCipherDatabase(path, PASSWORD) - db2 = SQLCipherDatabase._open_database( - path, PASSWORD, - document_factory=TestAlternativeDocument) - doc = db2.create_doc({}) - self.assertTrue(isinstance(doc, SoledadDocument)) + # SQLCipherDatabase has no _open_database() method. + pass def test__open_database_non_existent(self): temp_dir = self.createTempDir(prefix='u1db-test-') path = temp_dir + '/non-existent.sqlite' self.assertRaises(errors.DatabaseDoesNotExist, - SQLCipherDatabase._open_database, - path, PASSWORD) + sqlcipher_open, + path, PASSWORD, create=False) def test__open_database_during_init(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/initialised.db' - db = SQLCipherDatabase.__new__( - SQLCipherDatabase) - db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed - db._syncers = {} - c = db._db_handle.cursor() - c.execute('PRAGMA key="%s"' % PASSWORD) - self.addCleanup(db.close) - observed = [] - - class SQLiteDatabaseTesting(SQLCipherDatabase): - WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL = 0.1 - - @classmethod - def _which_index_storage(cls, c): - res = SQLCipherDatabase._which_index_storage(c) - db._ensure_schema() # init db - observed.append(res[0]) - return res - - db2 = SQLiteDatabaseTesting._open_database(path, PASSWORD) - self.addCleanup(db2.close) - self.assertIsInstance(db2, SQLCipherDatabase) - self.assertEqual( - [None, - SQLCipherDatabase._index_storage_value], - observed) + # The purpose of this test is to ensure that _open_database() parallel + # db initialization behaviour is correct. As SQLCipherDatabase does + # not have an _open_database() method, we just do not implement this + # test. + pass def test__open_database_invalid(self): - class SQLiteDatabaseTesting(SQLCipherDatabase): - WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL = 0.1 + # This test was modified to ensure that an empty database file will + # raise a DatabaseIsNotEncrypted exception instead of a + # dbapi2.OperationalError exception. temp_dir = self.createTempDir(prefix='u1db-test-') path1 = temp_dir + '/invalid1.db' with open(path1, 'wb') as f: f.write("") - self.assertRaises(dbapi2.OperationalError, - SQLiteDatabaseTesting._open_database, path1, + self.assertRaises(DatabaseIsNotEncrypted, + sqlcipher_open, path1, PASSWORD) with open(path1, 'wb') as f: f.write("invalid") self.assertRaises(dbapi2.DatabaseError, - SQLiteDatabaseTesting._open_database, path1, + sqlcipher_open, path1, PASSWORD) def test_open_database_existing(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/existing.sqlite' - SQLCipherDatabase(path, PASSWORD) - db2 = SQLCipherDatabase.open_database(path, PASSWORD, create=False) - self.assertIsInstance(db2, SQLCipherDatabase) + # In the context of SQLCipherDatabase, where no _open_database() + # method exists and thus there's no call to _which_index_storage(), + # this test tests for the same functionality as + # test_open_database_create() below. So, we just pass. + pass def test_open_database_with_factory(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/existing.sqlite' - SQLCipherDatabase(path, PASSWORD) - db2 = SQLCipherDatabase.open_database( - path, PASSWORD, create=False, - document_factory=TestAlternativeDocument) - doc = db2.create_doc({}) - self.assertTrue(isinstance(doc, SoledadDocument)) + # SQLCipherDatabase's constructor has no factory parameter. + pass def test_open_database_create(self): + # SQLCipherDatabas has no open_database() method, so we just test for + # the actual database constructor effects. temp_dir = self.createTempDir(prefix='u1db-test-') path = temp_dir + '/new.sqlite' - SQLCipherDatabase.open_database(path, PASSWORD, create=True) - db2 = SQLCipherDatabase.open_database(path, PASSWORD, create=False) + db1 = sqlcipher_open(path, PASSWORD, create=True) + db2 = sqlcipher_open(path, PASSWORD, create=False) self.assertIsInstance(db2, SQLCipherDatabase) + db1.close() + db2.close() def test_create_database_initializes_schema(self): # This test had to be cloned because our implementation of SQLCipher @@ -331,7 +303,8 @@ class TestSQLCipherPartialExpandDatabase( c = raw_db.cursor() c.execute("SELECT * FROM u1db_config") config = dict([(r[0], r[1]) for r in c.fetchall()]) - self.assertEqual({'sql_schema': '0', 'replica_uid': 'test', + replica_uid = self.db._replica_uid + self.assertEqual({'sql_schema': '0', 'replica_uid': replica_uid, 'index_storage': 'expand referenced encrypted'}, config) @@ -353,394 +326,52 @@ class TestSQLCipherPartialExpandDatabase( # The following tests come from `u1db.tests.test_open`. #----------------------------------------------------------------------------- + class SQLCipherOpen(test_open.TestU1DBOpen): def test_open_no_create(self): self.assertRaises(errors.DatabaseDoesNotExist, - u1db_open, self.db_path, - password=PASSWORD, + sqlcipher_open, self.db_path, + PASSWORD, create=False) self.assertFalse(os.path.exists(self.db_path)) def test_open_create(self): - db = u1db_open(self.db_path, password=PASSWORD, create=True) + db = sqlcipher_open(self.db_path, PASSWORD, create=True) self.addCleanup(db.close) self.assertTrue(os.path.exists(self.db_path)) self.assertIsInstance(db, SQLCipherDatabase) def test_open_with_factory(self): - db = u1db_open(self.db_path, password=PASSWORD, create=True, + db = sqlcipher_open(self.db_path, PASSWORD, create=True, document_factory=TestAlternativeDocument) self.addCleanup(db.close) doc = db.create_doc({}) self.assertTrue(isinstance(doc, SoledadDocument)) def test_open_existing(self): - db = SQLCipherDatabase(self.db_path, PASSWORD) + db = sqlcipher_open(self.db_path, PASSWORD) self.addCleanup(db.close) doc = db.create_doc_from_json(tests.simple_doc) # Even though create=True, we shouldn't wipe the db - db2 = u1db_open(self.db_path, password=PASSWORD, create=True) + db2 = sqlcipher_open(self.db_path, PASSWORD, create=True) self.addCleanup(db2.close) doc2 = db2.get_doc(doc.doc_id) self.assertEqual(doc, doc2) def test_open_existing_no_create(self): - db = SQLCipherDatabase(self.db_path, PASSWORD) + db = sqlcipher_open(self.db_path, PASSWORD) self.addCleanup(db.close) - db2 = u1db_open(self.db_path, password=PASSWORD, create=False) + db2 = sqlcipher_open(self.db_path, PASSWORD, create=False) self.addCleanup(db2.close) self.assertIsInstance(db2, SQLCipherDatabase) #----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -#----------------------------------------------------------------------------- - -sync_scenarios = [] -for name, scenario in SQLCIPHER_SCENARIOS: - scenario = dict(scenario) - scenario['do_sync'] = test_sync.sync_via_synchronizer - sync_scenarios.append((name, scenario)) - scenario = dict(scenario) - - -def sync_via_synchronizer_and_leap(test, db_source, db_target, - trace_hook=None, trace_hook_shallow=None): - if trace_hook: - test.skipTest("full trace hook unsupported over http") - path = test._http_at[db_target] - target = SoledadSyncTarget.connect( - test.getURL(path), test._soledad._crypto) - target.set_token_credentials('user-uuid', 'auth-token') - if trace_hook_shallow: - target._set_trace_hook_shallow(trace_hook_shallow) - return sync.Synchronizer(db_source, target).sync() - - -sync_scenarios.append(('pyleap', { - 'make_database_for_test': test_sync.make_database_for_http_test, - 'copy_database_for_test': test_sync.copy_database_for_http_test, - 'make_document_for_test': make_document_for_test, - 'make_app_with_state': tests.test_remote_sync_target.make_http_app, - 'do_sync': test_sync.sync_via_synchronizer, -})) - - -class SQLCipherDatabaseSyncTests( - test_sync.DatabaseSyncTests, BaseSoledadTest): - """ - Test for succesfull sync between SQLCipher and LeapBackend. - - Some of the tests in this class had to be adapted because the remote - backend always receive encrypted content, and so it can not rely on - document's content comparison to try to autoresolve conflicts. - """ - - scenarios = sync_scenarios - - def setUp(self): - test_sync.DatabaseSyncTests.setUp(self) - - def tearDown(self): - test_sync.DatabaseSyncTests.tearDown(self) - - def test_sync_autoresolves(self): - """ - Test for sync autoresolve remote. - - This test was adapted because the remote database receives encrypted - content and so it can't compare documents contents to autoresolve. - """ - # The remote database can't autoresolve conflicts based on magic - # content convergence, so we modify this test to leave the possibility - # of the remode document ending up in conflicted state. - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') - rev1 = doc1.rev - doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc') - rev2 = doc2.rev - self.sync(self.db1, self.db2) - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - # if remote content is in conflicted state, then document revisions - # will be different. - #self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) - v = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) - - def test_sync_autoresolves_moar(self): - """ - Test for sync autoresolve local. - - This test was adapted to decrypt remote content before assert. - """ - # here we test that when a database that has a conflicted document is - # the source of a sync, and the target database has a revision of the - # conflicted document that is newer than the source database's, and - # that target's database's document's content is the same as the - # source's document's conflict's, the source's document's conflict gets - # autoresolved, and the source's document's revision bumped. - # - # idea is as follows: - # A B - # a1 - - # `-------> - # a1 a1 - # v v - # a2 a1b1 - # `-------> - # a1b1+a2 a1b1 - # v - # a1b1+a2 a1b2 (a1b2 has same content as a2) - # `-------> - # a3b2 a1b2 (autoresolved) - # `-------> - # a3b2 a3b2 - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db2) - # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertTrue(doc.has_conflicts) - # set db2 to have a doc of {} (same as db1 before the conflict) - doc = self.db2.get_doc('doc') - doc.set_json('{}') - self.db2.put_doc(doc) - rev2 = doc.rev - # sync it across - self.sync(self.db1, self.db2) - # tadaa! - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - vec1 = vectorclock.VectorClockRev(rev1) - vec2 = vectorclock.VectorClockRev(rev2) - vec3 = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(vec3.is_newer(vec1)) - self.assertTrue(vec3.is_newer(vec2)) - # because the conflict is on the source, sync it another time - self.sync(self.db1, self.db2) - # make sure db2 now has the exact same thing - doc1 = self.db1.get_doc('doc') - self.assertGetEncryptedDoc( - self.db2, - doc1.doc_id, doc1.rev, doc1.get_json(), False) - - def test_sync_autoresolves_moar_backwards(self): - # here we would test that when a database that has a conflicted - # document is the target of a sync, and the source database has a - # revision of the conflicted document that is newer than the target - # database's, and that source's database's document's content is the - # same as the target's document's conflict's, the target's document's - # conflict gets autoresolved, and the document's revision bumped. - # - # Despite that, in Soledad we suppose that the server never syncs, so - # it never has conflicted documents. Also, if it had, convergence - # would not be possible by checking document's contents because they - # would be encrypted in server. - # - # Therefore we suppress this test. - pass - - def test_sync_autoresolves_moar_backwards_three(self): - # here we would test that when a database that has a conflicted - # document is the target of a sync, and the source database has a - # revision of the conflicted document that is newer than the target - # database's, and that source's database's document's content is the - # same as the target's document's conflict's, the target's document's - # conflict gets autoresolved, and the document's revision bumped. - # - # We use the same reasoning from the last test to suppress this one. - pass - - def test_sync_propagates_resolution(self): - """ - Test if synchronization propagates resolution. - - This test was adapted to decrypt remote content before assert. - """ - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - db3 = self.create_database('test3', 'both') - self.sync(self.db2, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db2._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.sync(db3, self.db1) - # update on 2 - doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') - self.db2.put_doc(doc2) - self.sync(self.db2, db3) - self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) - # update on 1 - doc1.set_json('{"a": 3}') - self.db1.put_doc(doc1) - # conflicts - self.sync(self.db2, self.db1) - self.sync(db3, self.db1) - self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) - self.assertTrue(db3.get_doc('the-doc').has_conflicts) - # resolve - conflicts = self.db2.get_doc_conflicts('the-doc') - doc4 = self.make_document('the-doc', None, '{"a": 4}') - revs = [doc.rev for doc in conflicts] - self.db2.resolve_doc(doc4, revs) - doc2 = self.db2.get_doc('the-doc') - self.assertEqual(doc4.get_json(), doc2.get_json()) - self.assertFalse(doc2.has_conflicts) - self.sync(self.db2, db3) - doc3 = db3.get_doc('the-doc') - if ENC_SCHEME_KEY in doc3.content: - _crypto = self._soledad._crypto - key = _crypto.doc_passphrase(doc3.doc_id) - secret = _crypto.secret - doc3.set_json(decrypt_doc_dict( - doc3.content, - doc3.doc_id, doc3.rev, key, secret)) - self.assertEqual(doc4.get_json(), doc3.get_json()) - self.assertFalse(doc3.has_conflicts) - - def test_sync_puts_changes(self): - """ - Test if sync puts changes in remote replica. - - This test was adapted to decrypt remote content before assert. - """ - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(tests.simple_doc) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertGetEncryptedDoc( - self.db2, doc.doc_id, doc.rev, tests.simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - -def _make_local_db_and_token_http_target(test, path='test'): - test.startServer() - db = test.request_state._create_database(os.path.basename(path)) - st = SoledadSyncTarget.connect( - test.getURL(path), crypto=test._soledad._crypto) - st.set_token_credentials('user-uuid', 'auth-token') - return db, st - - -target_scenarios = [ - ('leap', { - 'create_db_and_target': _make_local_db_and_token_http_target, -# 'make_app_with_state': tests.test_remote_sync_target.make_http_app, - 'make_app_with_state': make_soledad_app, - 'do_sync': test_sync.sync_via_synchronizer}), -] - - -class SQLCipherSyncTargetTests( - SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests): - - scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS, - target_scenarios)) - - whitebox = False - - def setUp(self): - self.main_test_class = test_sync.DatabaseSyncTargetTests - SoledadWithCouchServerMixin.setUp(self) - - def test_sync_exchange(self): - """ - Modified to account for possibly receiving encrypted documents from - sever-side. - """ - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, - 'T-sid')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertTransactionLog(['doc-id'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, last_trans_id)) - self.assertEqual(10, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_push_many(self): - """ - Modified to account for possibly receiving encrypted documents from - sever-side. - """ - docs_by_gen = [ - (self.make_document( - 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), - (self.make_document('doc-id2', 'replica:1', tests.nested_doc), 11, - 'T-2')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, 'doc-id2', 'replica:1', tests.nested_doc, False) - self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(11, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_returns_many_new_docs(self): - """ - Modified to account for JSON serialization differences. - """ - doc = self.db.create_doc_from_json(tests.simple_doc) - doc2 = self.db.create_doc_from_json(tests.nested_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - self.assertEqual(2, new_gen) - self.assertEqual( - [(doc.doc_id, doc.rev, 1), - (doc2.doc_id, doc2.rev, 2)], - [c[:2] + c[3:4] for c in self.other_changes]) - self.assertEqual( - json.dumps(tests.simple_doc), - json.dumps(self.other_changes[0][2])) - self.assertEqual( - json.loads(tests.nested_doc), - json.loads(self.other_changes[1][2])) - if self.whitebox: - self.assertEqual( - self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': - [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) - - -#----------------------------------------------------------------------------- # Tests for actual encryption of the database #----------------------------------------------------------------------------- -class SQLCipherEncryptionTest(BaseLeapTest): +class SQLCipherEncryptionTest(BaseSoledadTest): """ Tests to guarantee SQLCipher is indeed encrypting data when storing. """ @@ -751,17 +382,43 @@ class SQLCipherEncryptionTest(BaseLeapTest): os.unlink(dbfile) def setUp(self): + # the following come from BaseLeapTest.setUpClass, because + # twisted.trial doesn't support such class methods for setting up + # test classes. + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + # this is our own stuff self.DB_FILE = os.path.join(self.tempdir, 'test.db') self._delete_dbfiles() def tearDown(self): self._delete_dbfiles() + # the following come from BaseLeapTest.tearDownClass, because + # twisted.trial doesn't support such class methods for tearing down + # test classes. + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check! please do not wipe my home... + # XXX needs to adapt to non-linuces + soledad_assert( + self.tempdir.startswith('/tmp/leap_tests-') or + self.tempdir.startswith('/var/folder'), + "beware! tried to remove a dir which does not " + "live in temporal folder!") + shutil.rmtree(self.tempdir) def test_try_to_open_encrypted_db_with_sqlite_backend(self): """ SQLite backend should not succeed to open SQLCipher databases. """ - db = SQLCipherDatabase(self.DB_FILE, PASSWORD) + db = sqlcipher_open(self.DB_FILE, PASSWORD) doc = db.create_doc_from_json(tests.simple_doc) db.close() try: @@ -774,10 +431,11 @@ class SQLCipherEncryptionTest(BaseLeapTest): # at this point we know that the regular U1DB sqlcipher backend # did not succeed on opening the database, so it was indeed # encrypted. - db = SQLCipherDatabase(self.DB_FILE, PASSWORD) + db = sqlcipher_open(self.DB_FILE, PASSWORD) doc = db.get_doc(doc.doc_id) self.assertEqual(tests.simple_doc, doc.get_json(), 'decrypted content mismatch') + db.close() def test_try_to_open_raw_db_with_sqlcipher_backend(self): """ @@ -790,12 +448,10 @@ class SQLCipherEncryptionTest(BaseLeapTest): try: # trying to open the a non-encrypted database with sqlcipher # backend should raise a DatabaseIsNotEncrypted exception. - SQLCipherDatabase(self.DB_FILE, PASSWORD) + db = sqlcipher_open(self.DB_FILE, PASSWORD) + db.close() raise dbapi2.DatabaseError( "SQLCipher backend should not be able to open non-encrypted " "dbs.") except DatabaseIsNotEncrypted: pass - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py new file mode 100644 index 00000000..83c3449e --- /dev/null +++ b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py @@ -0,0 +1,398 @@ +# -*- coding: utf-8 -*- +# test_sqlcipher.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Test sqlcipher backend sync. +""" + + +import os +import simplejson as json +from u1db import ( + sync, + vectorclock, +) + +from testscenarios import TestWithScenarios + +from leap.soledad.common.crypto import ENC_SCHEME_KEY +from leap.soledad.client.target import SoledadSyncTarget +from leap.soledad.client.crypto import decrypt_doc_dict +from leap.soledad.client.sqlcipher import ( + SQLCipherDatabase, +) + +from leap.soledad.common.tests import u1db_tests as tests +from leap.soledad.common.tests.u1db_tests import test_sync +from leap.soledad.common.tests.test_sqlcipher import SQLCIPHER_SCENARIOS +from leap.soledad.common.tests.util import ( + make_soledad_app, + BaseSoledadTest, + SoledadWithCouchServerMixin, +) + + +#----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sync`. +#----------------------------------------------------------------------------- + +def sync_via_synchronizer_and_soledad(test, db_source, db_target, + trace_hook=None, trace_hook_shallow=None): + if trace_hook: + test.skipTest("full trace hook unsupported over http") + path = test._http_at[db_target] + target = SoledadSyncTarget.connect( + test.getURL(path), test._soledad._crypto) + target.set_token_credentials('user-uuid', 'auth-token') + if trace_hook_shallow: + target._set_trace_hook_shallow(trace_hook_shallow) + return sync.Synchronizer(db_source, target).sync() + + +sync_scenarios = [] +for name, scenario in SQLCIPHER_SCENARIOS: + scenario['do_sync'] = test_sync.sync_via_synchronizer + sync_scenarios.append((name, scenario)) + + +class SQLCipherDatabaseSyncTests( + TestWithScenarios, + test_sync.DatabaseSyncTests, + BaseSoledadTest): + """ + Test for succesfull sync between SQLCipher and LeapBackend. + + Some of the tests in this class had to be adapted because the remote + backend always receive encrypted content, and so it can not rely on + document's content comparison to try to autoresolve conflicts. + """ + + scenarios = sync_scenarios + + #def setUp(self): + # test_sync.DatabaseSyncTests.setUp(self) + + def tearDown(self): + test_sync.DatabaseSyncTests.tearDown(self) + if hasattr(self, 'db1') and isinstance(self.db1, SQLCipherDatabase): + self.db1.close() + if hasattr(self, 'db1_copy') \ + and isinstance(self.db1_copy, SQLCipherDatabase): + self.db1_copy.close() + if hasattr(self, 'db2') \ + and isinstance(self.db2, SQLCipherDatabase): + self.db2.close() + if hasattr(self, 'db2_copy') \ + and isinstance(self.db2_copy, SQLCipherDatabase): + self.db2_copy.close() + if hasattr(self, 'db3') \ + and isinstance(self.db3, SQLCipherDatabase): + self.db3.close() + + def test_sync_autoresolves(self): + """ + Test for sync autoresolve remote. + + This test was adapted because the remote database receives encrypted + content and so it can't compare documents contents to autoresolve. + """ + # The remote database can't autoresolve conflicts based on magic + # content convergence, so we modify this test to leave the possibility + # of the remode document ending up in conflicted state. + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev1 = doc1.rev + doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev2 = doc2.rev + self.sync(self.db1, self.db2) + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + # if remote content is in conflicted state, then document revisions + # will be different. + #self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) + v = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) + + def test_sync_autoresolves_moar(self): + """ + Test for sync autoresolve local. + + This test was adapted to decrypt remote content before assert. + """ + # here we test that when a database that has a conflicted document is + # the source of a sync, and the target database has a revision of the + # conflicted document that is newer than the source database's, and + # that target's database's document's content is the same as the + # source's document's conflict's, the source's document's conflict gets + # autoresolved, and the source's document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # `-------> + # a3b2 a1b2 (autoresolved) + # `-------> + # a3b2 a3b2 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + # because the conflict is on the source, sync it another time + self.sync(self.db1, self.db2) + # make sure db2 now has the exact same thing + doc1 = self.db1.get_doc('doc') + self.assertGetEncryptedDoc( + self.db2, + doc1.doc_id, doc1.rev, doc1.get_json(), False) + + def test_sync_autoresolves_moar_backwards(self): + # here we would test that when a database that has a conflicted + # document is the target of a sync, and the source database has a + # revision of the conflicted document that is newer than the target + # database's, and that source's database's document's content is the + # same as the target's document's conflict's, the target's document's + # conflict gets autoresolved, and the document's revision bumped. + # + # Despite that, in Soledad we suppose that the server never syncs, so + # it never has conflicted documents. Also, if it had, convergence + # would not be possible by checking document's contents because they + # would be encrypted in server. + # + # Therefore we suppress this test. + pass + + def test_sync_autoresolves_moar_backwards_three(self): + # here we would test that when a database that has a conflicted + # document is the target of a sync, and the source database has a + # revision of the conflicted document that is newer than the target + # database's, and that source's database's document's content is the + # same as the target's document's conflict's, the target's document's + # conflict gets autoresolved, and the document's revision bumped. + # + # We use the same reasoning from the last test to suppress this one. + pass + + def test_sync_propagates_resolution(self): + """ + Test if synchronization propagates resolution. + + This test was adapted to decrypt remote content before assert. + """ + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + db3 = self.create_database('test3', 'both') + self.sync(self.db2, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db2._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.sync(db3, self.db1) + # update on 2 + doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') + self.db2.put_doc(doc2) + self.sync(self.db2, db3) + self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) + # update on 1 + doc1.set_json('{"a": 3}') + self.db1.put_doc(doc1) + # conflicts + self.sync(self.db2, self.db1) + self.sync(db3, self.db1) + self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) + self.assertTrue(db3.get_doc('the-doc').has_conflicts) + # resolve + conflicts = self.db2.get_doc_conflicts('the-doc') + doc4 = self.make_document('the-doc', None, '{"a": 4}') + revs = [doc.rev for doc in conflicts] + self.db2.resolve_doc(doc4, revs) + doc2 = self.db2.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc2.get_json()) + self.assertFalse(doc2.has_conflicts) + self.sync(self.db2, db3) + doc3 = db3.get_doc('the-doc') + if ENC_SCHEME_KEY in doc3.content: + _crypto = self._soledad._crypto + key = _crypto.doc_passphrase(doc3.doc_id) + secret = _crypto.secret + doc3.set_json(decrypt_doc_dict( + doc3.content, + doc3.doc_id, doc3.rev, key, secret)) + self.assertEqual(doc4.get_json(), doc3.get_json()) + self.assertFalse(doc3.has_conflicts) + self.db1.close() + self.db2.close() + db3.close() + + def test_sync_puts_changes(self): + """ + Test if sync puts changes in remote replica. + + This test was adapted to decrypt remote content before assert. + """ + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(tests.simple_doc) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertGetEncryptedDoc( + self.db2, doc.doc_id, doc.rev, tests.simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + +def _make_local_db_and_token_http_target(test, path='test'): + test.startServer() + db = test.request_state._create_database(os.path.basename(path)) + st = SoledadSyncTarget.connect( + test.getURL(path), crypto=test._soledad._crypto) + st.set_token_credentials('user-uuid', 'auth-token') + return db, st + + +target_scenarios = [ + ('leap', { + 'create_db_and_target': _make_local_db_and_token_http_target, +# 'make_app_with_state': tests.test_remote_sync_target.make_http_app, + 'make_app_with_state': make_soledad_app, + 'do_sync': sync_via_synchronizer_and_soledad}), +] + + +class SQLCipherSyncTargetTests( + TestWithScenarios, + SoledadWithCouchServerMixin, + test_sync.DatabaseSyncTargetTests): + + scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS, + target_scenarios)) + + whitebox = False + + def setUp(self): + self.main_test_class = test_sync.DatabaseSyncTargetTests + SoledadWithCouchServerMixin.setUp(self) + + def test_sync_exchange(self): + """ + Modified to account for possibly receiving encrypted documents from + sever-side. + """ + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, + 'T-sid')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetEncryptedDoc( + self.db, 'doc-id', 'replica:1', tests.simple_doc, False) + self.assertTransactionLog(['doc-id'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, last_trans_id)) + self.assertEqual(10, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_push_many(self): + """ + Modified to account for possibly receiving encrypted documents from + sever-side. + """ + docs_by_gen = [ + (self.make_document( + 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), + (self.make_document('doc-id2', 'replica:1', tests.nested_doc), 11, + 'T-2')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetEncryptedDoc( + self.db, 'doc-id', 'replica:1', tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db, 'doc-id2', 'replica:1', tests.nested_doc, False) + self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(11, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_returns_many_new_docs(self): + """ + Modified to account for JSON serialization differences. + """ + doc = self.db.create_doc_from_json(tests.simple_doc) + doc2 = self.db.create_doc_from_json(tests.nested_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + self.assertEqual(2, new_gen) + self.assertEqual( + [(doc.doc_id, doc.rev, 1), + (doc2.doc_id, doc2.rev, 2)], + [c[:2] + c[3:4] for c in self.other_changes]) + self.assertEqual( + json.dumps(tests.simple_doc), + json.dumps(self.other_changes[0][2])) + self.assertEqual( + json.loads(tests.nested_doc), + json.loads(self.other_changes[1][2])) + if self.whitebox: + self.assertEqual( + self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': + [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) diff --git a/common/src/leap/soledad/common/tests/test_sync.py b/common/src/leap/soledad/common/tests/test_sync.py index 0433fac9..92c601dc 100644 --- a/common/src/leap/soledad/common/tests/test_sync.py +++ b/common/src/leap/soledad/common/tests/test_sync.py @@ -16,43 +16,35 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -import mock -import os import json import tempfile import threading import time + from urlparse import urljoin +from twisted.internet import defer + +from testscenarios import TestWithScenarios from leap.soledad.common import couch +from leap.soledad.client import target +from leap.soledad.client import sync +from leap.soledad.server import SoledadApp -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.tests import test_sync_target from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import ( - TestCaseWithServer, - simple_doc, - test_backends, - test_sync -) -from leap.soledad.common.tests.test_couch import CouchDBTestCase -from leap.soledad.common.tests.test_target_soledad import ( - make_token_soledad_app, - make_leap_document_for_test, -) -from leap.soledad.common.tests.test_sync_target import token_leap_sync_target -from leap.soledad.client import ( - Soledad, - target, -) +from leap.soledad.common.tests.u1db_tests import TestCaseWithServer +from leap.soledad.common.tests.u1db_tests import simple_doc +from leap.soledad.common.tests.u1db_tests import test_sync +from leap.soledad.common.tests.util import make_token_soledad_app +from leap.soledad.common.tests.util import make_soledad_document_for_test +from leap.soledad.common.tests.util import token_soledad_sync_target +from leap.soledad.common.tests.util import BaseSoledadTest from leap.soledad.common.tests.util import SoledadWithCouchServerMixin -from leap.soledad.client.sync import SoledadSynchronizer -from leap.soledad.server import SoledadApp - +from leap.soledad.common.tests.test_couch import CouchDBTestCase class InterruptableSyncTestCase( - CouchDBTestCase, TestCaseWithServer): + BaseSoledadTest, CouchDBTestCase, TestCaseWithServer): """ Tests for encrypted sync using Soledad server backed by a couch database. """ @@ -61,51 +53,12 @@ class InterruptableSyncTestCase( def make_app_with_state(state): return make_token_soledad_app(state) - make_document_for_test = make_leap_document_for_test - - sync_target = token_leap_sync_target + make_document_for_test = make_soledad_document_for_test - def _soledad_instance(self, user='user-uuid', passphrase=u'123', - prefix='', - secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME, - local_db_path='soledad.u1db', server_url='', - cert_file=None, auth_token=None, secret_id=None): - """ - Instantiate Soledad. - """ - - # this callback ensures we save a document which is sent to the shared - # db. - def _put_doc_side_effect(doc): - self._doc_put = doc - - # we need a mocked shared db or else Soledad will try to access the - # network to find if there are uploaded secrets. - class MockSharedDB(object): - - get_doc = mock.Mock(return_value=None) - put_doc = mock.Mock(side_effect=_put_doc_side_effect) - lock = mock.Mock(return_value=('atoken', 300)) - unlock = mock.Mock() - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() - return Soledad( - user, - passphrase, - secrets_path=os.path.join(self.tempdir, prefix, secrets_path), - local_db_path=os.path.join( - self.tempdir, prefix, local_db_path), - server_url=server_url, - cert_file=cert_file, - auth_token=auth_token, - secret_id=secret_id) + sync_target = token_soledad_sync_target def make_app(self): - self.request_state = couch.CouchServerState( - self._couch_url, 'shared', 'tokens') + self.request_state = couch.CouchServerState(self._couch_url) return self.make_app_with_state(self.request_state) def setUp(self): @@ -135,7 +88,8 @@ class InterruptableSyncTestCase( def run(self): while db._get_generation() < 2: - time.sleep(1) + #print "WAITING %d" % db._get_generation() + time.sleep(0.1) self._soledad.stop_sync() time.sleep(1) @@ -143,16 +97,7 @@ class InterruptableSyncTestCase( self.startServer() # instantiate soledad and create a document - sol = self._soledad_instance( - # token is verified in test_target.make_token_soledad_app - auth_token='auth-token' - ) - _, doclist = sol.get_all_docs() - self.assertEqual([], doclist) - - # create many small files - for i in range(0, number_of_docs): - sol.create_doc(json.loads(simple_doc)) + sol = self._soledad_instance(user='user-uuid', server_url=self.getURL()) # ensure remote db exists before syncing db = couch.CouchDatabase.open_database( @@ -164,21 +109,35 @@ class InterruptableSyncTestCase( t = _SyncInterruptor(sol, db) t.start() - # sync with server - sol._server_url = self.getURL() - sol.sync() # this will be interrupted when couch db gen >= 2 - t.join() + d = sol.get_all_docs() + d.addCallback(lambda results: self.assertEqual([], results[1])) - # recover the sync process - sol.sync() + def _create_docs(results): + # create many small files + deferreds = [] + for i in range(0, number_of_docs): + deferreds.append(sol.create_doc(json.loads(simple_doc))) + return defer.DeferredList(deferreds) - gen, doclist = db.get_all_docs() - self.assertEqual(number_of_docs, len(doclist)) - - # delete remote database - db.delete_database() - db.close() - sol.close() + # sync with server + d.addCallback(_create_docs) + d.addCallback(lambda _: sol.get_all_docs()) + d.addCallback(lambda results: self.assertEqual(number_of_docs, len(results[1]))) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: t.join()) + d.addCallback(lambda _: db.get_all_docs()) + d.addCallback(lambda results: self.assertNotEqual(number_of_docs, len(results[1]))) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: db.get_all_docs()) + d.addCallback(lambda results: self.assertEqual(number_of_docs, len(results[1]))) + + def _tear_down(results): + db.delete_database() + db.close() + sol.close() + + d.addCallback(_tear_down) + return d def make_soledad_app(state): @@ -186,6 +145,7 @@ def make_soledad_app(state): class TestSoledadDbSync( + TestWithScenarios, SoledadWithCouchServerMixin, test_sync.TestDbSync): """ @@ -198,7 +158,7 @@ class TestSoledadDbSync( 'make_database_for_test': tests.make_memory_database_for_test, }), ('py-token-http', { - 'make_app_with_state': test_sync_target.make_token_soledad_app, + 'make_app_with_state': make_token_soledad_app, 'make_database_for_test': tests.make_memory_database_for_test, 'token': True }), @@ -211,10 +171,11 @@ class TestSoledadDbSync( """ Need to explicitely invoke inicialization on all bases. """ - tests.TestCaseWithServer.setUp(self) - self.main_test_class = test_sync.TestDbSync + #tests.TestCaseWithServer.setUp(self) + #self.main_test_class = test_sync.TestDbSync SoledadWithCouchServerMixin.setUp(self) self.startServer() + self.db = self.make_database_for_test(self, 'test1') self.db2 = couch.CouchDatabase.open_database( urljoin( 'http://localhost:' + str(self.wrapper.port), 'test'), @@ -227,7 +188,7 @@ class TestSoledadDbSync( """ self.db2.delete_database() SoledadWithCouchServerMixin.tearDown(self) - tests.TestCaseWithServer.tearDown(self) + #tests.TestCaseWithServer.tearDown(self) def do_sync(self, target_name): """ @@ -240,7 +201,7 @@ class TestSoledadDbSync( 'token': 'auth-token', }}) target_url = self.getURL(target_name) - return SoledadSynchronizer( + return sync.SoledadSynchronizer( self.db, target.SoledadSyncTarget( target_url, @@ -254,8 +215,10 @@ class TestSoledadDbSync( Adapted to check for encrypted content. """ + doc1 = self.db.create_doc_from_json(tests.simple_doc) doc2 = self.db2.create_doc_from_json(tests.nested_doc) + local_gen_before_sync = self.do_sync('test') gen, _, changes = self.db.whats_changed(local_gen_before_sync) self.assertEqual(1, len(changes)) @@ -287,6 +250,3 @@ class TestSoledadDbSync( s_gen, _ = db3._get_replica_gen_and_trans_id('test1') self.assertEqual(1, t_gen) self.assertEqual(1, s_gen) - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_sync_deferred.py b/common/src/leap/soledad/common/tests/test_sync_deferred.py index 48e3150f..010f5492 100644 --- a/common/src/leap/soledad/common/tests/test_sync_deferred.py +++ b/common/src/leap/soledad/common/tests/test_sync_deferred.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # test_sync_deferred.py # Copyright (C) 2014 LEAP # @@ -21,31 +20,31 @@ import time import os import random import string +import shutil + from urlparse import urljoin -from leap.soledad.common.tests import u1db_tests as tests, ADDRESS +from leap.soledad.common import couch +from leap.soledad.client.sqlcipher import ( + SQLCipherOptions, + SQLCipherDatabase, + SQLCipherU1DBSync, +) + +from testscenarios import TestWithScenarios + +from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_sync +from leap.soledad.common.tests.util import ADDRESS +from leap.soledad.common.tests.util import SoledadWithCouchServerMixin +from leap.soledad.common.tests.util import make_soledad_app -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common import couch -from leap.soledad.client import target -from leap.soledad.client.sync import SoledadSynchronizer # Just to make clear how this test is different... :) DEFER_DECRYPTION = True WAIT_STEP = 1 MAX_WAIT = 10 - -from leap.soledad.common.tests import test_sqlcipher as ts -from leap.soledad.server import SoledadApp - - -from leap.soledad.client.sqlcipher import open as open_sqlcipher -from leap.soledad.common.tests.util import SoledadWithCouchServerMixin -from leap.soledad.common.tests.util import make_soledad_app - - DBPASS = "pass" @@ -57,8 +56,10 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): defer_sync_encryption = True def setUp(self): + SoledadWithCouchServerMixin.setUp(self) # config info self.db1_file = os.path.join(self.tempdir, "db1.u1db") + os.unlink(self.db1_file) self.db_pass = DBPASS self.email = ADDRESS @@ -67,16 +68,21 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): # each local db. self.rand_prefix = ''.join( map(lambda x: random.choice(string.ascii_letters), range(6))) - # initialize soledad by hand so we can control keys - self._soledad = self._soledad_instance( - prefix=self.rand_prefix, user=self.email) - - # open test dbs: db1 will be the local sqlcipher db - # (which instantiates a syncdb) - self.db1 = open_sqlcipher(self.db1_file, DBPASS, create=True, - document_factory=SoledadDocument, - crypto=self._soledad._crypto, - defer_encryption=True) + + # open test dbs: db1 will be the local sqlcipher db (which + # instantiates a syncdb). We use the self._soledad instance that was + # already created on some setUp method. + import binascii + tohex = binascii.b2a_hex + key = tohex(self._soledad.secrets.get_local_storage_key()) + sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) + dbpath = self._soledad._local_db_path + + self.opts = SQLCipherOptions( + dbpath, key, is_raw_key=True, create=False, + defer_encryption=True, sync_db_key=sync_db_key) + self.db1 = SQLCipherDatabase(self.opts) + self.db2 = couch.CouchDatabase.open_database( urljoin( 'http://localhost:' + str(self.wrapper.port), 'test'), @@ -89,23 +95,8 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): self._soledad.close() # XXX should not access "private" attrs - for f in [self._soledad._local_db_path, - self._soledad._secrets_path, - self.db1._sync_db_path]: - if os.path.isfile(f): - os.unlink(f) - - -#SQLCIPHER_SCENARIOS = [ -# ('http', { -# #'make_app_with_state': test_sync_target.make_token_soledad_app, -# 'make_app_with_state': make_soledad_app, -# 'make_database_for_test': ts.make_sqlcipher_database_for_test, -# 'copy_database_for_test': ts.copy_sqlcipher_database_for_test, -# 'make_document_for_test': ts.make_document_for_test, -# 'token': True -# }), -#] + shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) + SoledadWithCouchServerMixin.tearDown(self) class SyncTimeoutError(Exception): @@ -116,8 +107,9 @@ class SyncTimeoutError(Exception): class TestSoledadDbSyncDeferredEncDecr( - BaseSoledadDeferredEncTest, - test_sync.TestDbSync): + TestWithScenarios, + test_sync.TestDbSync, + BaseSoledadDeferredEncTest): """ Test db.sync remote sync shortcut. Case with deferred encryption and decryption: using the intermediate @@ -134,13 +126,16 @@ class TestSoledadDbSyncDeferredEncDecr( oauth = False token = True + def make_app(self): + self.request_state = couch.CouchServerState(self._couch_url) + return self.make_app_with_state(self.request_state) + def setUp(self): """ Need to explicitely invoke inicialization on all bases. """ - tests.TestCaseWithServer.setUp(self) - self.main_test_class = test_sync.TestDbSync BaseSoledadDeferredEncTest.setUp(self) + self.server = self.server_thread = None self.startServer() self.syncer = None @@ -148,8 +143,10 @@ class TestSoledadDbSyncDeferredEncDecr( """ Need to explicitely invoke destruction on all bases. """ + dbsyncer = getattr(self, 'dbsyncer', None) + if dbsyncer: + dbsyncer.close() BaseSoledadDeferredEncTest.tearDown(self) - tests.TestCaseWithServer.tearDown(self) def do_sync(self, target_name): """ @@ -157,25 +154,20 @@ class TestSoledadDbSyncDeferredEncDecr( and Token auth. """ if self.token: - extra = dict(creds={'token': { + creds={'token': { 'uuid': 'user-uuid', 'token': 'auth-token', - }}) + }} target_url = self.getURL(target_name) - syncdb = getattr(self.db1, "_sync_db", None) - - syncer = SoledadSynchronizer( - self.db1, - target.SoledadSyncTarget( - target_url, - crypto=self._soledad._crypto, - sync_db=syncdb, - **extra)) - # Keep a reference to be able to know when the sync - # has finished. - self.syncer = syncer - return syncer.sync( - autocreate=True, defer_decryption=DEFER_DECRYPTION) + + # get a u1db syncer + crypto = self._soledad._crypto + replica_uid = self.db1._replica_uid + dbsyncer = SQLCipherU1DBSync(self.opts, crypto, replica_uid, + defer_encryption=True) + self.dbsyncer = dbsyncer + return dbsyncer.sync(target_url, creds=creds, + autocreate=True,defer_decryption=DEFER_DECRYPTION) else: return test_sync.TestDbSync.do_sync(self, target_name) @@ -200,28 +192,30 @@ class TestSoledadDbSyncDeferredEncDecr( """ doc1 = self.db1.create_doc_from_json(tests.simple_doc) doc2 = self.db2.create_doc_from_json(tests.nested_doc) + d = self.do_sync('test') - import time - # need to give time to the encryption to proceed - # TODO should implement a defer list to subscribe to the all-decrypted - # event - time.sleep(2) + def _assert_successful_sync(results): + import time + # need to give time to the encryption to proceed + # TODO should implement a defer list to subscribe to the all-decrypted + # event + time.sleep(2) + local_gen_before_sync = results + self.wait_for_sync() - local_gen_before_sync = self.do_sync('test') - self.wait_for_sync() + gen, _, changes = self.db1.whats_changed(local_gen_before_sync) + self.assertEqual(1, len(changes)) - gen, _, changes = self.db1.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) + self.assertEqual(doc2.doc_id, changes[0][0]) + self.assertEqual(1, gen - local_gen_before_sync) - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) + self.assertGetEncryptedDoc( + self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) + d.addCallback(_assert_successful_sync) + return d def test_db_sync_autocreate(self): pass - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_sync_target.py b/common/src/leap/soledad/common/tests/test_sync_target.py index edc4589b..941dc37c 100644 --- a/common/src/leap/soledad/common/tests/test_sync_target.py +++ b/common/src/leap/soledad/common/tests/test_sync_target.py @@ -19,101 +19,36 @@ Test Leap backend bits: sync target """ import cStringIO import os - +import time import simplejson as json import u1db +import random +import string +import shutil -from uuid import uuid4 - -from u1db.remote import http_database - -from u1db import SyncTarget -from u1db.sync import Synchronizer -from u1db.remote import ( - http_client, - http_database, - http_target, -) - -from leap.soledad import client -from leap.soledad.client import ( - target, - auth, - crypto, - VerifiedHTTPSConnection, - sync, -) -from leap.soledad.common.document import SoledadDocument -from leap.soledad.server.auth import SoledadTokenAuthMiddleware +from testscenarios import TestWithScenarios +from urlparse import urljoin +from leap.soledad.client import target +from leap.soledad.client import crypto +from leap.soledad.client.sqlcipher import SQLCipherU1DBSync +from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client.sqlcipher import SQLCipherDatabase + +from leap.soledad.common import couch +from leap.soledad.common.document import SoledadDocument from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.tests.util import ( - make_sqlcipher_database_for_test, - make_soledad_app, - make_token_soledad_app, - SoledadWithCouchServerMixin, -) -from leap.soledad.common.tests.u1db_tests import test_backends +from leap.soledad.common.tests.util import make_sqlcipher_database_for_test +from leap.soledad.common.tests.util import make_soledad_app +from leap.soledad.common.tests.util import make_token_soledad_app +from leap.soledad.common.tests.util import make_soledad_document_for_test +from leap.soledad.common.tests.util import token_soledad_sync_target +from leap.soledad.common.tests.util import BaseSoledadTest +from leap.soledad.common.tests.util import SoledadWithCouchServerMixin +from leap.soledad.common.tests.util import ADDRESS from leap.soledad.common.tests.u1db_tests import test_remote_sync_target from leap.soledad.common.tests.u1db_tests import test_sync -from leap.soledad.common.tests.test_couch import ( - CouchDBTestCase, - CouchDBWrapper, -) - -from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -#----------------------------------------------------------------------------- - -def make_leap_document_for_test(test, doc_id, rev, content, - has_conflicts=False): - return SoledadDocument( - doc_id, rev, content, has_conflicts=has_conflicts) - - -LEAP_SCENARIOS = [ - ('http', { - 'make_database_for_test': test_backends.make_http_database_for_test, - 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': make_leap_document_for_test, - 'make_app_with_state': make_soledad_app}), -] - - -def make_token_http_database_for_test(test, replica_uid): - test.startServer() - test.request_state._create_database(replica_uid) - - class _HTTPDatabaseWithToken( - http_database.HTTPDatabase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - http_db = _HTTPDatabaseWithToken(test.getURL('test')) - http_db.set_token_credentials('user-uuid', 'auth-token') - return http_db - - -def copy_token_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - http_db = test.request_state._copy_database(db) - http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') - return http_db #----------------------------------------------------------------------------- @@ -143,12 +78,6 @@ class TestSoledadParsingSyncStream( target. """ - def setUp(self): - test_remote_sync_target.TestParsingSyncStream.setUp(self) - - def tearDown(self): - test_remote_sync_target.TestParsingSyncStream.tearDown(self) - def test_extra_comma(self): """ Test adapted to use encrypted content. @@ -230,17 +159,6 @@ class TestSoledadParsingSyncStream( # functions for TestRemoteSyncTargets # -def leap_sync_target(test, path): - return target.SoledadSyncTarget( - test.getURL(path), crypto=test._soledad._crypto) - - -def token_leap_sync_target(test, path): - st = leap_sync_target(test, path) - st.set_token_credentials('user-uuid', 'auth-token') - return st - - def make_local_db_and_soledad_target(test, path='test'): test.startServer() db = test.request_state._create_database(os.path.basename(path)) @@ -256,31 +174,32 @@ def make_local_db_and_token_soledad_target(test): class TestSoledadSyncTarget( + TestWithScenarios, SoledadWithCouchServerMixin, test_remote_sync_target.TestRemoteSyncTargets): scenarios = [ ('token_soledad', {'make_app_with_state': make_token_soledad_app, - 'make_document_for_test': make_leap_document_for_test, + 'make_document_for_test': make_soledad_document_for_test, 'create_db_and_target': make_local_db_and_token_soledad_target, 'make_database_for_test': make_sqlcipher_database_for_test, - 'sync_target': token_leap_sync_target}), + 'sync_target': token_soledad_sync_target}), ] def setUp(self): - tests.TestCaseWithServer.setUp(self) - self.main_test_class = test_remote_sync_target.TestRemoteSyncTargets + TestWithScenarios.setUp(self) SoledadWithCouchServerMixin.setUp(self) self.startServer() self.db1 = make_sqlcipher_database_for_test(self, 'test1') self.db2 = self.request_state._create_database('test2') def tearDown(self): + #db2, _ = self.request_state.ensure_database('test2') + self.db2.delete_database() + self.db1.close() SoledadWithCouchServerMixin.tearDown(self) - tests.TestCaseWithServer.tearDown(self) - db, _ = self.request_state.ensure_database('test2') - db.delete_database() + TestWithScenarios.tearDown(self) def test_sync_exchange_send(self): """ @@ -288,7 +207,6 @@ class TestSoledadSyncTarget( This test was adapted to decrypt remote content before assert. """ - self.startServer() db = self.request_state._create_database('test') remote_target = self.getSyncTarget('test') other_docs = [] @@ -309,14 +227,9 @@ class TestSoledadSyncTarget( """ Test for sync exchange failure and retry. - This test was adapted to: - - decrypt remote content before assert. - - not expect a bounced document because soledad has stateful - recoverable sync. + This test was adapted to decrypt remote content before assert. """ - self.startServer() - def blackhole_getstderr(inst): return cStringIO.StringIO() @@ -352,8 +265,9 @@ class TestSoledadSyncTarget( doc2 = self.make_document('doc-here2', 'replica:1', '{"value": "here2"}') - # we do not expect an HTTPError because soledad sync fails gracefully - remote_target.sync_exchange( + self.assertRaises( + u1db.errors.HTTPError, + remote_target.sync_exchange, [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], 'replica', last_known_generation=0, last_known_trans_id=None, return_doc_cb=receive_doc) @@ -384,7 +298,6 @@ class TestSoledadSyncTarget( This test was adapted to decrypt remote content before assert. """ - self.startServer() remote_target = self.getSyncTarget('test') other_docs = [] replica_uid_box = [] @@ -425,7 +338,9 @@ target_scenarios = [ class SoledadDatabaseSyncTargetTests( - SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests): + TestWithScenarios, + SoledadWithCouchServerMixin, + test_sync.DatabaseSyncTargetTests): scenarios = ( tests.multiply_scenarios( @@ -520,8 +435,25 @@ class SoledadDatabaseSyncTargetTests( [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) +# Just to make clear how this test is different... :) +DEFER_DECRYPTION = False + +WAIT_STEP = 1 +MAX_WAIT = 10 +DBPASS = "pass" + + +class SyncTimeoutError(Exception): + """ + Dummy exception to notify timeout during sync. + """ + pass + + class TestSoledadDbSync( - SoledadWithCouchServerMixin, test_sync.TestDbSync): + TestWithScenarios, + SoledadWithCouchServerMixin, + test_sync.TestDbSync): """Test db.sync remote sync shortcut""" scenarios = [ @@ -536,9 +468,66 @@ class TestSoledadDbSync( oauth = False token = False + + def make_app(self): + self.request_state = couch.CouchServerState(self._couch_url) + return self.make_app_with_state(self.request_state) + def setUp(self): - self.main_test_class = test_sync.TestDbSync + """ + Need to explicitely invoke inicialization on all bases. + """ SoledadWithCouchServerMixin.setUp(self) + self.server = self.server_thread = None + self.startServer() + self.syncer = None + + # config info + self.db1_file = os.path.join(self.tempdir, "db1.u1db") + os.unlink(self.db1_file) + self.db_pass = DBPASS + self.email = ADDRESS + + # get a random prefix for each test, so we do not mess with + # concurrency during initialization and shutting down of + # each local db. + self.rand_prefix = ''.join( + map(lambda x: random.choice(string.ascii_letters), range(6))) + + # open test dbs: db1 will be the local sqlcipher db (which + # instantiates a syncdb). We use the self._soledad instance that was + # already created on some setUp method. + import binascii + tohex = binascii.b2a_hex + key = tohex(self._soledad.secrets.get_local_storage_key()) + sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) + dbpath = self._soledad._local_db_path + + self.opts = SQLCipherOptions( + dbpath, key, is_raw_key=True, create=False, + defer_encryption=True, sync_db_key=sync_db_key) + self.db1 = SQLCipherDatabase(self.opts) + + self.db2 = couch.CouchDatabase.open_database( + urljoin( + 'http://localhost:' + str(self.wrapper.port), 'test'), + create=True, + ensure_ddocs=True) + + def tearDown(self): + """ + Need to explicitely invoke destruction on all bases. + """ + dbsyncer = getattr(self, 'dbsyncer', None) + if dbsyncer: + dbsyncer.close() + self.db1.close() + self.db2.close() + self._soledad.close() + + # XXX should not access "private" attrs + shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) + SoledadWithCouchServerMixin.tearDown(self) def do_sync(self, target_name): """ @@ -546,44 +535,71 @@ class TestSoledadDbSync( and Token auth. """ if self.token: - extra = dict(creds={'token': { + creds={'token': { 'uuid': 'user-uuid', 'token': 'auth-token', - }}) + }} target_url = self.getURL(target_name) - return sync.SoledadSynchronizer( - self.db, - target.SoledadSyncTarget( - target_url, - crypto=self._soledad._crypto, - **extra)).sync(autocreate=True, - defer_decryption=False) + + # get a u1db syncer + crypto = self._soledad._crypto + replica_uid = self.db1._replica_uid + dbsyncer = SQLCipherU1DBSync(self.opts, crypto, replica_uid, + defer_encryption=True) + self.dbsyncer = dbsyncer + return dbsyncer.sync(target_url, creds=creds, + autocreate=True,defer_decryption=DEFER_DECRYPTION) else: return test_sync.TestDbSync.do_sync(self, target_name) + def wait_for_sync(self): + """ + Wait for sync to finish. + """ + wait = 0 + syncer = self.syncer + if syncer is not None: + while syncer.syncing: + time.sleep(WAIT_STEP) + wait += WAIT_STEP + if wait >= MAX_WAIT: + raise SyncTimeoutError + def test_db_sync(self): """ Test sync. Adapted to check for encrypted content. """ - doc1 = self.db.create_doc_from_json(tests.simple_doc) + doc1 = self.db1.create_doc_from_json(tests.simple_doc) doc2 = self.db2.create_doc_from_json(tests.nested_doc) - local_gen_before_sync = self.do_sync('test2') - gen, _, changes = self.db.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False) + d = self.do_sync('test') + + def _assert_successful_sync(results): + import time + # need to give time to the encryption to proceed + # TODO should implement a defer list to subscribe to the all-decrypted + # event + time.sleep(2) + local_gen_before_sync = results + self.wait_for_sync() + + gen, _, changes = self.db1.whats_changed(local_gen_before_sync) + self.assertEqual(1, len(changes)) + + self.assertEqual(doc2.doc_id, changes[0][0]) + self.assertEqual(1, gen - local_gen_before_sync) + + self.assertGetEncryptedDoc( + self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) + + d.addCallback(_assert_successful_sync) + return d def test_db_sync_autocreate(self): """ We bypass this test because we never need to autocreate databases. """ pass - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_target.py b/common/src/leap/soledad/common/tests/test_target.py deleted file mode 100644 index 6242099d..00000000 --- a/common/src/leap/soledad/common/tests/test_target.py +++ /dev/null @@ -1,794 +0,0 @@ -# -*- coding: utf-8 -*- -# test_target.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -Test Leap backend bits. -""" - -import u1db -import os -import ssl -import simplejson as json -import cStringIO - - -from u1db import SyncTarget -from u1db.sync import Synchronizer -from u1db.remote import ( - http_client, - http_database, - http_target, -) - -from leap.soledad import client -from leap.soledad.client import ( - target, - auth, - VerifiedHTTPSConnection, - sync, -) -from leap.soledad.common.document import SoledadDocument -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.tests.util import ( - make_sqlcipher_database_for_test, - make_soledad_app, - make_token_soledad_app, - SoledadWithCouchServerMixin, -) -from leap.soledad.common.tests.u1db_tests import test_backends -from leap.soledad.common.tests.u1db_tests import test_http_database -from leap.soledad.common.tests.u1db_tests import test_http_client -from leap.soledad.common.tests.u1db_tests import test_document -from leap.soledad.common.tests.u1db_tests import test_remote_sync_target -from leap.soledad.common.tests.u1db_tests import test_https -from leap.soledad.common.tests.u1db_tests import test_sync -from leap.soledad.common.tests.test_couch import ( - CouchDBTestCase, - CouchDBWrapper, -) - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -#----------------------------------------------------------------------------- - -def make_leap_document_for_test(test, doc_id, rev, content, - has_conflicts=False): - return SoledadDocument( - doc_id, rev, content, has_conflicts=has_conflicts) - - -LEAP_SCENARIOS = [ - ('http', { - 'make_database_for_test': test_backends.make_http_database_for_test, - 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': make_leap_document_for_test, - 'make_app_with_state': make_soledad_app}), -] - - -def make_token_http_database_for_test(test, replica_uid): - test.startServer() - test.request_state._create_database(replica_uid) - - class _HTTPDatabaseWithToken( - http_database.HTTPDatabase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - http_db = _HTTPDatabaseWithToken(test.getURL('test')) - http_db.set_token_credentials('user-uuid', 'auth-token') - return http_db - - -def copy_token_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - http_db = test.request_state._copy_database(db) - http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') - return http_db - - -class SoledadTests(test_backends.AllDatabaseTests, BaseSoledadTest): - - scenarios = LEAP_SCENARIOS + [ - ('token_http', {'make_database_for_test': - make_token_http_database_for_test, - 'copy_database_for_test': - copy_token_http_database_for_test, - 'make_document_for_test': make_leap_document_for_test, - 'make_app_with_state': make_token_soledad_app, - }) - ] - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_http_client`. -#----------------------------------------------------------------------------- - -class TestSoledadClientBase(test_http_client.TestHTTPClientBase): - """ - This class should be used to test Token auth. - """ - - def getClientWithToken(self, **kwds): - self.startServer() - - class _HTTPClientWithToken( - http_client.HTTPClientBase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - return _HTTPClientWithToken(self.getURL('dbase'), **kwds) - - def test_oauth(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def test_oauth_ctr_creds(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def test_oauth_Unauthorized(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def app(self, environ, start_response): - res = test_http_client.TestHTTPClientBase.app( - self, environ, start_response) - if res is not None: - return res - # mime solead application here. - if '/token' in environ['PATH_INFO']: - auth = environ.get(SoledadTokenAuthMiddleware.HTTP_AUTH_KEY) - if not auth: - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": e.message})] - scheme, encoded = auth.split(None, 1) - if scheme.lower() != 'token': - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": e.message})] - uuid, token = encoded.decode('base64').split(':', 1) - if uuid != 'user-uuid' and token != 'auth-token': - return unauth_err("Incorrect address or token.") - start_response("200 OK", [('Content-Type', 'application/json')]) - return [json.dumps([environ['PATH_INFO'], uuid, token])] - - def test_token(self): - """ - Test if token is sent correctly. - """ - cli = self.getClientWithToken() - cli.set_token_credentials('user-uuid', 'auth-token') - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) - - def test_token_ctr_creds(self): - cli = self.getClientWithToken(creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_document`. -#----------------------------------------------------------------------------- - -class TestSoledadDocument(test_document.TestDocument, BaseSoledadTest): - - scenarios = ([( - 'leap', {'make_document_for_test': make_leap_document_for_test})]) - - -class TestSoledadPyDocument(test_document.TestPyDocument, BaseSoledadTest): - - scenarios = ([( - 'leap', {'make_document_for_test': make_leap_document_for_test})]) - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_remote_sync_target`. -#----------------------------------------------------------------------------- - -class TestSoledadSyncTargetBasics( - test_remote_sync_target.TestHTTPSyncTargetBasics): - """ - Some tests had to be copied to this class so we can instantiate our own - target. - """ - - def test_parse_url(self): - remote_target = target.SoledadSyncTarget('http://127.0.0.1:12345/') - self.assertEqual('http', remote_target._url.scheme) - self.assertEqual('127.0.0.1', remote_target._url.hostname) - self.assertEqual(12345, remote_target._url.port) - self.assertEqual('/', remote_target._url.path) - - -class TestSoledadParsingSyncStream( - test_remote_sync_target.TestParsingSyncStream, - BaseSoledadTest): - """ - Some tests had to be copied to this class so we can instantiate our own - target. - """ - - def setUp(self): - test_remote_sync_target.TestParsingSyncStream.setUp(self) - - def tearDown(self): - test_remote_sync_target.TestParsingSyncStream.tearDown(self) - - def test_extra_comma(self): - """ - Test adapted to use encrypted content. - """ - doc = SoledadDocument('i', rev='r') - doc.content = {} - enc_json = target.encrypt_doc(self._soledad._crypto, doc) - tgt = target.SoledadSyncTarget( - "http://foo/foo", crypto=self._soledad._crypto) - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "[\r\n{},\r\n]", None) - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, - '[\r\n{},\r\n{"id": "i", "rev": "r", ' - '"content": %s, "gen": 3, "trans_id": "T-sid"}' - ',\r\n]' % json.dumps(enc_json), - lambda doc, gen, trans_id: None) - - def test_wrong_start(self): - tgt = target.SoledadSyncTarget("http://foo/foo") - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "{}\r\n]", None) - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "\r\n{}\r\n]", None) - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "", None) - - def test_wrong_end(self): - tgt = target.SoledadSyncTarget("http://foo/foo") - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "[\r\n{}", None) - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "[\r\n", None) - - def test_missing_comma(self): - tgt = target.SoledadSyncTarget("http://foo/foo") - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, - '[\r\n{}\r\n{"id": "i", "rev": "r", ' - '"content": "c", "gen": 3}\r\n]', None) - - def test_no_entries(self): - tgt = target.SoledadSyncTarget("http://foo/foo") - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "[\r\n]", None) - - def test_error_in_stream(self): - tgt = target.SoledadSyncTarget("http://foo/foo") - - self.assertRaises(u1db.errors.Unavailable, - tgt._parse_sync_stream, - '[\r\n{"new_generation": 0},' - '\r\n{"error": "unavailable"}\r\n', None) - - self.assertRaises(u1db.errors.Unavailable, - tgt._parse_sync_stream, - '[\r\n{"error": "unavailable"}\r\n', None) - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, - '[\r\n{"error": "?"}\r\n', None) - - -# -# functions for TestRemoteSyncTargets -# - -def leap_sync_target(test, path): - return target.SoledadSyncTarget( - test.getURL(path), crypto=test._soledad._crypto) - - -def token_leap_sync_target(test, path): - st = leap_sync_target(test, path) - st.set_token_credentials('user-uuid', 'auth-token') - return st - - -def make_local_db_and_soledad_target(test, path='test'): - test.startServer() - db = test.request_state._create_database(os.path.basename(path)) - st = target.SoledadSyncTarget.connect( - test.getURL(path), crypto=test._soledad._crypto) - return db, st - - -def make_local_db_and_token_soledad_target(test): - db, st = make_local_db_and_soledad_target(test, 'test') - st.set_token_credentials('user-uuid', 'auth-token') - return db, st - - -class TestSoledadSyncTarget( - SoledadWithCouchServerMixin, - test_remote_sync_target.TestRemoteSyncTargets): - - scenarios = [ - ('token_soledad', - {'make_app_with_state': make_token_soledad_app, - 'make_document_for_test': make_leap_document_for_test, - 'create_db_and_target': make_local_db_and_token_soledad_target, - 'make_database_for_test': make_sqlcipher_database_for_test, - 'sync_target': token_leap_sync_target}), - ] - - def setUp(self): - tests.TestCaseWithServer.setUp(self) - self.main_test_class = test_remote_sync_target.TestRemoteSyncTargets - SoledadWithCouchServerMixin.setUp(self) - self.startServer() - self.db1 = make_sqlcipher_database_for_test(self, 'test1') - self.db2 = self.request_state._create_database('test2') - - def tearDown(self): - SoledadWithCouchServerMixin.tearDown(self) - tests.TestCaseWithServer.tearDown(self) - db, _ = self.request_state.ensure_database('test2') - db.delete_database() - - def test_sync_exchange_send(self): - """ - Test for sync exchanging send of document. - - This test was adapted to decrypt remote content before assert. - """ - self.startServer() - db = self.request_state._create_database('test') - remote_target = self.getSyncTarget('test') - other_docs = [] - - def receive_doc(doc, gen, trans_id): - other_docs.append((doc.doc_id, doc.rev, doc.get_json())) - - doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - new_gen, trans_id = remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=receive_doc) - self.assertEqual(1, new_gen) - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', False) - - def test_sync_exchange_send_failure_and_retry_scenario(self): - """ - Test for sync exchange failure and retry. - - This test was adapted to: - - decrypt remote content before assert. - - not expect a bounced document because soledad has stateful - recoverable sync. - """ - - self.startServer() - - def blackhole_getstderr(inst): - return cStringIO.StringIO() - - self.patch(self.server.RequestHandlerClass, 'get_stderr', - blackhole_getstderr) - db = self.request_state._create_database('test') - _put_doc_if_newer = db._put_doc_if_newer - trigger_ids = ['doc-here2'] - - def bomb_put_doc_if_newer(self, doc, save_conflict, - replica_uid=None, replica_gen=None, - replica_trans_id=None, number_of_docs=None, - doc_idx=None, sync_id=None): - if doc.doc_id in trigger_ids: - raise Exception - return _put_doc_if_newer(doc, save_conflict=save_conflict, - replica_uid=replica_uid, - replica_gen=replica_gen, - replica_trans_id=replica_trans_id, - number_of_docs=number_of_docs, - doc_idx=doc_idx, - sync_id=sync_id) - from leap.soledad.common.tests.test_couch import IndexedCouchDatabase - self.patch( - IndexedCouchDatabase, '_put_doc_if_newer', bomb_put_doc_if_newer) - remote_target = self.getSyncTarget('test') - other_changes = [] - - def receive_doc(doc, gen, trans_id): - other_changes.append( - (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) - - doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - doc2 = self.make_document('doc-here2', 'replica:1', - '{"value": "here2"}') - # We do not expect an exception here because the sync fails gracefully - remote_target.sync_exchange( - [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], - 'replica', last_known_generation=0, last_known_trans_id=None, - return_doc_cb=receive_doc) - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', - False) - self.assertEqual( - (10, 'T-sid'), db._get_replica_gen_and_trans_id('replica')) - self.assertEqual([], other_changes) - # retry - trigger_ids = [] - new_gen, trans_id = remote_target.sync_exchange( - [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=receive_doc) - self.assertGetEncryptedDoc( - db, 'doc-here2', 'replica:1', '{"value": "here2"}', - False) - self.assertEqual( - (11, 'T-sud'), db._get_replica_gen_and_trans_id('replica')) - self.assertEqual(2, new_gen) - self.assertEqual( - ('doc-here', 'replica:1', '{"value": "here"}', 1), - other_changes[0][:-1]) - - def test_sync_exchange_send_ensure_callback(self): - """ - Test for sync exchange failure and retry. - - This test was adapted to decrypt remote content before assert. - """ - self.startServer() - remote_target = self.getSyncTarget('test') - other_docs = [] - replica_uid_box = [] - - def receive_doc(doc, gen, trans_id): - other_docs.append((doc.doc_id, doc.rev, doc.get_json())) - - def ensure_cb(replica_uid): - replica_uid_box.append(replica_uid) - - doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - new_gen, trans_id = remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=receive_doc, - ensure_callback=ensure_cb) - self.assertEqual(1, new_gen) - db = self.request_state.open_database('test') - self.assertEqual(1, len(replica_uid_box)) - self.assertEqual(db._replica_uid, replica_uid_box[0]) - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', False) - - def test_sync_exchange_in_stream_error(self): - # we bypass this test because our sync_exchange process does not - # return u1db error 503 "unavailable" for now. - pass - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_https`. -#----------------------------------------------------------------------------- - -def token_leap_https_sync_target(test, host, path): - _, port = test.server.server_address - st = target.SoledadSyncTarget( - 'https://%s:%d/%s' % (host, port, path), - crypto=test._soledad._crypto) - st.set_token_credentials('user-uuid', 'auth-token') - return st - - -class TestSoledadSyncTargetHttpsSupport( - test_https.TestHttpSyncTargetHttpsSupport, - BaseSoledadTest): - - scenarios = [ - ('token_soledad_https', - {'server_def': test_https.https_server_def, - 'make_app_with_state': make_token_soledad_app, - 'make_document_for_test': make_leap_document_for_test, - 'sync_target': token_leap_https_sync_target}), - ] - - def setUp(self): - # the parent constructor undoes our SSL monkey patch to ensure tests - # run smoothly with standard u1db. - test_https.TestHttpSyncTargetHttpsSupport.setUp(self) - # so here monkey patch again to test our functionality. - http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection - client.SOLEDAD_CERT = http_client.CA_CERTS - - def test_working(self): - """ - Test that SSL connections work well. - - This test was adapted to patch Soledad's HTTPS connection custom class - with the intended CA certificates. - """ - self.startServer() - db = self.request_state._create_database('test') - self.patch(client, 'SOLEDAD_CERT', self.cacert_pem) - remote_target = self.getSyncTarget('localhost', 'test') - remote_target.record_sync_info('other-id', 2, 'T-id') - self.assertEqual( - (2, 'T-id'), db._get_replica_gen_and_trans_id('other-id')) - - def test_host_mismatch(self): - """ - Test that SSL connections to a hostname different than the one in the - certificate raise CertificateError. - - This test was adapted to patch Soledad's HTTPS connection custom class - with the intended CA certificates. - """ - self.startServer() - self.request_state._create_database('test') - self.patch(client, 'SOLEDAD_CERT', self.cacert_pem) - remote_target = self.getSyncTarget('127.0.0.1', 'test') - self.assertRaises( - http_client.CertificateError, remote_target.record_sync_info, - 'other-id', 2, 'T-id') - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_http_database`. -#----------------------------------------------------------------------------- - -class _HTTPDatabase(http_database.HTTPDatabase, auth.TokenBasedAuth): - """ - Wraps our token auth implementation. - """ - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - -class TestHTTPDatabaseWithCreds( - test_http_database.TestHTTPDatabaseCtrWithCreds): - - def test_get_sync_target_inherits_token_credentials(self): - # this test was from TestDatabaseSimpleOperations but we put it here - # for convenience. - self.db = _HTTPDatabase('dbase') - self.db.set_token_credentials('user-uuid', 'auth-token') - st = self.db.get_sync_target() - self.assertEqual(self.db._creds, st._creds) - - def test_ctr_with_creds(self): - db1 = _HTTPDatabase('http://dbs/db', creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - self.assertIn('token', db1._creds) - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -#----------------------------------------------------------------------------- - -target_scenarios = [ - ('token_leap', {'create_db_and_target': - make_local_db_and_token_soledad_target, - 'make_app_with_state': make_soledad_app}), -] - - -class SoledadDatabaseSyncTargetTests( - SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests): - - scenarios = ( - tests.multiply_scenarios( - tests.DatabaseBaseTests.scenarios, - target_scenarios)) - - whitebox = False - - def setUp(self): - self.main_test_class = test_sync.DatabaseSyncTargetTests - SoledadWithCouchServerMixin.setUp(self) - - def test_sync_exchange(self): - """ - Test sync exchange. - - This test was adapted to decrypt remote content before assert. - """ - sol, _ = make_local_db_and_soledad_target(self) - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, - 'T-sid')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertTransactionLog(['doc-id'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, last_trans_id)) - self.assertEqual(10, self.st.get_sync_info('replica')[3]) - sol.close() - - def test_sync_exchange_push_many(self): - """ - Test sync exchange. - - This test was adapted to decrypt remote content before assert. - """ - docs_by_gen = [ - (self.make_document( - 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), - (self.make_document( - 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, 'doc-id2', 'replica:1', tests.nested_doc, False) - self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(11, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_returns_many_new_docs(self): - """ - Test sync exchange. - - This test was adapted to avoid JSON serialization comparison as local - and remote representations might differ. It looks directly at the - doc's contents instead. - """ - doc = self.db.create_doc_from_json(tests.simple_doc) - doc2 = self.db.create_doc_from_json(tests.nested_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - self.assertEqual(2, new_gen) - self.assertEqual( - [(doc.doc_id, doc.rev, 1), - (doc2.doc_id, doc2.rev, 2)], - [c[:-3] + c[-2:-1] for c in self.other_changes]) - self.assertEqual( - json.loads(tests.simple_doc), - json.loads(self.other_changes[0][2])) - self.assertEqual( - json.loads(tests.nested_doc), - json.loads(self.other_changes[1][2])) - if self.whitebox: - self.assertEqual( - self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': - [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) - - -class TestSoledadDbSync( - SoledadWithCouchServerMixin, test_sync.TestDbSync): - """Test db.sync remote sync shortcut""" - - scenarios = [ - ('py-token-http', { - 'create_db_and_target': make_local_db_and_token_soledad_target, - 'make_app_with_state': make_token_soledad_app, - 'make_database_for_test': make_sqlcipher_database_for_test, - 'token': True - }), - ] - - oauth = False - token = False - - def setUp(self): - self.main_test_class = test_sync.TestDbSync - SoledadWithCouchServerMixin.setUp(self) - - def do_sync(self, target_name): - """ - Perform sync using SoledadSyncTarget and Token auth. - """ - if self.token: - extra = dict(creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - target_url = self.getURL(target_name) - return Synchronizer( - self.db, - target.SoledadSyncTarget( - target_url, - crypto=self._soledad._crypto, - **extra)).sync(autocreate=True) - else: - return test_sync.TestDbSync.do_sync(self, target_name) - - def test_db_sync(self): - """ - Test sync. - - Adapted to check for encrypted content. - """ - doc1 = self.db.create_doc_from_json(tests.simple_doc) - doc2 = self.db2.create_doc_from_json(tests.nested_doc) - local_gen_before_sync = self.do_sync('test2') - gen, _, changes = self.db.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False) - - def test_db_sync_autocreate(self): - """ - We bypass this test because we never need to autocreate databases. - """ - pass - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_target_soledad.py b/common/src/leap/soledad/common/tests/test_target_soledad.py deleted file mode 100644 index 899203b8..00000000 --- a/common/src/leap/soledad/common/tests/test_target_soledad.py +++ /dev/null @@ -1,102 +0,0 @@ -from u1db.remote import ( - http_database, -) - -from leap.soledad.client import ( - auth, - VerifiedHTTPSConnection, -) -from leap.soledad.common.document import SoledadDocument -from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.tests.u1db_tests import test_backends - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -#----------------------------------------------------------------------------- - -def make_leap_document_for_test(test, doc_id, rev, content, - has_conflicts=False): - return SoledadDocument( - doc_id, rev, content, has_conflicts=has_conflicts) - - -def make_soledad_app(state): - return SoledadApp(state) - - -def make_token_soledad_app(state): - app = SoledadApp(state) - - def _verify_authentication_data(uuid, auth_data): - if uuid == 'user-uuid' and auth_data == 'auth-token': - return True - return False - - # we test for action authorization in leap.soledad.common.tests.test_server - def _verify_authorization(uuid, environ): - return True - - application = SoledadTokenAuthMiddleware(app) - application._verify_authentication_data = _verify_authentication_data - application._verify_authorization = _verify_authorization - return application - - -LEAP_SCENARIOS = [ - ('http', { - 'make_database_for_test': test_backends.make_http_database_for_test, - 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': make_leap_document_for_test, - 'make_app_with_state': make_soledad_app}), -] - - -def make_token_http_database_for_test(test, replica_uid): - test.startServer() - test.request_state._create_database(replica_uid) - - class _HTTPDatabaseWithToken( - http_database.HTTPDatabase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - http_db = _HTTPDatabaseWithToken(test.getURL('test')) - http_db.set_token_credentials('user-uuid', 'auth-token') - return http_db - - -def copy_token_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - http_db = test.request_state._copy_database(db) - http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') - return http_db - - -class SoledadTests(test_backends.AllDatabaseTests, BaseSoledadTest): - - scenarios = LEAP_SCENARIOS + [ - ('token_http', {'make_database_for_test': - make_token_http_database_for_test, - 'copy_database_for_test': - copy_token_http_database_for_test, - 'make_document_for_test': make_leap_document_for_test, - 'make_app_with_state': make_token_soledad_app, - }) - ] - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py index ad66fb06..6efeb87f 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py @@ -35,7 +35,7 @@ from pysqlcipher import dbapi2 from StringIO import StringIO import testscenarios -import testtools +from twisted.trial import unittest from u1db import ( errors, @@ -50,7 +50,7 @@ from u1db.remote import ( ) -class TestCase(testtools.TestCase): +class TestCase(unittest.TestCase): def createTempDir(self, prefix='u1db-tmp-'): """Create a temporary directory to do some work in. diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py index 86e76fad..27fc50dc 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py @@ -40,6 +40,8 @@ from u1db.remote import ( http_database, ) +from unittest import skip + def make_http_database_for_test(test, replica_uid, path='test', *args): test.startServer() @@ -79,6 +81,7 @@ class TestAlternativeDocument(DocumentBase): """A (not very) alternative implementation of Document.""" +@skip("Skiping tests imported from U1DB.") class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): scenarios = tests.LOCAL_DATABASES_SCENARIOS + [ @@ -327,6 +330,7 @@ class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) +@skip("Skiping tests imported from U1DB.") class DocumentSizeTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS @@ -351,6 +355,7 @@ class DocumentSizeTests(tests.DatabaseBaseTests): self.assertEqual(1000000, self.db.document_size_limit) +@skip("Skiping tests imported from U1DB.") class LocalDatabaseTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS @@ -363,6 +368,7 @@ class LocalDatabaseTests(tests.DatabaseBaseTests): db2 = self.create_database('other-uid') doc2 = db2.create_doc_from_json(simple_doc) self.assertNotEqual(doc1.doc_id, doc2.doc_id) + db2.close() def test_put_doc_refuses_slashes_picky(self): doc = self.make_document('/a', None, simple_doc) @@ -608,6 +614,7 @@ class LocalDatabaseTests(tests.DatabaseBaseTests): self.db.whats_changed(2)) +@skip("Skiping tests imported from U1DB.") class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS @@ -632,6 +639,7 @@ class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests): self.db.validate_gen_and_trans_id, gen + 1, trans_id) +@skip("Skiping tests imported from U1DB.") class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS @@ -651,6 +659,7 @@ class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests): self.db._validate_source, 'other', 1, 'T-sad') +@skip("Skiping tests imported from U1DB.") class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests): # test supporting/functionality around storing conflicts @@ -1027,6 +1036,7 @@ class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests): self.assertRaises(errors.ConflictedDoc, self.db.delete_doc, doc2) +@skip("Skiping tests imported from U1DB.") class DatabaseIndexTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS @@ -1833,6 +1843,7 @@ class DatabaseIndexTests(tests.DatabaseBaseTests): self.assertParseError('combine(lower(x)x,foo)') +@skip("Skiping tests imported from U1DB.") class PythonBackendTests(tests.DatabaseBaseTests): def setUp(self): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py index 8b30ed51..d8a27f51 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py @@ -15,11 +15,13 @@ # along with u1db. If not, see <http://www.gnu.org/licenses/>. +from unittest import skip from u1db import errors from leap.soledad.common.tests import u1db_tests as tests +@skip("Skiping tests imported from U1DB.") class TestDocument(tests.TestCase): scenarios = ([( @@ -83,6 +85,7 @@ class TestDocument(tests.TestCase): self.assertEqual(len('a' + 'b'), doc_a.get_size()) +@skip("Skiping tests imported from U1DB.") class TestPyDocument(tests.TestCase): scenarios = ([( diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py index 789006ba..522eb476 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py @@ -24,6 +24,8 @@ except ImportError: import json # noqa import StringIO +from unittest import skip + from u1db import ( __version__ as _u1db_version, errors, @@ -38,6 +40,7 @@ from u1db.remote import ( ) +@skip("Skiping tests imported from U1DB.") class TestFencedReader(tests.TestCase): def test_init(self): @@ -145,6 +148,7 @@ class TestFencedReader(tests.TestCase): self.assertRaises(http_app.BadRequest, reader.getline) +@skip("Skiping tests imported from U1DB.") class TestHTTPMethodDecorator(tests.TestCase): def test_args(self): @@ -253,6 +257,7 @@ class parameters: max_entry_size = 100000 +@skip("Skiping tests imported from U1DB.") class TestHTTPInvocationByMethodWithBody(tests.TestCase): def test_get(self): @@ -433,6 +438,7 @@ class TestHTTPInvocationByMethodWithBody(tests.TestCase): self.assertRaises(http_app.BadRequest, invoke) +@skip("Skiping tests imported from U1DB.") class TestHTTPResponder(tests.TestCase): def start_response(self, status, headers): @@ -521,6 +527,7 @@ class TestHTTPResponder(tests.TestCase): responder.content) +@skip("Skiping tests imported from U1DB.") class TestHTTPApp(tests.TestCase): def setUp(self): @@ -949,6 +956,7 @@ class TestHTTPApp(tests.TestCase): self.assertEqual({'error': 'unavailable'}, json.loads(parts[2])) +@skip("Skiping tests imported from U1DB.") class TestRequestHooks(tests.TestCase): def setUp(self): @@ -1000,12 +1008,14 @@ class TestRequestHooks(tests.TestCase): self.assertEqual(['begin', 'bad-request'], calls) +@skip("Skiping tests imported from U1DB.") class TestHTTPErrors(tests.TestCase): def test_wire_description_to_status(self): self.assertNotIn("error", http_errors.wire_description_to_status) +@skip("Skiping tests imported from U1DB.") class TestHTTPAppErrorHandling(tests.TestCase): def setUp(self): @@ -1113,6 +1123,7 @@ class TestHTTPAppErrorHandling(tests.TestCase): self.assertEqual(self.exc, exc) +@skip("Skiping tests imported from U1DB.") class TestPluggableSyncExchange(tests.TestCase): def setUp(self): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py index 08e9714e..f9e09cbd 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py @@ -26,6 +26,8 @@ from u1db import ( errors, ) +from unittest import skip + from leap.soledad.common.tests import u1db_tests as tests from u1db.remote import ( @@ -33,6 +35,7 @@ from u1db.remote import ( ) +@skip("Skiping tests imported from U1DB.") class TestEncoder(tests.TestCase): def test_encode_string(self): @@ -45,6 +48,7 @@ class TestEncoder(tests.TestCase): self.assertEqual("false", http_client._encode_query_parameter(False)) +@skip("Skiping tests imported from U1DB.") class TestHTTPClientBase(tests.TestCaseWithServer): def setUp(self): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py index 9251000e..bf7ed5d3 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py @@ -27,6 +27,8 @@ from u1db import ( Document, ) +from unittest import skip + from leap.soledad.common.tests import u1db_tests as tests from u1db.remote import ( @@ -38,6 +40,7 @@ from leap.soledad.common.tests.u1db_tests.test_remote_sync_target import ( ) +@skip("Skiping tests imported from U1DB.") class TestHTTPDatabaseSimpleOperations(tests.TestCase): def setUp(self): @@ -190,6 +193,7 @@ class TestHTTPDatabaseSimpleOperations(tests.TestCase): self.assertEqual(self.db._creds, st._creds) +@skip("Skiping tests imported from U1DB.") class TestHTTPDatabaseCtrWithCreds(tests.TestCase): def test_ctr_with_creds(self): @@ -202,6 +206,7 @@ class TestHTTPDatabaseCtrWithCreds(tests.TestCase): self.assertIn('oauth', db1._creds) +@skip("Skiping tests imported from U1DB.") class TestHTTPDatabaseIntegration(tests.TestCaseWithServer): make_app_with_state = staticmethod(make_http_app) diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py index c086fbc0..cea175d6 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py @@ -5,6 +5,7 @@ import ssl import sys from paste import httpserver +from unittest import skip from u1db.remote import ( http_client, @@ -51,6 +52,7 @@ def oauth_https_sync_target(test, host, path): return st +@skip("Skiping tests imported from U1DB.") class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): scenarios = [ @@ -75,7 +77,7 @@ class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): # order to maintain the compatibility with u1db default tests, we undo # that replacement here. http_client._VerifiedHTTPSConnection = \ - soledad.client.old__VerifiedHTTPSConnection + soledad.client.api.old__VerifiedHTTPSConnection super(TestHttpSyncTargetHttpsSupport, self).setUp() def getSyncTarget(self, host, path=None): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py index 63406245..ee249e6e 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py @@ -22,12 +22,14 @@ from u1db import ( errors, open as u1db_open, ) +from unittest import skip from leap.soledad.common.tests import u1db_tests as tests from u1db.backends import sqlite_backend from leap.soledad.common.tests.u1db_tests.test_backends \ import TestAlternativeDocument +@skip("Skiping tests imported from U1DB.") class TestU1DBOpen(tests.TestCase): def setUp(self): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py b/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py index 3793e0df..bd7e4103 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py @@ -22,6 +22,8 @@ from u1db import ( errors, ) +from unittest import skip + from leap.soledad.common.tests import u1db_tests as tests from u1db.remote import ( @@ -31,6 +33,7 @@ from u1db.remote import ( ) +@skip("Skiping tests imported from U1DB.") class TestHTTPSyncTargetBasics(tests.TestCase): def test_parse_url(self): @@ -41,6 +44,7 @@ class TestHTTPSyncTargetBasics(tests.TestCase): self.assertEqual('/', remote_target._url.path) +@skip("Skiping tests imported from U1DB.") class TestParsingSyncStream(tests.TestCase): def test_wrong_start(self): @@ -130,6 +134,7 @@ def oauth_http_sync_target(test, path): return st +@skip("Skiping tests imported from U1DB.") class TestRemoteSyncTargets(tests.TestCaseWithServer): scenarios = [ diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py index 8292dd07..aed8a6e5 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py @@ -27,6 +27,8 @@ from u1db import ( query_parser, ) +from unittest import skip + from leap.soledad.common.tests import u1db_tests as tests from u1db.backends import sqlite_backend @@ -38,6 +40,7 @@ simple_doc = '{"key": "value"}' nested_doc = '{"key": "value", "sub": {"doc": "underneath"}}' +@skip("Skiping tests imported from U1DB.") class TestSQLiteDatabase(tests.TestCase): def test_atomic_initialize(self): @@ -83,6 +86,7 @@ class TestSQLiteDatabase(tests.TestCase): self.assertTrue(db2._is_initialized(db1._get_sqlite_handle().cursor())) +@skip("Skiping tests imported from U1DB.") class TestSQLitePartialExpandDatabase(tests.TestCase): def setUp(self): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py index 5e2bec86..bac1f177 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py @@ -26,6 +26,8 @@ from u1db import ( SyncTarget, ) +from unittest import skip + from leap.soledad.common.tests import u1db_tests as tests from u1db.backends import ( @@ -74,6 +76,7 @@ target_scenarios = [ ] +@skip("Skiping tests imported from U1DB.") class DatabaseSyncTargetTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): @@ -462,6 +465,7 @@ sync_scenarios.append(('pyhttp', { })) +@skip("Skiping tests imported from U1DB.") class DatabaseSyncTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): @@ -1118,6 +1122,7 @@ class DatabaseSyncTests(tests.DatabaseBaseTests, errors.InvalidTransactionId, self.sync, self.db1, self.db2_copy) +@skip("Skiping tests imported from U1DB.") class TestDbSync(tests.TestCaseWithServer): """Test db.sync remote sync shortcut""" @@ -1190,6 +1195,7 @@ class TestDbSync(tests.TestCaseWithServer): self.assertEqual(1, s_gen) +@skip("Skiping tests imported from U1DB.") class TestRemoteSyncIntegration(tests.TestCaseWithServer): """Integration tests for the most common sync scenario local -> remote""" diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py index 249cbdaa..17ed3855 100644 --- a/common/src/leap/soledad/common/tests/util.py +++ b/common/src/leap/soledad/common/tests/util.py @@ -21,33 +21,55 @@ Utilities used by multiple test suites. """ +import os import tempfile import shutil +import random +import string +import u1db +import subprocess +import time +import re +import traceback + +from mock import Mock from urlparse import urljoin - from StringIO import StringIO from pysqlcipher import dbapi2 + from u1db.errors import DatabaseDoesNotExist +from u1db.remote import http_database + +from twisted.trial import unittest +from leap.common.files import mkdir_p from leap.soledad.common import soledad_assert +from leap.soledad.common.document import SoledadDocument from leap.soledad.common.couch import CouchDatabase, CouchServerState -from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware +from leap.soledad.common.crypto import ENC_SCHEME_KEY +from leap.soledad.client import Soledad +from leap.soledad.client import target +from leap.soledad.client import auth +from leap.soledad.client.crypto import decrypt_doc_dict -from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest -from leap.soledad.common.tests.test_couch import CouchDBWrapper, CouchDBTestCase - +from leap.soledad.server import SoledadApp +from leap.soledad.server.auth import SoledadTokenAuthMiddleware -from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client.sqlcipher import ( + SQLCipherDatabase, + SQLCipherOptions, +) PASSWORD = '123456' +ADDRESS = 'leap@leap.se' def make_sqlcipher_database_for_test(test, replica_uid): - db = SQLCipherDatabase(':memory:', PASSWORD) + db = SQLCipherDatabase( + SQLCipherOptions(':memory:', PASSWORD)) db._set_replica_uid(replica_uid) return db @@ -58,7 +80,7 @@ def copy_sqlcipher_database_for_test(test, db): # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR # HOUSE. - new_db = SQLCipherDatabase(':memory:', PASSWORD) + new_db = make_sqlcipher_database_for_test(test, None) tmpfile = StringIO() for line in db._db_handle.iterdump(): if not 'sqlite_sequence' in line: # work around bug in iterdump @@ -94,6 +116,324 @@ def make_token_soledad_app(state): return application +def make_soledad_document_for_test(test, doc_id, rev, content, + has_conflicts=False): + return SoledadDocument( + doc_id, rev, content, has_conflicts=has_conflicts) + + +def make_token_http_database_for_test(test, replica_uid): + test.startServer() + test.request_state._create_database(replica_uid) + + class _HTTPDatabaseWithToken( + http_database.HTTPDatabase, auth.TokenBasedAuth): + + def set_token_credentials(self, uuid, token): + auth.TokenBasedAuth.set_token_credentials(self, uuid, token) + + def _sign_request(self, method, url_query, params): + return auth.TokenBasedAuth._sign_request( + self, method, url_query, params) + + http_db = _HTTPDatabaseWithToken(test.getURL('test')) + http_db.set_token_credentials('user-uuid', 'auth-token') + return http_db + + +def copy_token_http_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + http_db = test.request_state._copy_database(db) + http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') + return http_db + + +class MockedSharedDBTest(object): + + def get_default_shared_mock(self, put_doc_side_effect=None, + get_doc_return_value=None): + """ + Get a default class for mocking the shared DB + """ + class defaultMockSharedDB(object): + get_doc = Mock(return_value=get_doc_return_value) + put_doc = Mock(side_effect=put_doc_side_effect) + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) + open = Mock(return_value=None) + syncable = True + + def __call__(self): + return self + return defaultMockSharedDB + + +def soledad_sync_target(test, path): + return target.SoledadSyncTarget( + test.getURL(path), crypto=test._soledad._crypto) + + +def token_soledad_sync_target(test, path): + st = soledad_sync_target(test, path) + st.set_token_credentials('user-uuid', 'auth-token') + return st + + +class BaseSoledadTest(unittest.TestCase, MockedSharedDBTest): + """ + Instantiates Soledad for usage in tests. + """ + defer_sync_encryption = False + + def setUp(self): + # The following snippet comes from BaseLeapTest.setUpClass, but we + # repeat it here because twisted.trial does not work with + # setUpClass/tearDownClass. + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + + # config info + self.db1_file = os.path.join(self.tempdir, "db1.u1db") + self.db2_file = os.path.join(self.tempdir, "db2.u1db") + self.email = ADDRESS + # open test dbs + self._db1 = u1db.open(self.db1_file, create=True, + document_factory=SoledadDocument) + self._db2 = u1db.open(self.db2_file, create=True, + document_factory=SoledadDocument) + # get a random prefix for each test, so we do not mess with + # concurrency during initialization and shutting down of + # each local db. + self.rand_prefix = ''.join( + map(lambda x: random.choice(string.ascii_letters), range(6))) + # initialize soledad by hand so we can control keys + # XXX check if this soledad is actually used + self._soledad = self._soledad_instance( + prefix=self.rand_prefix, user=self.email) + + def tearDown(self): + self._db1.close() + self._db2.close() + self._soledad.close() + + # restore paths + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + + def _delete_temporary_dirs(): + # XXX should not access "private" attrs + for f in [self._soledad.local_db_path, + self._soledad.secrets.secrets_path]: + if os.path.isfile(f): + os.unlink(f) + # The following snippet comes from BaseLeapTest.setUpClass, but we + # repeat it here because twisted.trial does not work with + # setUpClass/tearDownClass. + soledad_assert( + self.tempdir.startswith('/tmp/leap_tests-'), + "beware! tried to remove a dir which does not " + "live in temporal folder!") + shutil.rmtree(self.tempdir) + + from twisted.internet import reactor + reactor.addSystemEventTrigger( + "after", "shutdown", _delete_temporary_dirs) + + + def _soledad_instance(self, user=ADDRESS, passphrase=u'123', + prefix='', + secrets_path='secrets.json', + local_db_path='soledad.u1db', + server_url='https://127.0.0.1/', + cert_file=None, + shared_db_class=None, + auth_token='auth-token'): + + def _put_doc_side_effect(doc): + self._doc_put = doc + + if shared_db_class is not None: + MockSharedDB = shared_db_class + else: + MockSharedDB = self.get_default_shared_mock( + _put_doc_side_effect) + + return Soledad( + user, + passphrase, + secrets_path=os.path.join( + self.tempdir, prefix, secrets_path), + local_db_path=os.path.join( + self.tempdir, prefix, local_db_path), + server_url=server_url, # Soledad will fail if not given an url. + cert_file=cert_file, + defer_encryption=self.defer_sync_encryption, + shared_db=MockSharedDB(), + auth_token=auth_token) + + def assertGetEncryptedDoc( + self, db, doc_id, doc_rev, content, has_conflicts): + """ + Assert that the document in the database looks correct. + """ + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + doc = db.get_doc(doc_id) + + if ENC_SCHEME_KEY in doc.content: + # XXX check for SYM_KEY too + key = self._soledad._crypto.doc_passphrase(doc.doc_id) + secret = self._soledad._crypto.secret + decrypted = decrypt_doc_dict( + doc.content, doc.doc_id, doc.rev, + key, secret) + doc.set_json(decrypted) + self.assertEqual(exp_doc.doc_id, doc.doc_id) + self.assertEqual(exp_doc.rev, doc.rev) + self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts) + self.assertEqual(exp_doc.content, doc.content) + + +#----------------------------------------------------------------------------- +# A wrapper for running couchdb locally. +#----------------------------------------------------------------------------- + +# from: https://github.com/smcq/paisley/blob/master/paisley/test/util.py +# TODO: include license of above project. +class CouchDBWrapper(object): + """ + Wrapper for external CouchDB instance which is started and stopped for + testing. + """ + BOOT_TIMEOUT_SECONDS = 5 + RETRY_LIMIT = 3 + + def start(self): + tries = 0 + while tries < self.RETRY_LIMIT and not hasattr(self, 'port'): + try: + self._try_start() + return + except Exception, e: + print traceback.format_exc() + self.stop() + tries += 1 + raise Exception("Check your couchdb: Tried to start 3 times and failed badly") + + def _try_start(self): + """ + Start a CouchDB instance for a test. + """ + self.tempdir = tempfile.mkdtemp(suffix='.couch.test') + + path = os.path.join(os.path.dirname(__file__), + 'couchdb.ini.template') + handle = open(path) + conf = handle.read() % { + 'tempdir': self.tempdir, + } + handle.close() + + shutil.copy('/etc/couchdb/default.ini', self.tempdir) + defaultConfPath = os.path.join(self.tempdir, 'default.ini') + + confPath = os.path.join(self.tempdir, 'test.ini') + handle = open(confPath, 'w') + handle.write(conf) + handle.close() + + # create the dirs from the template + mkdir_p(os.path.join(self.tempdir, 'lib')) + mkdir_p(os.path.join(self.tempdir, 'log')) + args = ['/usr/bin/couchdb', '-n', '-a', defaultConfPath, '-a', confPath] + null = open('/dev/null', 'w') + + self.process = subprocess.Popen( + args, env=None, stdout=null.fileno(), stderr=null.fileno(), + close_fds=True) + boot_time = time.time() + # find port + logPath = os.path.join(self.tempdir, 'log', 'couch.log') + while not os.path.exists(logPath): + if self.process.poll() is not None: + got_stdout, got_stderr = "", "" + if self.process.stdout is not None: + got_stdout = self.process.stdout.read() + + if self.process.stderr is not None: + got_stderr = self.process.stderr.read() + raise Exception(""" +couchdb exited with code %d. +stdout: +%s +stderr: +%s""" % ( + self.process.returncode, got_stdout, got_stderr)) + time.sleep(0.01) + if (time.time() - boot_time) > self.BOOT_TIMEOUT_SECONDS: + self.stop() + raise Exception("Timeout starting couch") + while os.stat(logPath).st_size == 0: + time.sleep(0.01) + if (time.time() - boot_time) > self.BOOT_TIMEOUT_SECONDS: + self.stop() + raise Exception("Timeout starting couch") + PORT_RE = re.compile( + 'Apache CouchDB has started on http://127.0.0.1:(?P<port>\d+)') + + handle = open(logPath) + line = handle.read() + handle.close() + m = PORT_RE.search(line) + if not m: + self.stop() + raise Exception("Cannot find port in line %s" % line) + self.port = int(m.group('port')) + + def stop(self): + """ + Terminate the CouchDB instance. + """ + try: + self.process.terminate() + self.process.communicate() + except: + # just to clean up + # if it can't, the process wasn't created anyway + pass + shutil.rmtree(self.tempdir) + + +class CouchDBTestCase(unittest.TestCase, MockedSharedDBTest): + """ + TestCase base class for tests against a real CouchDB server. + """ + + def setUp(self): + """ + Make sure we have a CouchDB instance for a test. + """ + self.wrapper = CouchDBWrapper() + self.wrapper.start() + #self.db = self.wrapper.db + + def tearDown(self): + """ + Stop CouchDB instance for test. + """ + self.wrapper.stop() + class CouchServerStateForTests(CouchServerState): """ This is a slightly modified CouchDB server state that allows for creating @@ -122,43 +462,15 @@ class SoledadWithCouchServerMixin( BaseSoledadTest, CouchDBTestCase): - @classmethod - def setUpClass(cls): - """ - Make sure we have a CouchDB instance for a test. - """ - # from BaseLeapTest - cls.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - # from CouchDBTestCase - cls.wrapper = CouchDBWrapper() - cls.wrapper.start() - #self.db = self.wrapper.db - - @classmethod - def tearDownClass(cls): - """ - Stop CouchDB instance for test. - """ - # from BaseLeapTest - soledad_assert( - cls.tempdir.startswith('/tmp/leap_tests-'), - "beware! tried to remove a dir which does not " - "live in temporal folder!") - shutil.rmtree(cls.tempdir) - # from CouchDBTestCase - cls.wrapper.stop() - def setUp(self): - BaseSoledadTest.setUp(self) CouchDBTestCase.setUp(self) + BaseSoledadTest.setUp(self) main_test_class = getattr(self, 'main_test_class', None) if main_test_class is not None: main_test_class.setUp(self) self._couch_url = 'http://localhost:%d' % self.wrapper.port def tearDown(self): - BaseSoledadTest.tearDown(self) - CouchDBTestCase.tearDown(self) main_test_class = getattr(self, 'main_test_class', None) if main_test_class is not None: main_test_class.tearDown(self) @@ -168,10 +480,11 @@ class SoledadWithCouchServerMixin( db.delete_database() except DatabaseDoesNotExist: pass + BaseSoledadTest.tearDown(self) + CouchDBTestCase.tearDown(self) def make_app(self): couch_url = urljoin( 'http://localhost:' + str(self.wrapper.port), 'tests') - self.request_state = CouchServerStateForTests( - couch_url, 'shared', 'tokens') + self.request_state = CouchServerStateForTests(couch_url) return self.make_app_with_state(self.request_state) diff --git a/docs/debian-repackaging.rst b/docs/debian-repackaging.rst index a7488a84..888d6c03 100644 --- a/docs/debian-repackaging.rst +++ b/docs/debian-repackaging.rst @@ -6,7 +6,7 @@ How to repackage latest code Enter debian branch:: - git checkout debian + git checkout debian/experimental Merge your latest and greatest:: diff --git a/docs/leap-commit-template b/docs/leap-commit-template new file mode 100644 index 00000000..8a5c7cd0 --- /dev/null +++ b/docs/leap-commit-template @@ -0,0 +1,7 @@ +[bug|feat|docs|style|refactor|test|pkg|i18n] ... +... + +- Resolves: #XYZ +- Related: #XYZ +- Documentation: #XYZ +- Releases: XYZ diff --git a/docs/leap-commit-template.README b/docs/leap-commit-template.README new file mode 100644 index 00000000..ce8809e7 --- /dev/null +++ b/docs/leap-commit-template.README @@ -0,0 +1,47 @@ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +HOW TO USE THIS TEMPLATE: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Run `git config commit.template docs/leap-commit-template` or +edit the .git/config for this project and add +`template = docs/leap-commit-template` +under the [commit] block + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +COMMIT TEMPLATE FORMAT EXPLAINED +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +[type] <subject> + +<body> +<footer> + +Type should be one of the following: +- bug (bug fix) +- feat (new feature) +- docs (changes to documentation) +- style (formatting, pep8 violations, etc; no code change) +- refactor (refactoring production code) +- test (adding missing tests, refactoring tests; no production code change) +- pkg (packaging related changes; no production code change) +- i18n translation related changes + +Subject should use imperative tone and say what you did. +For example, use 'change', NOT 'changed' or 'changes'. + +The body should go into detail about changes made. + +The footer should contain any issue references or actions. +You can use one or several of the following: + +- Resolves: #XYZ +- Related: #XYZ +- Documentation: #XYZ +- Releases: XYZ + +The Documentation field should be included in every new feature commit, and it +should link to an issue in the bug tracker where the new feature is analyzed +and documented. + +For a full example of how to write a good commit message, check out +https://github.com/sparkbox/how_to/tree/master/style/git diff --git a/scripts/build_debian_package.sh b/scripts/build_debian_package.sh index 1ec9b00a..b9fb93a9 100755 --- a/scripts/build_debian_package.sh +++ b/scripts/build_debian_package.sh @@ -26,7 +26,7 @@ export GIT_DIR=${workdir}/soledad/.git export GIT_WORK_TREE=${workdir}/soledad git remote add leapcode ${SOLEDAD_MAIN_REPO} git fetch leapcode -git checkout -b debian leapcode/debian +git checkout -b debian/experimental leapcode/debian/experimental git merge --no-edit ${branch} (cd ${workdir}/soledad && debuild -uc -us) echo "Packages generated in ${workdir}" diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index 6c456c41..1d8d32e2 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -2,23 +2,22 @@ # This script gives client-side access to one Soledad user database. - -import sys import os import argparse -import re import tempfile import getpass import requests -import json import srp._pysrp as srp import binascii import logging +import json +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks -from leap.common.config import get_path_prefix from leap.soledad.client import Soledad - +from leap.keymanager import KeyManager +from leap.keymanager.openpgp import OpenPGPKey from util import ValidateUserHandle @@ -26,37 +25,37 @@ from util import ValidateUserHandle # create a logger logger = logging.getLogger(__name__) LOG_FORMAT = '%(asctime)s %(message)s' -logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) +logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) safe_unhexlify = lambda x: binascii.unhexlify(x) if ( len(x) % 2 == 0) else binascii.unhexlify('0' + x) -def fail(reason): +def _fail(reason): logger.error('Fail: ' + reason) exit(2) -def get_api_info(provider): +def _get_api_info(provider): info = requests.get( 'https://'+provider+'/provider.json', verify=False).json() return info['api_uri'], info['api_version'] -def login(username, passphrase, provider, api_uri, api_version): +def _login(username, passphrase, provider, api_uri, api_version): usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024) auth = None try: - auth = authenticate(api_uri, api_version, usr).json() + auth = _authenticate(api_uri, api_version, usr).json() except requests.exceptions.ConnectionError: - fail('Could not connect to server.') + _fail('Could not connect to server.') if 'errors' in auth: - fail(str(auth['errors'])) + _fail(str(auth['errors'])) return api_uri, api_version, auth -def authenticate(api_uri, api_version, usr): +def _authenticate(api_uri, api_version, usr): api_url = "%s/%s" % (api_uri, api_version) session = requests.session() uname, A = usr.start_authentication() @@ -64,16 +63,16 @@ def authenticate(api_uri, api_version, usr): init = session.post( api_url + '/sessions', data=params, verify=False).json() if 'errors' in init: - fail('test user not found') + _fail('test user not found') M = usr.process_challenge( safe_unhexlify(init['salt']), safe_unhexlify(init['B'])) return session.put(api_url + '/sessions/' + uname, verify=False, data={'client_auth': binascii.hexlify(M)}) -def get_soledad_info(username, provider, passphrase, basedir): - api_uri, api_version = get_api_info(provider) - auth = login(username, passphrase, provider, api_uri, api_version) +def _get_soledad_info(username, provider, passphrase, basedir): + api_uri, api_version = _get_api_info(provider) + auth = _login(username, passphrase, provider, api_uri, api_version) # get soledad server url service_url = '%s/%s/config/soledad-service.json' % \ (api_uri, api_version) @@ -101,10 +100,9 @@ def get_soledad_info(username, provider, passphrase, basedir): return auth[2]['id'], server_url, cert_file, auth[2]['token'] -def get_soledad_instance(username, provider, passphrase, basedir): +def _get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file, + token): # setup soledad info - uuid, server_url, cert_file, token = \ - get_soledad_info(username, provider, passphrase, basedir) logger.info('UUID is %s' % uuid) logger.info('Server URL is %s' % server_url) secrets_path = os.path.join( @@ -119,37 +117,135 @@ def get_soledad_instance(username, provider, passphrase, basedir): local_db_path=local_db_path, server_url=server_url, cert_file=cert_file, - auth_token=token) - - -# main program - -if __name__ == '__main__': - + auth_token=token, + defer_encryption=False) + + +def _get_keymanager_instance(username, provider, soledad, token, + ca_cert_path=None, api_uri=None, api_version=None, uid=None, + gpgbinary=None): + return KeyManager( + "{username}@{provider}".format(username=username, provider=provider), + "http://uri", + soledad, + token=token, + ca_cert_path=ca_cert_path, + api_uri=api_uri, + api_version=api_version, + uid=uid, + gpgbinary=gpgbinary) + + +def _parse_args(): # parse command line parser = argparse.ArgumentParser() parser.add_argument( 'user@provider', action=ValidateUserHandle, help='the user handle') parser.add_argument( - '-b', dest='basedir', required=False, default=None, + '--basedir', '-b', default=None, help='soledad base directory') parser.add_argument( - '-p', dest='passphrase', required=False, default=None, + '--passphrase', '-p', default=None, help='the user passphrase') - args = parser.parse_args() + parser.add_argument( + '--get-all-docs', '-a', action='store_true', + help='get all documents from the local database') + parser.add_argument( + '--create-doc', '-c', default=None, + help='create a document with give content') + parser.add_argument( + '--sync', '-s', action='store_true', + help='synchronize with the server replica') + parser.add_argument( + '--export-public-key', help="export the public key to a file") + parser.add_argument( + '--export-private-key', help="export the private key to a file") + parser.add_argument( + '--export-incoming-messages', + help="export incoming messages to a directory") + return parser.parse_args() - # get the password + +def _get_passphrase(args): passphrase = args.passphrase if passphrase is None: passphrase = getpass.getpass( 'Password for %s@%s: ' % (args.username, args.provider)) + return passphrase + - # get the basedir +def _get_basedir(args): basedir = args.basedir if basedir is None: basedir = tempfile.mkdtemp() + elif not os.path.isdir(basedir): + os.mkdir(basedir) logger.info('Using %s as base directory.' % basedir) + return basedir + + +@inlineCallbacks +def _export_key(args, km, fname, private=False): + address = args.username + "@" + args.provider + pkey = yield km.get_key(address, OpenPGPKey, private=private, fetch_remote=False) + with open(args.export_private_key, "w") as f: + f.write(pkey.key_data) + + +@inlineCallbacks +def _export_incoming_messages(soledad, directory): + yield soledad.create_index("by-incoming", "bool(incoming)") + docs = yield soledad.get_from_index("by-incoming", '1') + i = 1 + for doc in docs: + with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f: + f.write(doc.content["_enc_json"]) + i += 1 + + +@inlineCallbacks +def _get_all_docs(soledad): + _, docs = yield soledad.get_all_docs() + for doc in docs: + print json.dumps(doc.content, indent=4) - # get the soledad instance - s = get_soledad_instance( - args.username, args.provider, passphrase, basedir) + +# main program + +@inlineCallbacks +def _main(soledad, km, args): + try: + if args.create_doc: + yield soledad.create_doc({'content': args.create_doc}) + if args.sync: + yield soledad.sync() + if args.get_all_docs: + yield _get_all_docs(soledad) + if args.export_private_key: + yield _export_key(args, km, args.export_private_key, private=True) + if args.export_public_key: + yield _export_key(args, km, args.expoert_public_key, private=False) + if args.export_incoming_messages: + yield _export_incoming_messages(soledad, args.export_incoming_messages) + except: + pass + finally: + reactor.stop() + + +if __name__ == '__main__': + args = _parse_args() + passphrase = _get_passphrase(args) + basedir = _get_basedir(args) + uuid, server_url, cert_file, token = \ + _get_soledad_info(args.username, args.provider, passphrase, basedir) + soledad = _get_soledad_instance( + uuid, passphrase, basedir, server_url, cert_file, token) + km = _get_keymanager_instance( + args.username, + args.provider, + soledad, + token, + uid=uuid) + _main(soledad, km, args) + reactor.run() diff --git a/scripts/db_access/reset_db.py b/scripts/db_access/reset_db.py index 80871856..7c6d281b 100644 --- a/scripts/db_access/reset_db.py +++ b/scripts/db_access/reset_db.py @@ -5,20 +5,21 @@ # WARNING: running this script over a database will delete all documents but # the one with id u1db_config (which contains db metadata) and design docs # needed for couch backend. +# +# Run it like this to get some help: +# +# ./reset_db.py --help -import sys -from ConfigParser import ConfigParser import threading import logging -from couchdb import Database as CouchDatabase - +import argparse +import re -if len(sys.argv) != 2: - print 'Usage: %s <uuid>' % sys.argv[0] - exit(1) -uuid = sys.argv[1] +from ConfigParser import ConfigParser +from couchdb import Database as CouchDatabase +from couchdb import Server as CouchServer # create a logger @@ -27,23 +28,6 @@ LOG_FORMAT = '%(asctime)s %(message)s' logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) -# get couch url -cp = ConfigParser() -cp.read('/etc/leap/soledad-server.conf') -url = cp.get('soledad-server', 'couch_url') - - -# confirm -yes = raw_input("Are you sure you want to reset the database for user %s " - "(type YES)? " % uuid) -if yes != 'YES': - print 'Bailing out...' - exit(2) - - -db = CouchDatabase('%s/user-%s' % (url, uuid)) - - class _DeleterThread(threading.Thread): def __init__(self, db, doc_id, release_fun): @@ -59,21 +43,95 @@ class _DeleterThread(threading.Thread): self._release_fun() -semaphore_pool = threading.BoundedSemaphore(value=20) - - -threads = [] -for doc_id in db: - if doc_id != 'u1db_config' and not doc_id.startswith('_design'): +def get_confirmation(noconfirm, uuid, shared): + msg = "Are you sure you want to reset %s (type YES)? " + if shared: + msg = msg % "the shared database" + elif uuid: + msg = msg % ("the database for user %s" % uuid) + else: + msg = msg % "all databases" + if noconfirm is False: + yes = raw_input(msg) + if yes != 'YES': + print 'Bailing out...' + exit(2) + + +def get_url(empty): + url = None + if empty is False: + # get couch url + cp = ConfigParser() + cp.read('/etc/leap/soledad-server.conf') + url = cp.get('soledad-server', 'couch_url') + else: + with open('/etc/couchdb/couchdb.netrc') as f: + netrc = f.read() + admin_password = re.match('^.* password (.*)$', netrc).groups()[0] + url = 'http://admin:%s@127.0.0.1:5984' % admin_password + return url + + +def reset_all_dbs(url, empty): + server = CouchServer('%s' % (url)) + for dbname in server: + if dbname.startswith('user-') or dbname == 'shared': + reset_db(url, dbname, empty) + + +def reset_db(url, dbname, empty): + db = CouchDatabase('%s/%s' % (url, dbname)) + semaphore_pool = threading.BoundedSemaphore(value=20) + + # launch threads for deleting docs + threads = [] + for doc_id in db: + if empty is False: + if doc_id == 'u1db_config' or doc_id.startswith('_design'): + continue semaphore_pool.acquire() logger.info('[main] launching thread for doc: %s' % doc_id) t = _DeleterThread(db, doc_id, semaphore_pool.release) t.start() threads.append(t) - -logger.info('[main] waiting for threads.') -map(lambda thread: thread.join(), threads) - - -logger.info('[main] done.') + # wait for threads to finish + logger.info('[main] waiting for threads.') + map(lambda thread: thread.join(), threads) + logger.info('[main] done.') + + +def _parse_args(): + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group() + group.add_argument('-u', dest='uuid', default=False, + help='Reset database of given user.') + group.add_argument('-s', dest='shared', action='store_true', default=False, + help='Reset the shared database.') + group.add_argument('-a', dest='all', action='store_true', default=False, + help='Reset all user databases.') + parser.add_argument( + '-e', dest='empty', action='store_true', required=False, default=False, + help='Empty database (do not preserve minimal set of u1db documents).') + parser.add_argument( + '-y', dest='noconfirm', action='store_true', required=False, + default=False, + help='Do not ask for confirmation.') + return parser.parse_args(), parser + + +if __name__ == '__main__': + args, parser = _parse_args() + if not (args.uuid or args.shared or args.all): + parser.print_help() + exit(1) + + url = get_url(args.empty) + get_confirmation(args.noconfirm, args.uuid, args.shared) + if args.uuid: + reset_db(url, "user-%s" % args.uuid, args.empty) + elif args.shared: + reset_db(url, "shared", args.empty) + elif args.all: + reset_all_dbs(url, args.empty) diff --git a/scripts/db_access/util.py b/scripts/db_access/util.py new file mode 120000 index 00000000..368734f7 --- /dev/null +++ b/scripts/db_access/util.py @@ -0,0 +1 @@ +../profiling/util.py
\ No newline at end of file diff --git a/scripts/ddocs/update_design_docs.py b/scripts/ddocs/update_design_docs.py index e7b5a29c..2e2fa8f0 100644 --- a/scripts/ddocs/update_design_docs.py +++ b/scripts/ddocs/update_design_docs.py @@ -11,84 +11,83 @@ import re import threading import binascii - +from urlparse import urlparse from getpass import getpass from ConfigParser import ConfigParser -from couchdb.client import Server -from couchdb.http import Resource, Session -from datetime import datetime -from urlparse import urlparse +from couchdb.client import Server +from couchdb.http import Resource +from couchdb.http import Session +from couchdb.http import ResourceNotFound from leap.soledad.common import ddocs -# parse command line for the log file name -logger_fname = "/tmp/update-design-docs_%s.log" % \ - str(datetime.now()).replace(' ', '_') -parser = argparse.ArgumentParser() -parser.add_argument('--log', action='store', default=logger_fname, type=str, - required=False, help='the name of the log file', nargs=1) -args = parser.parse_args() +MAX_THREADS = 20 +DESIGN_DOCS = { + '_design/docs': json.loads(binascii.a2b_base64(ddocs.docs)), + '_design/syncs': json.loads(binascii.a2b_base64(ddocs.syncs)), + '_design/transactions': json.loads( + binascii.a2b_base64(ddocs.transactions)), +} -# configure the logger +# create a logger logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -print "Logging to %s." % args.log -logging.basicConfig( - filename=args.log, - format="%(asctime)-15s %(message)s") +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) -# configure threads -max_threads = 20 -semaphore_pool = threading.BoundedSemaphore(value=max_threads) -threads = [] +def _parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-u', dest='uuid', default=None, type=str, + help='the UUID of the user') + parser.add_argument('-t', dest='threads', default=MAX_THREADS, type=int, + help='the number of parallel threads') + return parser.parse_args() -# get couch url -cp = ConfigParser() -cp.read('/etc/leap/soledad-server.conf') -url = urlparse(cp.get('soledad-server', 'couch_url')) -# get admin password -netloc = re.sub('^.*@', '', url.netloc) -url = url._replace(netloc=netloc) -password = getpass("Admin password for %s: " % url.geturl()) -url = url._replace(netloc='admin:%s@%s' % (password, netloc)) +def _get_url(): + # get couch url + cp = ConfigParser() + cp.read('/etc/leap/soledad-server.conf') + url = urlparse(cp.get('soledad-server', 'couch_url')) + # get admin password + netloc = re.sub('^.*@', '', url.netloc) + url = url._replace(netloc=netloc) + password = getpass("Admin password for %s: " % url.geturl()) + return url._replace(netloc='admin:%s@%s' % (password, netloc)) -resource = Resource(url.geturl(), Session(retry_delays=[1,2,4,8], timeout=10)) -server = Server(url=resource) -hidden_url = re.sub( - 'http://(.*):.*@', - 'http://\\1:xxxxx@', - url.geturl()) +def _get_server(url): + resource = Resource( + url.geturl(), Session(retry_delays=[1, 2, 4, 8], timeout=10)) + return Server(url=resource) -print """ -========== -ATTENTION! -========== -This script will modify Soledad's shared and user databases in: +def _confirm(url): + hidden_url = re.sub( + 'http://(.*):.*@', + 'http://\\1:xxxxx@', + url.geturl()) - %s + print """ + ========== + ATTENTION! + ========== -This script does not make a backup of the couch db data, so make sure you -have a copy or you may loose data. -""" % hidden_url -confirm = raw_input("Proceed (type uppercase YES)? ") + This script will modify Soledad's shared and user databases in: -if confirm != "YES": - exit(1) + %s -# convert design doc content + This script does not make a backup of the couch db data, so make sure you + have a copy or you may loose data. + """ % hidden_url + confirm = raw_input("Proceed (type uppercase YES)? ") + + if confirm != "YES": + exit(1) -design_docs = { - '_design/docs': json.loads(binascii.a2b_base64(ddocs.docs)), - '_design/syncs': json.loads(binascii.a2b_base64(ddocs.syncs)), - '_design/transactions': json.loads(binascii.a2b_base64(ddocs.transactions)), -} # # Thread @@ -106,42 +105,66 @@ class DBWorkerThread(threading.Thread): def run(self): - logger.info("(%d/%d) Updating db %s." % (self._db_idx, self._db_len, - self._dbname)) + logger.info( + "(%d/%d) Updating db %s." + % (self._db_idx, self._db_len, self._dbname)) - for doc_id in design_docs: - doc = self._cdb[doc_id] + for doc_id in DESIGN_DOCS: + try: + doc = self._cdb[doc_id] + except ResourceNotFound: + doc = {'_id': doc_id} for key in ['lists', 'views', 'updates']: - if key in design_docs[doc_id]: - doc[key] = design_docs[doc_id][key] + if key in DESIGN_DOCS[doc_id]: + doc[key] = DESIGN_DOCS[doc_id][key] self._cdb.save(doc) # release the semaphore self._release_fun() -db_idx = 0 -db_len = len(server) -for dbname in server: - - db_idx += 1 - - if not (dbname.startswith('user-') or dbname == 'shared') \ - or dbname == 'user-test-db': - logger.info("(%d/%d) Skipping db %s." % (db_idx, db_len, dbname)) - continue - - - # get access to couch db - cdb = Server(url.geturl())[dbname] - - #--------------------------------------------------------------------- - # Start DB worker thread - #--------------------------------------------------------------------- - semaphore_pool.acquire() - thread = DBWorkerThread(server, dbname, db_idx, db_len, semaphore_pool.release) +def _launch_update_design_docs_thread( + server, dbname, db_idx, db_len, semaphore_pool): + semaphore_pool.acquire() # wait for an available working slot + thread = DBWorkerThread( + server, dbname, db_idx, db_len, semaphore_pool.release) thread.daemon = True thread.start() - threads.append(thread) - -map(lambda thread: thread.join(), threads) + return thread + + +def _update_design_docs(args, server): + + # find the actual databases to be updated + dbs = [] + if args.uuid: + dbs.append('user-%s' % args.uuid) + else: + for dbname in server: + if dbname.startswith('user-') or dbname == 'shared': + dbs.append(dbname) + else: + logger.info("Skipping db %s." % dbname) + + db_idx = 0 + db_len = len(dbs) + semaphore_pool = threading.BoundedSemaphore(value=args.threads) + threads = [] + + # launch the update + for db in dbs: + db_idx += 1 + threads.append( + _launch_update_design_docs_thread( + server, db, db_idx, db_len, semaphore_pool)) + + # wait for all threads to finish + map(lambda thread: thread.join(), threads) + + +if __name__ == "__main__": + args = _parse_args() + url = _get_url() + _confirm(url) + server = _get_server(url) + _update_design_docs(args, server) diff --git a/scripts/profiling/mail/__init__.py b/scripts/profiling/mail/__init__.py new file mode 100644 index 00000000..352faae6 --- /dev/null +++ b/scripts/profiling/mail/__init__.py @@ -0,0 +1,184 @@ +import threading +import time +import logging +import argparse + +from twisted.internet import reactor + +from util import log +from couchdb_server import get_couchdb_wrapper_and_u1db +from mx import put_lots_of_messages +from soledad_server import get_soledad_server +from soledad_client import SoledadClient +from mail import get_imap_server + + +UUID = 'blah' +AUTH_TOKEN = 'bleh' + + +logging.basicConfig(level=logging.DEBUG) + +modules = [ + 'gnupg', + 'leap.common', + 'leap.keymanager', + 'taskthread', +] + +for module in modules: + logger = logging.getLogger(name=module) + logger.setLevel(logging.WARNING) + + +class TestWatcher(threading.Thread): + + def __init__(self, couchdb_wrapper, couchdb_u1db, soledad_server, + soledad_client, imap_service, number_of_msgs, lock): + threading.Thread.__init__(self) + self._couchdb_wrapper = couchdb_wrapper + self._couchdb_u1db = couchdb_u1db + self._soledad_server = soledad_server + self._soledad_client = soledad_client + self._imap_service = imap_service + self._number_of_msgs = number_of_msgs + self._lock = lock + self._mails_available_time = None + self._mails_available_time_lock = threading.Lock() + self._conditions = None + + def run(self): + self._set_conditions() + while not self._test_finished(): + time.sleep(5) + log("TestWatcher: Tests finished, cleaning up...", + line_break=False) + self._stop_reactor() + self._cleanup() + log("done.") + self._lock.release() + + def _set_conditions(self): + self._conditions = [] + + # condition 1: number of received messages is equal to number of + # expected messages + def _condition1(*args): + msgcount = self._imap_service._inbox.getMessageCount() + cond = msgcount == self._number_of_msgs + log("[condition 1] received messages: %d (expected: %d) :: %s" + % (msgcount, self._number_of_msgs, cond)) + if self.mails_available_time == None \ + and cond: + with self._mails_available_time_lock: + self._mails_available_time = time.time() + return cond + + + # condition 2: number of documents in server is equal to in client + def _condition2(client_docs, server_docs): + cond = client_docs == server_docs + log("[condition 2] number of documents: client %d; server %d :: %s" + % (client_docs, server_docs, cond)) + return cond + + # condition 3: number of documents bigger than 3 x number of msgs + def _condition3(client_docs, *args): + cond = client_docs > (2 * self._number_of_msgs) + log("[condition 3] documents (%d) > 2 * msgs (%d) :: %s" + % (client_docs, self._number_of_msgs, cond)) + return cond + + # condition 4: not syncing + def _condition4(*args): + cond = not self._soledad_client.instance.syncing + log("[condition 4] not syncing :: %s" % cond) + return cond + + self._conditions.append(_condition1) + self._conditions.append(_condition2) + self._conditions.append(_condition3) + self._conditions.append(_condition4) + + def _test_finished(self): + client_docs = self._get_soledad_client_number_of_docs() + server_docs = self._get_couchdb_number_of_docs() + return not bool(filter(lambda x: not x(client_docs, server_docs), + self._conditions)) + + def _stop_reactor(self): + reactor.stop() + + def _cleanup(self): + self._imap_service.stop() + self._soledad_client.close() + self._soledad_server.stop() + self._couchdb_wrapper.stop() + + def _get_soledad_client_number_of_docs(self): + c = self._soledad_client.instance._db._db_handle.cursor() + c.execute('SELECT COUNT(*) FROM document WHERE content IS NOT NULL') + row = c.fetchone() + return int(row[0]) + + def _get_couchdb_number_of_docs(self): + couchdb = self._couchdb_u1db._database + view = couchdb.view('_all_docs', include_docs=True) + return len(filter( + lambda r: '_attachments' in r.values()[1] + and 'u1db_content' in r.values()[1]['_attachments'], + view.rows)) + + @property + def mails_available_time(self): + with self._mails_available_time_lock: + return self._mails_available_time + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('number_of_msgs', help="The number of documents", + type=int) + parser.add_argument('report_file', help="The name of the report file", + type=str) + args = parser.parse_args() + + # start a couchdb server + couchdb_wrapper, couchdb_u1db = get_couchdb_wrapper_and_u1db( + UUID, AUTH_TOKEN) + + put_time = put_lots_of_messages(couchdb_u1db, args.number_of_msgs) + + soledad_server = get_soledad_server(couchdb_wrapper.port) + + soledad_client = SoledadClient( + uuid='blah', + server_url='http://127.0.0.1:%d' % soledad_server.port, + auth_token=AUTH_TOKEN) + + imap_service = get_imap_server( + soledad_client.instance, UUID, 'snowden@bitmask.net', AUTH_TOKEN) + + lock = threading.Lock() + lock.acquire() + test_watcher = TestWatcher( + couchdb_wrapper, couchdb_u1db, soledad_server, soledad_client, + imap_service, args.number_of_msgs, lock) + test_watcher.start() + + # reactor.run() will block until TestWatcher stops the reactor. + start_time = time.time() + reactor.run() + log("Reactor stopped.") + end_time = time.time() + lock.acquire() + mails_available_time = test_watcher.mails_available_time - start_time + sync_time = end_time - start_time + log("Total syncing time: %f" % sync_time) + log("# number_of_msgs put_time mails_available_time sync_time") + result = "%d %f %f %f" \ + % (args.number_of_msgs, put_time, mails_available_time, + sync_time) + log(result) + with open(args.report_file, 'a') as f: + f.write(result + "\n") diff --git a/scripts/profiling/mail/couchdb.ini.template b/scripts/profiling/mail/couchdb.ini.template new file mode 100644 index 00000000..1fc2205b --- /dev/null +++ b/scripts/profiling/mail/couchdb.ini.template @@ -0,0 +1,224 @@ +; etc/couchdb/default.ini.tpl. Generated from default.ini.tpl.in by configure. + +; Upgrading CouchDB will overwrite this file. + +[couchdb] +database_dir = %(tempdir)s/lib +view_index_dir = %(tempdir)s/lib +max_document_size = 4294967296 ; 4 GB +os_process_timeout = 120000 ; 120 seconds. for view and external servers. +max_dbs_open = 100 +delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned +uri_file = %(tempdir)s/lib/couch.uri +file_compression = snappy + +[database_compaction] +; larger buffer sizes can originate smaller files +doc_buffer_size = 524288 ; value in bytes +checkpoint_after = 5242880 ; checkpoint after every N bytes were written + +[view_compaction] +; larger buffer sizes can originate smaller files +keyvalue_buffer_size = 2097152 ; value in bytes + +[httpd] +port = 0 +bind_address = 127.0.0.1 +authentication_handlers = {couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler} +default_handler = {couch_httpd_db, handle_request} +secure_rewrites = true +vhost_global_handlers = _utils, _uuids, _session, _oauth, _users +allow_jsonp = false +; Options for the MochiWeb HTTP server. +;server_options = [{backlog, 128}, {acceptor_pool_size, 16}] +; For more socket options, consult Erlang's module 'inet' man page. +;socket_options = [{recbuf, 262144}, {sndbuf, 262144}, {nodelay, true}] +log_max_chunk_size = 1000000 + +[log] +file = %(tempdir)s/log/couch.log +level = info +include_sasl = true + +[couch_httpd_auth] +authentication_db = _users +authentication_redirect = /_utils/session.html +require_valid_user = false +timeout = 600 ; number of seconds before automatic logout +auth_cache_size = 50 ; size is number of cache entries +allow_persistent_cookies = false ; set to true to allow persistent cookies + +[couch_httpd_oauth] +; If set to 'true', oauth token and consumer secrets will be looked up +; in the authentication database (_users). These secrets are stored in +; a top level property named "oauth" in user documents. Example: +; { +; "_id": "org.couchdb.user:joe", +; "type": "user", +; "name": "joe", +; "password_sha": "fe95df1ca59a9b567bdca5cbaf8412abd6e06121", +; "salt": "4e170ffeb6f34daecfd814dfb4001a73" +; "roles": ["foo", "bar"], +; "oauth": { +; "consumer_keys": { +; "consumerKey1": "key1Secret", +; "consumerKey2": "key2Secret" +; }, +; "tokens": { +; "token1": "token1Secret", +; "token2": "token2Secret" +; } +; } +; } +use_users_db = false + +[query_servers] +; javascript = %(tempdir)s/server/main.js +javascript = /usr/bin/couchjs /usr/share/couchdb/server/main.js +coffeescript = /usr/bin/couchjs /usr/share/couchdb/server/main-coffee.js + + +; Changing reduce_limit to false will disable reduce_limit. +; If you think you're hitting reduce_limit with a "good" reduce function, +; please let us know on the mailing list so we can fine tune the heuristic. +[query_server_config] +reduce_limit = true +os_process_limit = 25 + +[daemons] +view_manager={couch_view, start_link, []} +external_manager={couch_external_manager, start_link, []} +query_servers={couch_query_servers, start_link, []} +vhosts={couch_httpd_vhost, start_link, []} +httpd={couch_httpd, start_link, []} +stats_aggregator={couch_stats_aggregator, start, []} +stats_collector={couch_stats_collector, start, []} +uuids={couch_uuids, start, []} +auth_cache={couch_auth_cache, start_link, []} +replication_manager={couch_replication_manager, start_link, []} +os_daemons={couch_os_daemons, start_link, []} +compaction_daemon={couch_compaction_daemon, start_link, []} + +[httpd_global_handlers] +/ = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>} + +_all_dbs = {couch_httpd_misc_handlers, handle_all_dbs_req} +_active_tasks = {couch_httpd_misc_handlers, handle_task_status_req} +_config = {couch_httpd_misc_handlers, handle_config_req} +_replicate = {couch_httpd_replicator, handle_req} +_uuids = {couch_httpd_misc_handlers, handle_uuids_req} +_restart = {couch_httpd_misc_handlers, handle_restart_req} +_stats = {couch_httpd_stats_handlers, handle_stats_req} +_log = {couch_httpd_misc_handlers, handle_log_req} +_session = {couch_httpd_auth, handle_session_req} +_oauth = {couch_httpd_oauth, handle_oauth_req} + +[httpd_db_handlers] +_view_cleanup = {couch_httpd_db, handle_view_cleanup_req} +_compact = {couch_httpd_db, handle_compact_req} +_design = {couch_httpd_db, handle_design_req} +_temp_view = {couch_httpd_view, handle_temp_view_req} +_changes = {couch_httpd_db, handle_changes_req} + +; The external module takes an optional argument allowing you to narrow it to a +; single script. Otherwise the script name is inferred from the first path section +; after _external's own path. +; _mypath = {couch_httpd_external, handle_external_req, <<"mykey">>} +; _external = {couch_httpd_external, handle_external_req} + +[httpd_design_handlers] +_view = {couch_httpd_view, handle_view_req} +_show = {couch_httpd_show, handle_doc_show_req} +_list = {couch_httpd_show, handle_view_list_req} +_info = {couch_httpd_db, handle_design_info_req} +_rewrite = {couch_httpd_rewrite, handle_rewrite_req} +_update = {couch_httpd_show, handle_doc_update_req} + +; enable external as an httpd handler, then link it with commands here. +; note, this api is still under consideration. +; [external] +; mykey = /path/to/mycommand + +; Here you can setup commands for CouchDB to manage +; while it is alive. It will attempt to keep each command +; alive if it exits. +; [os_daemons] +; some_daemon_name = /path/to/script -with args + + +[uuids] +; Known algorithms: +; random - 128 bits of random awesome +; All awesome, all the time. +; sequential - monotonically increasing ids with random increments +; First 26 hex characters are random. Last 6 increment in +; random amounts until an overflow occurs. On overflow, the +; random prefix is regenerated and the process starts over. +; utc_random - Time since Jan 1, 1970 UTC with microseconds +; First 14 characters are the time in hex. Last 18 are random. +algorithm = sequential + +[stats] +; rate is in milliseconds +rate = 1000 +; sample intervals are in seconds +samples = [0, 60, 300, 900] + +[attachments] +compression_level = 8 ; from 1 (lowest, fastest) to 9 (highest, slowest), 0 to disable compression +compressible_types = text/*, application/javascript, application/json, application/xml + +[replicator] +db = _replicator +; Maximum replicaton retry count can be a non-negative integer or "infinity". +max_replication_retry_count = 10 +; More worker processes can give higher network throughput but can also +; imply more disk and network IO. +worker_processes = 4 +; With lower batch sizes checkpoints are done more frequently. Lower batch sizes +; also reduce the total amount of used RAM memory. +worker_batch_size = 500 +; Maximum number of HTTP connections per replication. +http_connections = 20 +; HTTP connection timeout per replication. +; Even for very fast/reliable networks it might need to be increased if a remote +; database is too busy. +connection_timeout = 30000 +; If a request fails, the replicator will retry it up to N times. +retries_per_request = 10 +; Some socket options that might boost performance in some scenarios: +; {nodelay, boolean()} +; {sndbuf, integer()} +; {recbuf, integer()} +; {priority, integer()} +; See the `inet` Erlang module's man page for the full list of options. +socket_options = [{keepalive, true}, {nodelay, false}] +; Path to a file containing the user's certificate. +;cert_file = /full/path/to/server_cert.pem +; Path to file containing user's private PEM encoded key. +;key_file = /full/path/to/server_key.pem +; String containing the user's password. Only used if the private keyfile is password protected. +;password = somepassword +; Set to true to validate peer certificates. +verify_ssl_certificates = false +; File containing a list of peer trusted certificates (in the PEM format). +;ssl_trusted_certificates_file = /etc/ssl/certs/ca-certificates.crt +; Maximum peer certificate depth (must be set even if certificate validation is off). +ssl_certificate_max_depth = 3 + +[compaction_daemon] +; The delay, in seconds, between each check for which database and view indexes +; need to be compacted. +check_interval = 300 +; If a database or view index file is smaller then this value (in bytes), +; compaction will not happen. Very small files always have a very high +; fragmentation therefore it's not worth to compact them. +min_file_size = 131072 + +[compactions] +; List of compaction rules for the compaction daemon. + + +;[admins] +;testuser = -hashed-f50a252c12615697c5ed24ec5cd56b05d66fe91e,b05471ba260132953930cf9f97f327f5 +; pass for above user is 'testpass' diff --git a/scripts/profiling/mail/couchdb_server.py b/scripts/profiling/mail/couchdb_server.py new file mode 100644 index 00000000..2cf0a3fd --- /dev/null +++ b/scripts/profiling/mail/couchdb_server.py @@ -0,0 +1,42 @@ +import hashlib +import couchdb + +from leap.soledad.common.couch import CouchDatabase + +from util import log +from couchdb_wrapper import CouchDBWrapper + + +def start_couchdb_wrapper(): + log("Starting couchdb... ", line_break=False) + couchdb_wrapper = CouchDBWrapper() + couchdb_wrapper.start() + log("couchdb started on port %d." % couchdb_wrapper.port) + return couchdb_wrapper + + +def get_u1db_database(dbname, port): + return CouchDatabase.open_database( + 'http://127.0.0.1:%d/%s' % (port, dbname), + True, + ensure_ddocs=True) + + +def create_tokens_database(port, uuid, token_value): + tokens_database = couchdb.Server( + 'http://127.0.0.1:%d' % port).create('tokens') + token = couchdb.Document() + token['_id'] = hashlib.sha512(token_value).hexdigest() + token['user_id'] = uuid + token['type'] = 'Token' + tokens_database.save(token) + + +def get_couchdb_wrapper_and_u1db(uuid, token_value): + couchdb_wrapper = start_couchdb_wrapper() + + couchdb_u1db = get_u1db_database('user-%s' % uuid, couchdb_wrapper.port) + get_u1db_database('shared', couchdb_wrapper.port) + create_tokens_database(couchdb_wrapper.port, uuid, token_value) + + return couchdb_wrapper, couchdb_u1db diff --git a/scripts/profiling/mail/couchdb_wrapper.py b/scripts/profiling/mail/couchdb_wrapper.py new file mode 100644 index 00000000..cad1205b --- /dev/null +++ b/scripts/profiling/mail/couchdb_wrapper.py @@ -0,0 +1,84 @@ +import re +import os +import tempfile +import subprocess +import time +import shutil + + +from leap.common.files import mkdir_p + + +class CouchDBWrapper(object): + """ + Wrapper for external CouchDB instance. + """ + + def start(self): + """ + Start a CouchDB instance for a test. + """ + self.tempdir = tempfile.mkdtemp(suffix='.couch.test') + + path = os.path.join(os.path.dirname(__file__), + 'couchdb.ini.template') + handle = open(path) + conf = handle.read() % { + 'tempdir': self.tempdir, + } + handle.close() + + confPath = os.path.join(self.tempdir, 'test.ini') + handle = open(confPath, 'w') + handle.write(conf) + handle.close() + + # create the dirs from the template + mkdir_p(os.path.join(self.tempdir, 'lib')) + mkdir_p(os.path.join(self.tempdir, 'log')) + args = ['couchdb', '-n', '-a', confPath] + null = open('/dev/null', 'w') + + self.process = subprocess.Popen( + args, env=None, stdout=null.fileno(), stderr=null.fileno(), + close_fds=True) + # find port + logPath = os.path.join(self.tempdir, 'log', 'couch.log') + while not os.path.exists(logPath): + if self.process.poll() is not None: + got_stdout, got_stderr = "", "" + if self.process.stdout is not None: + got_stdout = self.process.stdout.read() + + if self.process.stderr is not None: + got_stderr = self.process.stderr.read() + raise Exception(""" +couchdb exited with code %d. +stdout: +%s +stderr: +%s""" % ( + self.process.returncode, got_stdout, got_stderr)) + time.sleep(0.01) + while os.stat(logPath).st_size == 0: + time.sleep(0.01) + PORT_RE = re.compile( + 'Apache CouchDB has started on http://127.0.0.1:(?P<port>\d+)') + + handle = open(logPath) + m = None + line = handle.readline() + while m is None: + m = PORT_RE.search(line) + line = handle.readline() + handle.close() + self.port = int(m.group('port')) + + def stop(self): + """ + Terminate the CouchDB instance. + """ + self.process.terminate() + self.process.communicate() + shutil.rmtree(self.tempdir) + diff --git a/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub new file mode 100644 index 00000000..fee53b6d --- /dev/null +++ b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub @@ -0,0 +1,30 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.12 (GNU/Linux) + +mQENBFQEwmABCADC4wYD3mFt8xJtl3gjxRPEGN+FcgvzxxECIhyjYCHszrJu3f65 +/nyruriYdQLGR4YmUdERIwsZ7AMkAM1NAXe7sMq/gRPCb4PwrE7pRKzPAmaLeJMQ +DC9CSCP+2gUmzeKHS71GkddcUI1HFr1AX9lLVW2ScvmSzOllenyUoFKRvz2uGkLG +r5pvKsxJUHl9enpHRZV/0X5Y6PCinb4+eN2/ZTdpAywOycU+L+zflA0SOTCtf+dg +8k839T30piuBulDLNeOX84YcyXTW7XeCeRTg/ryoFaYhbOGt68BwnP9xlpU62LW0 +8vzSZ0mLm4Ttz2uaALEoLmsa91nyLi9pLtrRABEBAAG0IEVkIFNub3dkZW4gPHNu +b3dkZW5AYml0bWFzay5uZXQ+iQE4BBMBAgAiBQJUBMJgAhsDBgsJCAcDAgYVCAIJ +CgsEFgIDAQIeAQIXgAAKCRAbRQ5mX+Y1cx4RCACzEiHpmknl+HnB3bHGcr8VZvU9 +hIoclVR/OBjWQFUynr66XmaMHMOLAVoZkIPnezWQe3gDY7QlFCNCfz8SC2++4WtB +aBzal9IREkVnQBdnWalxLRviNH+zoFQ0URunBAyH4QAJRUC5tWfNj4yI6BCFPwXL +o0CCISIN+VMRAnwjABQD840/TbcMHDqmJyk/vpPYPFQqQudN3eB2hphKUkZMistP +O9++ui6glso+MgsbIUdqgnblM3FSrbjfLKekC+MeunFr8qRjettdaVyFD4GLg2SH +/JpsjZKYoZStatpdJcrNjUMsGtXLxaCPl+VldNuOKIsA85TZJomMiaBDqG9YuQEN +BFQEwmABCACrYiPXyGWHvs/aFKM63y9l6Th/+SKfzeq+ksLUI6fJIQytGORiiYZC +1LrhOTmir+dY3IygkFlldxehGt/OMUKLB774WhBDRI43rAhImwhNutTIuUTO7DsD +y7u83oVQH6xGZW5afs5BEU56Oa8DdUUA5gLfnpqAJG2mLB12JhClxzOYXK/VB0wJ +QsIWl+zyN7uLQr5xZOthzvP6p7MmsAjhzU1imwyEm8s91DLhwonuqadkMGKi2qHW +xuwxnr9aHQmobzy68/vOiBFeumr0YarirUdEDiUIti4rqy+0oteTNeMtXWo5rTtx +xeayw+TjjaOT2fZ6CAbq0I+lOW0aJrPFABEBAAGJAR8EGAECAAkFAlQEwmACGwwA +CgkQG0UOZl/mNXM0SggAuXzaLafCZiWx28K6mPKdgDOwTMm2rD7ukf3JiswlIyIU +/K19BENu82iHRSu4nb9amhHOLEhaf1Ep2JTf2Trmd+/SNh0kv3dSBNjCrvrMvtcA +qVxGc3DtRufGeRoy8ow/sEg+BCcfxJgR1efHOSQfMELDz2v8vbLbkR3Ubm7YRtKr +Ri2HWYrAXRrwFC07yqO2zptCND/LBtnMrp08AOSSLpRWVD/Ww6IE1v1UEN53aGsm +D+L/1XkuP4L9cqG3E2NYfsOPiblqRiKSe1adVid/rLn94u+fpE4kuvxoGKn1FJ/m +FqU8aPtxvPbsMkSoNOalxqJGpuWRTXTLb5I+Ed2Szw== +=yRE/ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec new file mode 100644 index 00000000..64cb6c2a --- /dev/null +++ b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec @@ -0,0 +1,57 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.12 (GNU/Linux) + +lQOYBFQEwmABCADC4wYD3mFt8xJtl3gjxRPEGN+FcgvzxxECIhyjYCHszrJu3f65 +/nyruriYdQLGR4YmUdERIwsZ7AMkAM1NAXe7sMq/gRPCb4PwrE7pRKzPAmaLeJMQ +DC9CSCP+2gUmzeKHS71GkddcUI1HFr1AX9lLVW2ScvmSzOllenyUoFKRvz2uGkLG +r5pvKsxJUHl9enpHRZV/0X5Y6PCinb4+eN2/ZTdpAywOycU+L+zflA0SOTCtf+dg +8k839T30piuBulDLNeOX84YcyXTW7XeCeRTg/ryoFaYhbOGt68BwnP9xlpU62LW0 +8vzSZ0mLm4Ttz2uaALEoLmsa91nyLi9pLtrRABEBAAEAB/0cLb885/amczMC7ZfN +dD17aS1ImkjoIqxu5ofFh6zgFLLwHOEr+4QDQKhYQvL3wHfBKqtUEwET6nA50HPe +4otxdAqczgkRYBZvwjpWuDtUY0B4giKhe2GJ7+xkeRmtlq9eaLEhdwzwqCUFVmBe +4n0Ey4FgX4d+lmpY5fEFfHjz4bZpoCrNZKtiGtOqdlKXm8PnU+ek+G7DFuavJ+g5 +B4fiqkLAYFX/IDFfaTSBYzNDPbSQR5n4Q4r9PdKazPXg7bnLuxAIY4i6KEXq2YpS +T1vLanCnBd4BEDUODCPZdc/AtbE0U+XoKTBjTvk3UEGIRJSsju8A1vWOG7UCl+0d +UMmRBADaiQYnp9QiwPDbpqxzlWN8Ms/+tAyRnBbhghcRqtrDSke6fSJAqXzVGVmF +FSJPMFf4mBYbr1U3YlYOJrlrb3tVhVN+7PTZDIaaENbtcsUAu7hTr7Ko6r1+WONC +yhtrtOR9sWHVbTZ09ZvyvjHnBqZVA2PuZLUn2wrimnIJbVNdlwQA5EwgoS8UuDob +hs6tLg29bAEDZRBHXQcDuEwdAX0KCHW0oQ0UE7exbDXXfQJSD9X3fDeqI+BdI+qQ +Yuauz+fJxKl+qHAcy5l5NT7qomEjHCzjGUnn4NJzkn6a3T4SrBdSMFY2hL/tJN0i +v1hXVNatjCEotqqsor+C6bf+Sl4I59cEAK+tYLTo/d+KOWtW4XbVhcYHjTBKtJGH +p2/wNb49ibYpkgOUqW2ebiCB0Lg6QEupomcaMOJGol3v8vwBKsuwQJhWJrAXC2sT +Bck5mI+DbabyAbYFtZgNHbcdDy62ADg1xD2Je7IjUDcpYaGB3VFhpD2rSvWDeSjR +3jTG3PPINfoBODK0IEVkIFNub3dkZW4gPHNub3dkZW5AYml0bWFzay5uZXQ+iQE4 +BBMBAgAiBQJUBMJgAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAbRQ5m +X+Y1cx4RCACzEiHpmknl+HnB3bHGcr8VZvU9hIoclVR/OBjWQFUynr66XmaMHMOL +AVoZkIPnezWQe3gDY7QlFCNCfz8SC2++4WtBaBzal9IREkVnQBdnWalxLRviNH+z +oFQ0URunBAyH4QAJRUC5tWfNj4yI6BCFPwXLo0CCISIN+VMRAnwjABQD840/TbcM +HDqmJyk/vpPYPFQqQudN3eB2hphKUkZMistPO9++ui6glso+MgsbIUdqgnblM3FS +rbjfLKekC+MeunFr8qRjettdaVyFD4GLg2SH/JpsjZKYoZStatpdJcrNjUMsGtXL +xaCPl+VldNuOKIsA85TZJomMiaBDqG9YnQOYBFQEwmABCACrYiPXyGWHvs/aFKM6 +3y9l6Th/+SKfzeq+ksLUI6fJIQytGORiiYZC1LrhOTmir+dY3IygkFlldxehGt/O +MUKLB774WhBDRI43rAhImwhNutTIuUTO7DsDy7u83oVQH6xGZW5afs5BEU56Oa8D +dUUA5gLfnpqAJG2mLB12JhClxzOYXK/VB0wJQsIWl+zyN7uLQr5xZOthzvP6p7Mm +sAjhzU1imwyEm8s91DLhwonuqadkMGKi2qHWxuwxnr9aHQmobzy68/vOiBFeumr0 +YarirUdEDiUIti4rqy+0oteTNeMtXWo5rTtxxeayw+TjjaOT2fZ6CAbq0I+lOW0a +JrPFABEBAAEAB/4kyb13Z4MRyy37OkRakgdu2QvhfoVF59Hso/yxxFCTHibGLkpx +82LQTDEsQNgkGZ2vp7IBElM6MkDuemIRtOW7icdesJh+lAPyI9moWi0DYGgmCQzh +3PgDBdPQBDT6IL5eYw3323HjKjeeCW1NsPnFqlnyDe3MtWUbDyuozZ1ztA+Rekhb +UhEDK8ZccEKwpzrE2H5zBZLeY0OKKROGnwd1RBVXnHMgVRF7vbellYaR4h2odxOp +X8Ho4Xbs1h2VRNIuZwtfXxTIuTIfujlIPXMtVY40dgnEGt9PosJNr9IfGpfE3JCu +k9PTvq37aZkQbYj52nccwKdos+sLQgqAdHhZBADHg7B5jyRRObsCUXQ+jMHXxuqT +5l1twwOovvLC7YZoC8NAl4Bi0rh1Zj0ZEJJLFGzeiH+15C4qFTY+ospWpGu6X6g5 +I8ZWya8m2NSEWyJZNI1zKIU0iXucLevVTx+ctnovUNnb89v52/+BKr4k2iRISAzT +7RL63aFTgnLw9GKweQQA2+eU5jcQ6LobPY/fZZImnhwLDq/OaUV+7u1RfB04GA15 +HOGQV77np/QTM6b+ezKTFhG/HMCTqxf+HPHfzohBPF9zvboLvCkqaHBDiV9qYE96 +id/el3ZeWloLcEe62sMGbv0YYmsYWgJxL8BFGw5v1QpYbfQCnXLjyG+/9f6Ygq0D +/0W9X/NxWUyAXOv5KRy+rpkpNVxvie4tduvyVUa/9XHF7D/DMaXqkIvVX8yZUIDR +bjuIvGZkZ9QP8zf8NKkB98zbqZi6CbNrerjrDpb7Pj7uQd3GIcjW4UmENGA6t7U9 +IWen966PAXSzh3996tRHxwXexVIEdX5n4pO39ZiodEIOPzmJAR8EGAECAAkFAlQE +wmACGwwACgkQG0UOZl/mNXM0SggAuXzaLafCZiWx28K6mPKdgDOwTMm2rD7ukf3J +iswlIyIU/K19BENu82iHRSu4nb9amhHOLEhaf1Ep2JTf2Trmd+/SNh0kv3dSBNjC +rvrMvtcAqVxGc3DtRufGeRoy8ow/sEg+BCcfxJgR1efHOSQfMELDz2v8vbLbkR3U +bm7YRtKrRi2HWYrAXRrwFC07yqO2zptCND/LBtnMrp08AOSSLpRWVD/Ww6IE1v1U +EN53aGsmD+L/1XkuP4L9cqG3E2NYfsOPiblqRiKSe1adVid/rLn94u+fpE4kuvxo +GKn1FJ/mFqU8aPtxvPbsMkSoNOalxqJGpuWRTXTLb5I+Ed2Szw== +=9xZX +-----END PGP PRIVATE KEY BLOCK----- diff --git a/scripts/profiling/mail/mail.py b/scripts/profiling/mail/mail.py new file mode 100644 index 00000000..8504c762 --- /dev/null +++ b/scripts/profiling/mail/mail.py @@ -0,0 +1,50 @@ +import os +import threading + +from twisted.internet import reactor + +from leap.mail.imap.service import imap +from leap.keymanager import KeyManager + +from util import log + + +class IMAPServerThread(threading.Thread): + def __init__(self, imap_service): + threading.Thread.__init__(self) + self._imap_service = imap_service + + def run(self): + self._imap_service.start_loop() + reactor.run() + + def stop(self): + self._imap_service.stop() + reactor.stop() + + +def get_imap_server(soledad, uuid, address, token): + log("Starting imap... ", line_break=False) + + keymanager = KeyManager(address, '', soledad, token=token, uid=uuid) + with open( + os.path.join( + os.path.dirname(__file__), + 'keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec'), 'r') as f: + pubkey, privkey = keymanager.parse_openpgp_ascii_key(f.read()) + keymanager.put_key(privkey) + + imap_service, imap_port, imap_factory = imap.run_service( + soledad, keymanager, userid=address, offline=False) + + imap_service.start_loop() + log("started.") + return imap_service + + #imap_server = IMAPServerThread(imap_service) + #try: + # imap_server.start() + #except Exception as e: + # print str(e) + + #return imap_server diff --git a/scripts/profiling/mail/mx.py b/scripts/profiling/mail/mx.py new file mode 100644 index 00000000..b6a1e5cf --- /dev/null +++ b/scripts/profiling/mail/mx.py @@ -0,0 +1,80 @@ +import datetime +import uuid +import json +import timeit + + +from leap.keymanager import openpgp +from leap.soledad.common.couch import CouchDocument +from leap.soledad.common.crypto import ( + EncryptionSchemes, + ENC_JSON_KEY, + ENC_SCHEME_KEY, +) + + +from util import log + + +message = """To: Ed Snowden <snowden@bitmask.net> +Date: %s +From: Glenn Greenwald <greenwald@bitmask.net> + +hi! + +""" + + +def get_message(): + return message % datetime.datetime.now().strftime("%a %b %d %H:%M:%S:%f %Y") + + +def get_enc_json(pubkey, message): + with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg: + gpg.import_keys(pubkey) + key = gpg.list_keys().pop() + # We don't care about the actual address, so we use a + # dummy one, we just care about the import of the pubkey + openpgp_key = openpgp._build_key_from_gpg("dummy@mail.com", + key, pubkey) + enc_json = str(gpg.encrypt( + json.dumps( + {'incoming': True, 'content': message}, + ensure_ascii=False), + openpgp_key.fingerprint, + symmetric=False)) + return enc_json + + +def get_new_doc(enc_json): + doc = CouchDocument(doc_id=str(uuid.uuid4())) + doc.content = { + 'incoming': True, + ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, + ENC_JSON_KEY: enc_json + } + return doc + + +def get_pubkey(): + with open('./keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub') as f: + return f.read() + + +def put_one_message(pubkey, db): + enc_json = get_enc_json(pubkey, get_message()) + doc = get_new_doc(enc_json) + db.put_doc(doc) + + +def put_lots_of_messages(db, number): + log("Populating database with %d encrypted messages... " + % number, line_break=False) + pubkey = get_pubkey() + def _put_one_message(): + put_one_message(pubkey, db) + time = timeit.timeit(_put_one_message, number=number) + log("done.") + average_time = time / number + log("put_one_message average time: %f" % average_time) + return average_time diff --git a/scripts/profiling/mail/soledad_client.py b/scripts/profiling/mail/soledad_client.py new file mode 100644 index 00000000..5ac8ce39 --- /dev/null +++ b/scripts/profiling/mail/soledad_client.py @@ -0,0 +1,40 @@ +import tempfile +import os +import shutil + +from leap.soledad.client import Soledad + + +class SoledadClient(object): + + def __init__(self, uuid, server_url, auth_token): + self._uuid = uuid + self._server_url = server_url + self._auth_token = auth_token + self._tempdir = None + self._soledad = None + + @property + def instance(self): + if self._soledad is None: + self._soledad = self._get_soledad_client() + return self._soledad + + def _get_soledad_client(self): + self._tempdir = tempfile.mkdtemp() + return Soledad( + uuid=self._uuid, + passphrase=u'123', + secrets_path=os.path.join(self._tempdir, 'secrets.json'), + local_db_path=os.path.join(self._tempdir, 'soledad.db'), + server_url=self._server_url, + cert_file=None, + auth_token=self._auth_token, + secret_id=None, + defer_encryption=True) + + def close(self): + if self._soledad is not None: + self._soledad.close() + if self._tempdir is not None: + shutil.rmtree(self._tempdir) diff --git a/scripts/profiling/mail/soledad_server.py b/scripts/profiling/mail/soledad_server.py new file mode 100644 index 00000000..ad014456 --- /dev/null +++ b/scripts/profiling/mail/soledad_server.py @@ -0,0 +1,48 @@ +import threading + +from wsgiref.simple_server import make_server + +from leap.soledad.common.couch import CouchServerState + +from leap.soledad.server import SoledadApp +from leap.soledad.server.gzip_middleware import GzipMiddleware +from leap.soledad.server.auth import SoledadTokenAuthMiddleware + +from util import log + + +class SoledadServerThread(threading.Thread): + def __init__(self, server): + threading.Thread.__init__(self) + self._server = server + + def run(self): + self._server.serve_forever() + + def stop(self): + self._server.shutdown() + + @property + def port(self): + return self._server.server_port + + +def make_soledad_server_thread(couch_port): + state = CouchServerState( + 'http://127.0.0.1:%d' % couch_port, + 'shared', + 'tokens') + application = GzipMiddleware( + SoledadTokenAuthMiddleware(SoledadApp(state))) + server = make_server('', 0, application) + t = SoledadServerThread(server) + return t + + +def get_soledad_server(couchdb_port): + log("Starting soledad server... ", line_break=False) + soledad_server = make_soledad_server_thread(couchdb_port) + soledad_server.start() + log("soledad server started on port %d." % soledad_server.port) + return soledad_server + diff --git a/scripts/profiling/mail/util.py b/scripts/profiling/mail/util.py new file mode 100644 index 00000000..86118e88 --- /dev/null +++ b/scripts/profiling/mail/util.py @@ -0,0 +1,8 @@ +import sys + + +def log(msg, line_break=True): + sys.stdout.write(msg) + if line_break: + sys.stdout.write("\n") + sys.stdout.flush() diff --git a/scripts/profiling/spam.py b/scripts/profiling/spam.py new file mode 100755 index 00000000..091a8c48 --- /dev/null +++ b/scripts/profiling/spam.py @@ -0,0 +1,123 @@ +#!/usr/bin/python + +# Send a lot of messages in parallel. + + +import string +import smtplib +import threading +import logging + +from argparse import ArgumentParser + + +SMTP_HOST = 'chipmonk.cdev.bitmask.net' +NUMBER_OF_THREADS = 20 + + +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + +def _send_email(host, subject, to_addr, from_addr, body_text): + """ + Send an email + """ + body = string.join(( + "From: %s" % from_addr, + "To: %s" % to_addr, + "Subject: %s" % subject, + "", + body_text + ), "\r\n") + server = smtplib.SMTP(host) + server.sendmail(from_addr, [to_addr], body) + server.quit() + + +def _parse_args(): + parser = ArgumentParser() + parser.add_argument( + 'target_address', + help='The target email address to spam') + parser.add_argument( + 'number_of_messages', type=int, + help='The amount of messages email address to spam') + parser.add_argument( + '-s', dest='server', default=SMTP_HOST, + help='The SMTP server to use') + parser.add_argument( + '-t', dest='threads', default=NUMBER_OF_THREADS, + help='The maximum number of parallel threads to launch') + return parser.parse_args() + + +class EmailSenderThread(threading.Thread): + + def __init__(self, host, subject, to_addr, from_addr, body_text, + finished_fun): + threading.Thread.__init__(self) + self._host = host + self._subject = subject + self._to_addr = to_addr + self._from_addr = from_addr + self._body_text = body_text + self._finished_fun = finished_fun + + def run(self): + _send_email( + self._host, self._subject, self._to_addr, self._from_addr, + self._body_text) + self._finished_fun() + + +def _launch_email_thread(host, subject, to_addr, from_addr, body_text, + finished_fun): + thread = EmailSenderThread( + host, subject, to_addr, from_addr, body_text, finished_fun) + thread.start() + return thread + + +class FinishedThreads(object): + + def __init__(self): + self._finished = 0 + self._lock = threading.Lock() + + def signal(self): + with self._lock: + self._finished = self._finished + 1 + logger.info('Number of messages sent: %d.' % self._finished) + + +def _send_messages(args): + host = args.server + subject = "Message from Soledad script" + to_addr = args.target_address + from_addr = args.target_address + body_text = "Test message" + + semaphore = threading.Semaphore(args.threads) + threads = [] + finished_threads = FinishedThreads() + + def _finished_fun(): + semaphore.release() + finished_threads.signal() + + for i in xrange(args.number_of_messages): + semaphore.acquire() + threads.append( + _launch_email_thread( + host, subject, to_addr, from_addr, body_text, + _finished_fun)) + + for t in threads: + t.join() + + +if __name__ == "__main__": + args = _parse_args() + _send_messages(args) diff --git a/scripts/profiling/storage/benchmark-storage.py b/scripts/profiling/storage/benchmark-storage.py new file mode 100644 index 00000000..79ee3270 --- /dev/null +++ b/scripts/profiling/storage/benchmark-storage.py @@ -0,0 +1,104 @@ +#!/usr/bin/python + +# scenarios: +# 1. soledad instantiation time. +# a. for unexisting db. +# b. for existing db. +# 2. soledad doc storage/retrieval. +# a. 1 KB document. +# b 10 KB. +# c. 100 KB. +# d. 1 MB. + + +import logging +import getpass +import tempfile +import argparse +import shutil +import timeit + + +from util import ValidateUserHandle + +# benchmarking args +REPEAT_NUMBER = 1000 +DOC_SIZE = 1024 + + +# create a logger +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + +def parse_args(): + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + 'user@provider', action=ValidateUserHandle, help='the user handle') + parser.add_argument( + '-b', dest='basedir', required=False, default=None, + help='soledad base directory') + parser.add_argument( + '-p', dest='passphrase', required=False, default=None, + help='the user passphrase') + parser.add_argument( + '-l', dest='logfile', required=False, default='/tmp/benchhmark-storage.log', + help='the file to which write the benchmark logs') + args = parser.parse_args() + # get the password + passphrase = args.passphrase + if passphrase is None: + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + # get the basedir + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + logger.info('Using %s as base directory.' % basedir) + + return args.username, args.provider, passphrase, basedir, args.logfile + + +if __name__ == '__main__': + username, provider, passphrase, basedir, logfile = parse_args() + create_results = [] + getall_results = [] + for i in [1, 200, 400, 600, 800, 1000]: + tempdir = tempfile.mkdtemp(dir=basedir) + setup_common = """ +import os +#from benchmark_storage_utils import benchmark_fun +#from benchmark_storage_utils import get_soledad_instance +from client_side_db import get_soledad_instance +sol = get_soledad_instance('%s', '%s', '%s', '%s') + """ % (username, provider, passphrase, tempdir) + + setup_create = setup_common + """ +content = {'data': os.urandom(%d/2).encode('hex')} +""" % (DOC_SIZE * i) + time = timeit.timeit( + 'sol.create_doc(content);', + setup=setup_create, number=REPEAT_NUMBER) + create_results.append((DOC_SIZE*i, time)) + print "CREATE: %d %f" % (DOC_SIZE*i, time) + + setup_get = setup_common + """ +doc_ids = [doc.doc_id for doc in sol.get_all_docs()[1]] +""" + + time = timeit.timeit( + "[sol.get_doc(doc_id) for doc_id in doc_ids]", + setup=setup_get, number=1) + getall_results.append((DOC_SIZE*i, time)) + print "GET_ALL: %d %f" % (DOC_SIZE*i, time) + shutil.rmtree(tempdir) + print "# size, time for creation of %d docs" % REPEAT_NUMBER + for size, time in create_results: + print size, time + print "# size, time for retrieval of %d docs" % REPEAT_NUMBER + for size, time in getall_results: + print size, time + shutil.rmtree(basedir) + diff --git a/scripts/profiling/storage/benchmark_storage_utils.py b/scripts/profiling/storage/benchmark_storage_utils.py new file mode 100644 index 00000000..fa8bb658 --- /dev/null +++ b/scripts/profiling/storage/benchmark_storage_utils.py @@ -0,0 +1,4 @@ +from client_side_db import get_soledad_instance + +def benchmark_fun(sol, content): + sol.create_doc(content) diff --git a/scripts/profiling/storage/client_side_db.py b/scripts/profiling/storage/client_side_db.py new file mode 120000 index 00000000..9e49a7f0 --- /dev/null +++ b/scripts/profiling/storage/client_side_db.py @@ -0,0 +1 @@ +../../db_access/client_side_db.py
\ No newline at end of file diff --git a/scripts/profiling/storage/plot.py b/scripts/profiling/storage/plot.py new file mode 100755 index 00000000..280b9375 --- /dev/null +++ b/scripts/profiling/storage/plot.py @@ -0,0 +1,94 @@ +#!/usr/bin/python + + +# Create a plot of the results of running the ./benchmark-storage.py script. + + +import argparse +from matplotlib import pyplot as plt + +from sets import Set + + +def plot(filename, subtitle=''): + + # config the plot + plt.xlabel('doc size (KB)') + plt.ylabel('operation time (s)') + title = 'soledad 1000 docs creation/retrieval times' + if subtitle != '': + title += '- %s' % subtitle + plt.title(title) + + x = Set() + ycreate = [] + yget = [] + + ys = [] + #ys.append((ycreate, 'creation time', 'r', '-')) + #ys.append((yget, 'retrieval time', 'b', '-')) + + # read data from file + with open(filename, 'r') as f: + f.readline() + for i in xrange(6): + size, y = f.readline().strip().split(' ') + x.add(int(size)) + ycreate.append(float(y)) + + f.readline() + for i in xrange(6): + size, y = f.readline().strip().split(' ') + x.add(int(size)) + yget.append(float(y)) + + # get doc size in KB + x = list(x) + x.sort() + x = map(lambda val: val / 1024, x) + + # get normalized results per KB + nycreate = [] + nyget = [] + for i in xrange(len(x)): + nycreate.append(ycreate[i]/x[i]) + nyget.append(yget[i]/x[i]) + + ys.append((nycreate, 'creation time per KB', 'r', '-.')) + ys.append((nyget, 'retrieval time per KB', 'b', '-.')) + + for y in ys: + kwargs = { + 'linewidth': 1.0, + 'marker': '.', + 'color': y[2], + 'linestyle': y[3], + } + # normalize by doc size + plt.plot( + x, + y[0], + label=y[1], **kwargs) + + #plt.axes().get_xaxis().set_ticks(x) + #plt.axes().get_xaxis().set_ticklabels(x) + + # annotate max and min values + plt.xlim(0, 1100) + #plt.ylim(0, 350) + plt.grid() + plt.legend() + plt.show() + + +if __name__ == '__main__': + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + 'datafile', + help='the data file to plot') + parser.add_argument( + '-s', dest='subtitle', required=False, default='', + help='a subtitle for the plot') + args = parser.parse_args() + plot(args.datafile, args.subtitle) diff --git a/scripts/profiling/storage/profile-format.py b/scripts/profiling/storage/profile-format.py new file mode 100644 index 00000000..262a52ab --- /dev/null +++ b/scripts/profiling/storage/profile-format.py @@ -0,0 +1,29 @@ +#!/usr/bin/python + +import argparse +import pstats + + +def parse_args(): + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + '-f', dest='statsfiles', action='append', required=True, + help='a stats file') + args = parser.parse_args() + return args.statsfiles + + +def format_stats(statsfiles): + for f in statsfiles: + ps = pstats.Stats(f) + ps.strip_dirs() + ps.sort_stats('time') + ps.print_stats() + ps.sort_stats('cumulative') + ps.print_stats() + + +if __name__ == '__main__': + statsfiles = parse_args() + format_stats(statsfiles) diff --git a/scripts/profiling/storage/profile-storage.py b/scripts/profiling/storage/profile-storage.py new file mode 100755 index 00000000..305e6d5a --- /dev/null +++ b/scripts/profiling/storage/profile-storage.py @@ -0,0 +1,107 @@ +#!/usr/bin/python + +import os +import logging +import getpass +import tempfile +import argparse +import cProfile +import shutil +import pstats +import StringIO +import datetime + + +from client_side_db import get_soledad_instance +from util import ValidateUserHandle + +# profiling args +NUM_DOCS = 1 +DOC_SIZE = 1024**2 + + +# create a logger +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + +def parse_args(): + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + 'user@provider', action=ValidateUserHandle, help='the user handle') + parser.add_argument( + '-b', dest='basedir', required=False, default=None, + help='soledad base directory') + parser.add_argument( + '-p', dest='passphrase', required=False, default=None, + help='the user passphrase') + parser.add_argument( + '-d', dest='logdir', required=False, default='/tmp/', + help='the direcroty to which write the profile stats') + args = parser.parse_args() + # get the password + passphrase = args.passphrase + if passphrase is None: + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + # get the basedir + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + logger.info('Using %s as base directory.' % basedir) + + return args.username, args.provider, passphrase, basedir, args.logdir + +created_docs = [] + +def create_docs(sol, content): + for i in xrange(NUM_DOCS): + doc = sol.create_doc(content) + created_docs.append(doc.doc_id) + +def get_all_docs(sol): + for doc_id in created_docs: + sol.get_doc(doc_id) + +def do_profile(logdir, sol): + fname_prefix = os.path.join( + logdir, + "profile_%s" \ + % datetime.datetime.now().strftime('%Y-%m-%d_%H-%m-%S')) + + # profile create docs + content = {'data': os.urandom(DOC_SIZE/2).encode('hex')} + pr = cProfile.Profile() + pr.runcall( + create_docs, + sol, content) + s = StringIO.StringIO() + ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') + ps.print_stats() + ps.dump_stats("%s_creation.stats" % fname_prefix) + print s.getvalue() + + # profile get all docs + pr = cProfile.Profile() + pr.runcall( + get_all_docs, + sol) + s = StringIO.StringIO() + ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') + ps.dump_stats("%s_retrieval.stats" % fname_prefix) + ps.print_stats() + print s.getvalue() + + +if __name__ == '__main__': + username, provider, passphrase, basedir, logdir = parse_args() + sol = get_soledad_instance( + username, + provider, + passphrase, + basedir) + do_profile(logdir, sol) + shutil.rmtree(basedir) + diff --git a/scripts/profiling/storage/util.py b/scripts/profiling/storage/util.py new file mode 120000 index 00000000..7f16d684 --- /dev/null +++ b/scripts/profiling/storage/util.py @@ -0,0 +1 @@ +../util.py
\ No newline at end of file diff --git a/scripts/profiling/sync/movingaverage.py b/scripts/profiling/sync/movingaverage.py new file mode 120000 index 00000000..098b0a01 --- /dev/null +++ b/scripts/profiling/sync/movingaverage.py @@ -0,0 +1 @@ +../movingaverage.py
\ No newline at end of file diff --git a/scripts/profiling/sync/profile-decoupled.py b/scripts/profiling/sync/profile-decoupled.py new file mode 100644 index 00000000..a844c3c6 --- /dev/null +++ b/scripts/profiling/sync/profile-decoupled.py @@ -0,0 +1,24 @@ +# test_name: soledad-sync +# start_time: 2014-06-12 20:09:11.232317+00:00 +# elapsed_time total_cpu total_memory proc_cpu proc_memory +0.000225 68.400000 46.100000 105.300000 0.527224 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.250167 0.000000 0.255160 +0.707006 76.200000 46.200000 90.000000 0.562369 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +1.413140 63.200000 46.100000 0.000000 0.360199 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +2.123962 0.000000 46.100000 0.000000 0.360199 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +2.833941 31.600000 46.100000 0.000000 0.360248 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +3.541532 5.300000 46.100000 0.000000 0.360298 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +4.253390 14.300000 46.100000 11.100000 0.360347 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +4.967365 5.000000 46.100000 0.000000 0.360347 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +5.680172 5.600000 46.100000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +6.390501 10.500000 46.100000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +7.101711 23.800000 46.000000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +7.810529 30.000000 46.000000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +8.517835 25.000000 46.100000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +9.227455 5.300000 46.000000 9.500000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +9.936479 9.500000 46.000000 10.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +10.645015 52.400000 46.200000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +11.355179 21.100000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +12.066252 36.800000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +12.777689 28.600000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +13.489886 0.000000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +# end_time: 2014-06-12 20:09:25.434677+00:00
\ No newline at end of file diff --git a/run_tests.sh b/scripts/run_tests.sh index e36466f8..e36466f8 100755 --- a/run_tests.sh +++ b/scripts/run_tests.sh diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip index be5d156b..df6ad95d 100644 --- a/server/pkg/requirements.pip +++ b/server/pkg/requirements.pip @@ -4,19 +4,12 @@ simplejson u1db routes PyOpenSSL<0.14 - -# TODO: maybe we just want twisted-web? -twisted>=12.0.0 +twisted # leap deps -- bump me! -leap.soledad.common>=0.3.0 - -# -# Things yet to fix: -# - -# oauth is not strictly needed by us, but we need it -# until u1db adds it to its release as a dep. +leap.soledad.common>=0.6.5 +# XXX -- fix me! +# oauth is not strictly needed by us, but we need it until u1db adds it to its +# release as a dep. oauth - diff --git a/server/pkg/soledad b/server/pkg/soledad-server index ccb3e9b0..811ad55b 100644 --- a/server/pkg/soledad +++ b/server/pkg/soledad-server @@ -34,8 +34,8 @@ case "${1}" in start) echo -n "Starting soledad: twistd" start-stop-daemon --start --quiet \ - --user=${USER} --group=${GROUP} \ --exec ${TWISTD_PATH} -- \ + --uid=${USER} --gid=${GROUP} \ --pidfile=${PIDFILE} \ --logfile=${LOGFILE} \ web \ diff --git a/server/setup.py b/server/setup.py index 573622ce..124ddd32 100644 --- a/server/setup.py +++ b/server/setup.py @@ -35,7 +35,7 @@ if isset('VIRTUAL_ENV') or isset('LEAP_SKIP_INIT'): data_files = None else: # XXX this should go only for linux/mac - data_files = [("/etc/init.d/", ["pkg/soledad"])] + data_files = [("/etc/init.d/", ["pkg/soledad-server"])] trove_classifiers = ( diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 57f600a1..7af4e54b 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -21,10 +21,10 @@ Authentication facilities for Soledad Server. """ +import time import httplib import simplejson as json - from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors from abc import ABCMeta, abstractmethod from routes.mapper import Mapper @@ -32,12 +32,8 @@ from couchdb.client import Server from twisted.python import log from hashlib import sha512 - -from leap.soledad.common import ( - SHARED_DB_NAME, - SHARED_DB_LOCK_DOC_ID_PREFIX, - USER_DB_PREFIX, -) +from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common import USER_DB_PREFIX from leap.soledad.common.errors import InvalidAuthTokenError @@ -354,7 +350,8 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): Token based authentication. """ - TOKENS_DB = "tokens" + TOKENS_DB_PREFIX = "tokens_" + TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds TOKENS_TYPE_KEY = "type" TOKENS_TYPE_DEF = "Token" TOKENS_USER_ID_KEY = "user_id" @@ -414,7 +411,14 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): invalid. """ server = Server(url=self._app.state.couch_url) - dbname = self.TOKENS_DB + # the tokens db rotates every 30 days, and the current db name is + # "tokens_NNN", where NNN is the number of seconds since epoch divided + # by the rotate period in seconds. When rotating, old and new tokens + # db coexist during a certain window of time and valid tokens are + # replicated from the old db to the new one. See: + # https://leap.se/code/issues/6785 + dbname = self.TOKENS_DB_PREFIX + \ + str(int(time.time() / self.TOKENS_DB_EXPIRE)) db = server[dbname] # lookup key is a hash of the token to prevent timing attacks. token = db.get(sha512(token).hexdigest()) |