summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.gitignore1
-rw-r--r--CHANGELOG44
-rw-r--r--README.rst12
-rw-r--r--client/changes/VERSION_COMPAT10
-rw-r--r--client/pkg/requirements.pip24
-rw-r--r--client/src/leap/soledad/client/__init__.py1347
-rw-r--r--client/src/leap/soledad/client/adbapi.py271
-rw-r--r--client/src/leap/soledad/client/api.py882
-rw-r--r--client/src/leap/soledad/client/auth.py9
-rw-r--r--client/src/leap/soledad/client/crypto.py862
-rw-r--r--client/src/leap/soledad/client/encdecpool.py746
-rw-r--r--client/src/leap/soledad/client/events.py60
-rw-r--r--client/src/leap/soledad/client/examples/README4
-rw-r--r--client/src/leap/soledad/client/examples/benchmarks/.gitignore1
-rwxr-xr-xclient/src/leap/soledad/client/examples/benchmarks/get_sample.sh3
-rw-r--r--client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py177
-rw-r--r--client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py177
-rw-r--r--client/src/leap/soledad/client/examples/compare.txt8
-rw-r--r--client/src/leap/soledad/client/examples/manifest.phk50
-rw-r--r--client/src/leap/soledad/client/examples/plot-async-db.py45
-rw-r--r--client/src/leap/soledad/client/examples/run_benchmark.py28
-rw-r--r--client/src/leap/soledad/client/examples/soledad_sync.py65
-rw-r--r--client/src/leap/soledad/client/examples/use_adbapi.py103
-rw-r--r--client/src/leap/soledad/client/examples/use_api.py67
-rw-r--r--client/src/leap/soledad/client/http_target.py622
-rw-r--r--client/src/leap/soledad/client/interfaces.py362
-rw-r--r--client/src/leap/soledad/client/pragmas.py379
-rw-r--r--client/src/leap/soledad/client/secrets.py787
-rw-r--r--client/src/leap/soledad/client/shared_db.py57
-rw-r--r--client/src/leap/soledad/client/sqlcipher.py1422
-rw-r--r--client/src/leap/soledad/client/sync.py178
-rw-r--r--client/src/leap/soledad/client/target.py1469
-rw-r--r--common/changes/VERSION_COMPAT10
-rw-r--r--common/pkg/requirements-testing.pip1
-rw-r--r--common/pkg/requirements.pip9
-rw-r--r--common/setup.cfg2
-rw-r--r--common/setup.py22
-rw-r--r--common/src/leap/soledad/common/__init__.py50
-rw-r--r--common/src/leap/soledad/common/couch.py9
-rw-r--r--common/src/leap/soledad/common/crypto.py22
-rw-r--r--common/src/leap/soledad/common/tests/__init__.py284
-rw-r--r--common/src/leap/soledad/common/tests/couchdb.ini.template208
-rw-r--r--common/src/leap/soledad/common/tests/hacker_crackdown.txt13005
-rw-r--r--common/src/leap/soledad/common/tests/test_async.py144
-rw-r--r--common/src/leap/soledad/common/tests/test_couch.py164
-rw-r--r--common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py209
-rw-r--r--common/src/leap/soledad/common/tests/test_crypto.py127
-rw-r--r--common/src/leap/soledad/common/tests/test_http.py5
-rw-r--r--common/src/leap/soledad/common/tests/test_http_client.py11
-rw-r--r--common/src/leap/soledad/common/tests/test_https.py42
-rw-r--r--common/src/leap/soledad/common/tests/test_server.py357
-rw-r--r--common/src/leap/soledad/common/tests/test_soledad.py273
-rw-r--r--common/src/leap/soledad/common/tests/test_soledad_app.py59
-rw-r--r--common/src/leap/soledad/common/tests/test_soledad_doc.py24
-rw-r--r--common/src/leap/soledad/common/tests/test_sqlcipher.py586
-rw-r--r--common/src/leap/soledad/common/tests/test_sqlcipher_sync.py398
-rw-r--r--common/src/leap/soledad/common/tests/test_sync.py156
-rw-r--r--common/src/leap/soledad/common/tests/test_sync_deferred.py156
-rw-r--r--common/src/leap/soledad/common/tests/test_sync_target.py308
-rw-r--r--common/src/leap/soledad/common/tests/test_target.py794
-rw-r--r--common/src/leap/soledad/common/tests/test_target_soledad.py102
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/__init__.py4
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_backends.py11
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_document.py3
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py11
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py4
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py5
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_https.py4
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_open.py2
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py5
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py4
-rw-r--r--common/src/leap/soledad/common/tests/u1db_tests/test_sync.py6
-rw-r--r--common/src/leap/soledad/common/tests/util.py393
-rw-r--r--docs/debian-repackaging.rst2
-rw-r--r--docs/leap-commit-template7
-rw-r--r--docs/leap-commit-template.README47
-rwxr-xr-xscripts/build_debian_package.sh2
-rw-r--r--scripts/db_access/client_side_db.py168
-rw-r--r--scripts/db_access/reset_db.py132
l---------scripts/db_access/util.py1
-rw-r--r--scripts/ddocs/update_design_docs.py191
-rw-r--r--scripts/profiling/mail/__init__.py184
-rw-r--r--scripts/profiling/mail/couchdb.ini.template224
-rw-r--r--scripts/profiling/mail/couchdb_server.py42
-rw-r--r--scripts/profiling/mail/couchdb_wrapper.py84
-rw-r--r--scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub30
-rw-r--r--scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec57
-rw-r--r--scripts/profiling/mail/mail.py50
-rw-r--r--scripts/profiling/mail/mx.py80
-rw-r--r--scripts/profiling/mail/soledad_client.py40
-rw-r--r--scripts/profiling/mail/soledad_server.py48
-rw-r--r--scripts/profiling/mail/util.py8
-rwxr-xr-xscripts/profiling/spam.py123
-rw-r--r--scripts/profiling/storage/benchmark-storage.py104
-rw-r--r--scripts/profiling/storage/benchmark_storage_utils.py4
l---------scripts/profiling/storage/client_side_db.py1
-rwxr-xr-xscripts/profiling/storage/plot.py94
-rw-r--r--scripts/profiling/storage/profile-format.py29
-rwxr-xr-xscripts/profiling/storage/profile-storage.py107
l---------scripts/profiling/storage/util.py1
l---------scripts/profiling/sync/movingaverage.py1
-rw-r--r--scripts/profiling/sync/profile-decoupled.py24
-rwxr-xr-xscripts/run_tests.sh (renamed from run_tests.sh)0
-rw-r--r--server/pkg/requirements.pip17
-rw-r--r--server/pkg/soledad-server (renamed from server/pkg/soledad)2
-rw-r--r--server/setup.py2
-rw-r--r--server/src/leap/soledad/server/auth.py22
107 files changed, 22461 insertions, 7728 deletions
diff --git a/.gitignore b/.gitignore
index bd170f79..c502541e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,4 @@ MANIFEST
*.pyc
*.log
*.*~
+*.csv
diff --git a/CHANGELOG b/CHANGELOG
index 4e3f2038..3c05d330 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,47 @@
+0.7.0 Jun 01 2015:
+Client:
+ o Do not depend on pysqlite2 (#2945).
+ o Reset syncer connection when getting HTTP error during sync (#5855).
+ o Wait for last post request to finish before starting a new one (#5975).
+ o Use TLS v1 in soledad client. Fixes partially #6437
+ o Retry on sqlcipher thread timeouts (#6625).
+ o Fix the order of insertion of documents when using workers for decrypting
+ incoming documents during a sync. Closes #6757.
+ o Add dependency on Twisted. Closes #6797.
+ o Fix the log message when a local secret is not found so it's less
+ confusing. Closes #6892.
+ o Always initialize the sync db to allow for both asynchronous encryption
+ and asynchronous decryption when syncing.
+ o Fallback to utf-8 if confidence on chardet guessing is too low.
+ o Refactor asynchronous encryption/decryption code to its own file.
+ o Fix logging and graceful failing when exceptions are raised during sync.
+ o Improve log messages when concurrently fetching documents from the server.
+ o Store all incoming documents in the sync db (#5895).
+ o Include the IV in the encrypted document MAC (#6400).
+ o Adapt soledad to the new events api on leap.common. Related to #6359.
+ o Add a pool of HTTP/HTTPS connections that is able to verify the server
+ certificate against a given CA certificate.
+ o Use twisted.enterprise.adbapi for access to the sync database.
+ o Use twisted.web.client for client sync.
+
+Common:
+ o Include couch design docs source files in source distribution and only
+ compile ddocs.py when building the package (#5896).
+ o Bail out if cdocs/ dir does not exist. Closes: #6671
+ o Remove unneeded parameters from CouchServerState initialization. Closes
+ #6833.
+ o Adapt soledad to the new events api on leap.common. Related to #6359.
+
+Server:
+ o Run daemon as user soledad (#6436).
+ o Avoid use of SSLv3 (#6437).
+ o Fix server initscript location (#6557).
+ o Add dependency on Twisted. Closes #6797.
+ o Remove unneeded parameters from CouchServerState initialization. Closes
+ #6833.
+ o Fix server daemon uid and gid by passing them to twistd on the initscript.
+ o Use monthly token databases. Closes #6785.
+
0.6.5 Apr 09 2015:
Server:
o Remove unneeded parameters from CouchServerState initialization. Closes
diff --git a/README.rst b/README.rst
index fb909120..887b3df1 100644
--- a/README.rst
+++ b/README.rst
@@ -27,10 +27,16 @@ repository:
:target: https://crate.io/packages/leap.soledad.server
-Library dependencies
---------------------
+Compatibility
+-------------
+
+* Soledad Server >= 0.7.0 is incompatible with client < 0.7.0 because of
+ modifications on encrypted document MAC calculation.
+
+* Soledad Server >= 0.7.0 is incompatible with LEAP Platform < 0.6.1 because
+ that platform version implements ephemeral tokens databases and Soledad
+ Server needs to act accordingly.
-* ``libsqlite3-dev``
Tests
-----
diff --git a/client/changes/VERSION_COMPAT b/client/changes/VERSION_COMPAT
index e69de29b..cc00ecf7 100644
--- a/client/changes/VERSION_COMPAT
+++ b/client/changes/VERSION_COMPAT
@@ -0,0 +1,10 @@
+#################################################
+# This file keeps track of the recent changes
+# introduced in internal leap dependencies.
+# Add your changes here so we can properly update
+# requirements.pip during the release process.
+# (leave header when resetting)
+#################################################
+#
+# BEGIN DEPENDENCY LIST -------------------------
+# leap.foo.bar>=x.y.z
diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip
index ae8d2dac..9fffdbe3 100644
--- a/client/pkg/requirements.pip
+++ b/client/pkg/requirements.pip
@@ -1,25 +1,17 @@
-pysqlcipher
+pysqlcipher>2.6.3
simplejson
u1db
scrypt
pycryptopp
cchardet
-taskthread
zope.proxy
+twisted
-#
-# leap deps
-#
+# leap deps -- bump me!
+leap.common>=0.4
+leap.soledad.common>=0.6.5
-leap.soledad.common>=0.3.8
-
-#
-# XXX things to fix yet:
-#
-
-# this is not strictly needed by us, but we need it
-# until u1db adds it to its release as a dep.
+# XXX -- fix me!
+# oauth is not strictly needed by us, but we need it until u1db adds it to its
+# release as a dep.
oauth
-
-# pysqlite should not be a dep, see #2945
-pysqlite
diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index 07255406..245a8971 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -16,1351 +16,12 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Soledad - Synchronization Of Locally Encrypted Data Among Devices.
-
-Soledad is the part of LEAP that manages storage and synchronization of
-application data. It is built on top of U1DB reference Python API and
-implements (1) a SQLCipher backend for local storage in the client, (2) a
-SyncTarget that encrypts data before syncing, and (3) a CouchDB backend for
-remote storage in the server side.
-"""
-import binascii
-import errno
-import httplib
-import logging
-import os
-import socket
-import ssl
-import urlparse
-import hmac
-
-from hashlib import sha256
-
-try:
- import cchardet as chardet
-except ImportError:
- import chardet
-
-from u1db.remote import http_client
-from u1db.remote.ssl_match_hostname import match_hostname
-
-import scrypt
-import simplejson as json
-
-from leap.common.config import get_path_prefix
-from leap.soledad.common import (
- SHARED_DB_NAME,
- soledad_assert,
- soledad_assert_type
-)
-from leap.soledad.common.errors import (
- InvalidTokenError,
- NotLockedError,
- AlreadyLockedError,
- LockTimedOutError,
-)
-from leap.soledad.common.crypto import (
- MacMethods,
- UnknownMacMethod,
- WrongMac,
- MAC_KEY,
- MAC_METHOD_KEY,
-)
-from leap.soledad.client.events import (
- SOLEDAD_CREATING_KEYS,
- SOLEDAD_DONE_CREATING_KEYS,
- SOLEDAD_DOWNLOADING_KEYS,
- SOLEDAD_DONE_DOWNLOADING_KEYS,
- SOLEDAD_UPLOADING_KEYS,
- SOLEDAD_DONE_UPLOADING_KEYS,
- SOLEDAD_NEW_DATA_TO_SYNC,
- SOLEDAD_DONE_DATA_SYNC,
- signal,
-)
-from leap.soledad.common.document import SoledadDocument
-from leap.soledad.client.crypto import SoledadCrypto
-from leap.soledad.client.shared_db import SoledadSharedDatabase
-from leap.soledad.client.sqlcipher import open as sqlcipher_open
-from leap.soledad.client.sqlcipher import SQLCipherDatabase
-from leap.soledad.client.target import SoledadSyncTarget
-
-
-logger = logging.getLogger(name=__name__)
-
-
-#
-# Constants
-#
-
-SOLEDAD_CERT = None
"""
-Path to the certificate file used to certify the SSL connection between
-Soledad client and server.
-"""
-
-
-#
-# Soledad: local encrypted storage and remote encrypted sync.
-#
-
-class NoStorageSecret(Exception):
- """
- Raised when trying to use a storage secret but none is available.
- """
- pass
-
-
-class PassphraseTooShort(Exception):
- """
- Raised when trying to change the passphrase but the provided passphrase is
- too short.
- """
-
-
-class BootstrapSequenceError(Exception):
- """
- Raised when an attempt to generate a secret and store it in a recovery
- documents on server failed.
- """
-
-
-class Soledad(object):
- """
- Soledad provides encrypted data storage and sync.
-
- A Soledad instance is used to store and retrieve data in a local encrypted
- database and synchronize this database with Soledad server.
-
- This class is also responsible for bootstrapping users' account by
- creating cryptographic secrets and/or storing/fetching them on Soledad
- server.
-
- Soledad uses C{leap.common.events} to signal events. The possible events
- to be signaled are:
-
- SOLEDAD_CREATING_KEYS: emitted during bootstrap sequence when key
- generation starts.
- SOLEDAD_DONE_CREATING_KEYS: emitted during bootstrap sequence when key
- generation finishes.
- SOLEDAD_UPLOADING_KEYS: emitted during bootstrap sequence when soledad
- starts sending keys to server.
- SOLEDAD_DONE_UPLOADING_KEYS: emitted during bootstrap sequence when
- soledad finishes sending keys to server.
- SOLEDAD_DOWNLOADING_KEYS: emitted during bootstrap sequence when
- soledad starts to retrieve keys from server.
- SOLEDAD_DONE_DOWNLOADING_KEYS: emitted during bootstrap sequence when
- soledad finishes downloading keys from server.
- SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when
- there's indeed new data to be synchronized between local database
- replica and server's replica.
- SOLEDAD_DONE_DATA_SYNC: emitted inside C{sync()} method when it has
- finished synchronizing with remote replica.
- """
-
- LOCAL_DATABASE_FILE_NAME = 'soledad.u1db'
- """
- The name of the local SQLCipher U1DB database file.
- """
-
- STORAGE_SECRETS_FILE_NAME = "soledad.json"
- """
- The name of the file where the storage secrets will be stored.
- """
-
- GENERATED_SECRET_LENGTH = 1024
- """
- The length of the generated secret used to derive keys for symmetric
- encryption for local and remote storage.
- """
-
- LOCAL_STORAGE_SECRET_LENGTH = 512
- """
- The length of the secret used to derive a passphrase for the SQLCipher
- database.
- """
-
- REMOTE_STORAGE_SECRET_LENGTH = \
- GENERATED_SECRET_LENGTH - LOCAL_STORAGE_SECRET_LENGTH
- """
- The length of the secret used to derive an encryption key and a MAC auth
- key for remote storage.
- """
-
- SALT_LENGTH = 64
- """
- The length of the salt used to derive the key for the storage secret
- encryption.
- """
-
- MINIMUM_PASSPHRASE_LENGTH = 6
- """
- The minimum length for a passphrase. The passphrase length is only checked
- when the user changes her passphrase, not when she instantiates Soledad.
- """
-
- IV_SEPARATOR = ":"
- """
- A separator used for storing the encryption initial value prepended to the
- ciphertext.
- """
-
- UUID_KEY = 'uuid'
- STORAGE_SECRETS_KEY = 'storage_secrets'
- SECRET_KEY = 'secret'
- CIPHER_KEY = 'cipher'
- LENGTH_KEY = 'length'
- KDF_KEY = 'kdf'
- KDF_SALT_KEY = 'kdf_salt'
- KDF_LENGTH_KEY = 'kdf_length'
- KDF_SCRYPT = 'scrypt'
- CIPHER_AES256 = 'aes256'
- """
- Keys used to access storage secrets in recovery documents.
- """
-
- DEFAULT_PREFIX = os.path.join(get_path_prefix(), 'leap', 'soledad')
- """
- Prefix for default values for path.
- """
-
- def __init__(self, uuid, passphrase, secrets_path, local_db_path,
- server_url, cert_file,
- auth_token=None, secret_id=None, defer_encryption=True):
- """
- Initialize configuration, cryptographic keys and dbs.
-
- :param uuid: User's uuid.
- :type uuid: str
-
- :param passphrase: The passphrase for locking and unlocking encryption
- secrets for local and remote storage.
- :type passphrase: unicode
-
- :param secrets_path: Path for storing encrypted key used for
- symmetric encryption.
- :type secrets_path: str
-
- :param local_db_path: Path for local encrypted storage db.
- :type local_db_path: str
-
- :param server_url: URL for Soledad server. This is used either to sync
- with the user's remote db and to interact with the
- shared recovery database.
- :type server_url: str
-
- :param cert_file: Path to the certificate of the ca used
- to validate the SSL certificate used by the remote
- soledad server.
- :type cert_file: str
-
- :param auth_token: Authorization token for accessing remote databases.
- :type auth_token: str
-
- :param secret_id: The id of the storage secret to be used.
- :type secret_id: str
-
- :param defer_encryption: Whether to defer encryption/decryption of
- documents, or do it inline while syncing.
- :type defer_encryption: bool
-
- :raise BootstrapSequenceError: Raised when the secret generation and
- storage on server sequence has failed
- for some reason.
- """
- # get config params
- self._uuid = uuid
- soledad_assert_type(passphrase, unicode)
- self._passphrase = passphrase
- # init crypto variables
- self._secrets = {}
- self._secret_id = secret_id
- self._defer_encryption = defer_encryption
-
- self._init_config(secrets_path, local_db_path, server_url)
-
- self._set_token(auth_token)
- self._shared_db_instance = None
- # configure SSL certificate
- global SOLEDAD_CERT
- SOLEDAD_CERT = cert_file
- # initiate bootstrap sequence
- self._bootstrap() # might raise BootstrapSequenceError()
-
- def _init_config(self, secrets_path, local_db_path, server_url):
- """
- Initialize configuration using default values for missing params.
- """
- # initialize secrets_path
- self._secrets_path = secrets_path
- if self._secrets_path is None:
- self._secrets_path = os.path.join(
- self.DEFAULT_PREFIX, self.STORAGE_SECRETS_FILE_NAME)
- # initialize local_db_path
- self._local_db_path = local_db_path
- if self._local_db_path is None:
- self._local_db_path = os.path.join(
- self.DEFAULT_PREFIX, self.LOCAL_DATABASE_FILE_NAME)
- # initialize server_url
- self._server_url = server_url
- soledad_assert(
- self._server_url is not None,
- 'Missing URL for Soledad server.')
-
- #
- # initialization/destruction methods
- #
-
- def _get_or_gen_crypto_secrets(self):
- """
- Retrieves or generates the crypto secrets.
-
- Might raise BootstrapSequenceError
- """
- doc = self._get_secrets_from_shared_db()
-
- if doc:
- logger.info(
- 'Found cryptographic secrets in shared recovery '
- 'database.')
- _, mac = self.import_recovery_document(doc.content)
- if mac is False:
- self.put_secrets_in_shared_db()
- self._store_secrets() # save new secrets in local file
- if self._secret_id is None:
- self._set_secret_id(self._secrets.items()[0][0])
- else:
- # STAGE 3 - there are no secrets in server also, so
- # generate a secret and store it in remote db.
- logger.info(
- 'No cryptographic secrets found, creating new '
- ' secrets...')
- self._set_secret_id(self._gen_secret())
- try:
- self._put_secrets_in_shared_db()
- except Exception as ex:
- # storing generated secret in shared db failed for
- # some reason, so we erase the generated secret and
- # raise.
- try:
- os.unlink(self._secrets_path)
- except OSError as e:
- if e.errno != errno.ENOENT: # no such file or directory
- logger.exception(e)
- logger.exception(ex)
- raise BootstrapSequenceError(
- 'Could not store generated secret in the shared '
- 'database, bailing out...')
-
- def _bootstrap(self):
- """
- Bootstrap local Soledad instance.
-
- Soledad Client bootstrap is the following sequence of stages:
-
- * stage 0 - local environment setup.
- - directory initialization.
- - crypto submodule initialization
- * stage 1 - local secret loading:
- - if secrets exist locally, load them.
- * stage 2 - remote secret loading:
- - else, if secrets exist in server, download them.
- * stage 3 - secret generation:
- - else, generate a new secret and store in server.
- * stage 4 - database initialization.
-
- This method decides which bootstrap stages have already been performed
- and performs the missing ones in order.
-
- :raise BootstrapSequenceError: Raised when the secret generation and
- storage on server sequence has failed for some reason.
- """
- # STAGE 0 - local environment setup
- self._init_dirs()
- self._crypto = SoledadCrypto(self)
-
- secrets_problem = None
-
- # STAGE 1 - verify if secrets exist locally
- if not self._has_secret(): # try to load from local storage.
-
- # STAGE 2 - there are no secrets in local storage, so try to fetch
- # encrypted secrets from server.
- logger.info(
- 'Trying to fetch cryptographic secrets from shared recovery '
- 'database...')
-
- # --- start of atomic operation in shared db ---
-
- # obtain lock on shared db
- token = timeout = None
- try:
- token, timeout = self._shared_db.lock()
- except AlreadyLockedError:
- raise BootstrapSequenceError('Database is already locked.')
- except LockTimedOutError:
- raise BootstrapSequenceError('Lock operation timed out.')
-
- try:
- self._get_or_gen_crypto_secrets()
- except Exception as e:
- secrets_problem = e
-
- # release the lock on shared db
- try:
- self._shared_db.unlock(token)
- self._shared_db.close()
- except NotLockedError:
- # for some reason the lock expired. Despite that, secret
- # loading or generation/storage must have been executed
- # successfully, so we pass.
- pass
- except InvalidTokenError:
- # here, our lock has not only expired but also some other
- # client application has obtained a new lock and is currently
- # doing its thing in the shared database. Using the same
- # reasoning as above, we assume everything went smooth and
- # pass.
- pass
- except Exception as e:
- logger.error("Unhandled exception when unlocking shared "
- "database.")
- logger.exception(e)
-
- # --- end of atomic operation in shared db ---
-
- # STAGE 4 - local database initialization
- if secrets_problem is None:
- self._init_db()
- else:
- raise secrets_problem
-
- def _init_dirs(self):
- """
- Create work directories.
-
- :raise OSError: in case file exists and is not a dir.
- """
- paths = map(
- lambda x: os.path.dirname(x),
- [self._local_db_path, self._secrets_path])
- for path in paths:
- try:
- if not os.path.isdir(path):
- logger.info('Creating directory: %s.' % path)
- os.makedirs(path)
- except OSError as exc:
- if exc.errno == errno.EEXIST and os.path.isdir(path):
- pass
- else:
- raise
-
- def _init_db(self):
- """
- Initialize the U1DB SQLCipher database for local storage.
-
- Currently, Soledad uses the default SQLCipher cipher, i.e.
- 'aes-256-cbc'. We use scrypt to derive a 256-bit encryption key and
- uses the 'raw PRAGMA key' format to handle the key to SQLCipher.
-
- The first C{self.REMOTE_STORAGE_SECRET_LENGTH} bytes of the storage
- secret are used for remote storage encryption. We use the next
- C{self.LOCAL_STORAGE_SECRET} bytes to derive a key for local storage.
- From these bytes, the first C{self.SALT_LENGTH} are used as the salt
- and the rest as the password for the scrypt hashing.
- """
- # salt indexes
- salt_start = self.REMOTE_STORAGE_SECRET_LENGTH
- salt_end = salt_start + self.SALT_LENGTH
- # password indexes
- pwd_start = salt_end
- pwd_end = salt_start + self.LOCAL_STORAGE_SECRET_LENGTH
- # calculate the key for local encryption
- secret = self._get_storage_secret()
- key = scrypt.hash(
- secret[pwd_start:pwd_end], # the password
- secret[salt_start:salt_end], # the salt
- buflen=32, # we need a key with 256 bits (32 bytes)
- )
-
- self._db = sqlcipher_open(
- self._local_db_path,
- binascii.b2a_hex(key), # sqlcipher only accepts the hex version
- create=True,
- document_factory=SoledadDocument,
- crypto=self._crypto,
- raw_key=True,
- defer_encryption=self._defer_encryption)
-
- def close(self):
- """
- Close underlying U1DB database.
- """
- logger.debug("Closing soledad")
- if hasattr(self, '_db') and isinstance(
- self._db,
- SQLCipherDatabase):
- self._db.stop_sync()
- self._db.close()
-
- #
- # Management of secret for symmetric encryption.
- #
-
- def _get_storage_secret(self):
- """
- Return the storage secret.
-
- Storage secret is encrypted before being stored. This method decrypts
- and returns the stored secret.
-
- :return: The storage secret.
- :rtype: str
- """
- # calculate the encryption key
- key = scrypt.hash(
- self._passphrase_as_string(),
- # the salt is stored base64 encoded
- binascii.a2b_base64(
- self._secrets[self._secret_id][self.KDF_SALT_KEY]),
- buflen=32, # we need a key with 256 bits (32 bytes).
- )
- # recover the initial value and ciphertext
- iv, ciphertext = self._secrets[self._secret_id][self.SECRET_KEY].split(
- self.IV_SEPARATOR, 1)
- ciphertext = binascii.a2b_base64(ciphertext)
- return self._crypto.decrypt_sym(ciphertext, key, iv=iv)
-
- def _set_secret_id(self, secret_id):
- """
- Define the id of the storage secret to be used.
-
- This method will also replace the secret in the crypto object.
-
- :param secret_id: The id of the storage secret to be used.
- :type secret_id: str
- """
- self._secret_id = secret_id
-
- def _load_secrets(self):
- """
- Load storage secrets from local file.
- """
- # does the file exist in disk?
- if not os.path.isfile(self._secrets_path):
- raise IOError('File does not exist: %s' % self._secrets_path)
- # read storage secrets from file
- content = None
- with open(self._secrets_path, 'r') as f:
- content = json.loads(f.read())
- _, mac = self.import_recovery_document(content)
- if mac is False:
- self._store_secrets()
- self._put_secrets_in_shared_db()
- # choose first secret if no secret_id was given
- if self._secret_id is None:
- self._set_secret_id(self._secrets.items()[0][0])
-
- def _has_secret(self):
- """
- Return whether there is a storage secret available for use or not.
-
- :return: Whether there's a storage secret for symmetric encryption.
- :rtype: bool
- """
- if self._secret_id is None or self._secret_id not in self._secrets:
- try:
- self._load_secrets() # try to load from disk
- except IOError, e:
- logger.warning('IOError: %s' % str(e))
- try:
- self._get_storage_secret()
- return True
- except Exception:
- return False
-
- def _gen_secret(self):
- """
- Generate a secret for symmetric encryption and store in a local
- encrypted file.
-
- This method emits the following signals:
-
- * SOLEDAD_CREATING_KEYS
- * SOLEDAD_DONE_CREATING_KEYS
-
- A secret has the following structure:
-
- {
- '<secret_id>': {
- 'kdf': 'scrypt',
- 'kdf_salt': '<b64 repr of salt>'
- 'kdf_length': <key length>
- 'cipher': 'aes256',
- 'length': <secret length>,
- 'secret': '<encrypted b64 repr of storage_secret>',
- }
- }
-
- :return: The id of the generated secret.
- :rtype: str
- """
- signal(SOLEDAD_CREATING_KEYS, self._uuid)
- # generate random secret
- secret = os.urandom(self.GENERATED_SECRET_LENGTH)
- secret_id = sha256(secret).hexdigest()
- # generate random salt
- salt = os.urandom(self.SALT_LENGTH)
- # get a 256-bit key
- key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
- iv, ciphertext = self._crypto.encrypt_sym(secret, key)
- self._secrets[secret_id] = {
- # leap.soledad.crypto submodule uses AES256 for symmetric
- # encryption.
- self.KDF_KEY: self.KDF_SCRYPT,
- self.KDF_SALT_KEY: binascii.b2a_base64(salt),
- self.KDF_LENGTH_KEY: len(key),
- self.CIPHER_KEY: self.CIPHER_AES256,
- self.LENGTH_KEY: len(secret),
- self.SECRET_KEY: '%s%s%s' % (
- str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
- }
- self._store_secrets()
- signal(SOLEDAD_DONE_CREATING_KEYS, self._uuid)
- return secret_id
-
- def _store_secrets(self):
- """
- Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}.
- """
- with open(self._secrets_path, 'w') as f:
- f.write(
- json.dumps(
- self.export_recovery_document()))
-
- def change_passphrase(self, new_passphrase):
- """
- Change the passphrase that encrypts the storage secret.
-
- :param new_passphrase: The new passphrase.
- :type new_passphrase: unicode
-
- :raise NoStorageSecret: Raised if there's no storage secret available.
- """
- # maybe we want to add more checks to guarantee passphrase is
- # reasonable?
- soledad_assert_type(new_passphrase, unicode)
- if len(new_passphrase) < self.MINIMUM_PASSPHRASE_LENGTH:
- raise PassphraseTooShort(
- 'Passphrase must be at least %d characters long!' %
- self.MINIMUM_PASSPHRASE_LENGTH)
- # ensure there's a secret for which the passphrase will be changed.
- if not self._has_secret():
- raise NoStorageSecret()
- secret = self._get_storage_secret()
- # generate random salt
- new_salt = os.urandom(self.SALT_LENGTH)
- # get a 256-bit key
- key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32)
- iv, ciphertext = self._crypto.encrypt_sym(secret, key)
- # XXX update all secrets in the dict
- self._secrets[self._secret_id] = {
- # leap.soledad.crypto submodule uses AES256 for symmetric
- # encryption.
- self.KDF_KEY: self.KDF_SCRYPT, # TODO: remove hard coded kdf
- self.KDF_SALT_KEY: binascii.b2a_base64(new_salt),
- self.KDF_LENGTH_KEY: len(key),
- self.CIPHER_KEY: self.CIPHER_AES256,
- self.LENGTH_KEY: len(secret),
- self.SECRET_KEY: '%s%s%s' % (
- str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
- }
- self._passphrase = new_passphrase
- self._store_secrets()
- self._put_secrets_in_shared_db()
-
- #
- # General crypto utility methods.
- #
-
- @property
- def _shared_db(self):
- """
- Return an instance of the shared recovery database object.
-
- :return: The shared database.
- :rtype: SoledadSharedDatabase
- """
- if self._shared_db_instance is None:
- self._shared_db_instance = SoledadSharedDatabase.open_database(
- urlparse.urljoin(self.server_url, SHARED_DB_NAME),
- self._uuid,
- False, # db should exist at this point.
- creds=self._creds)
- return self._shared_db_instance
-
- def _shared_db_doc_id(self):
- """
- Calculate the doc_id of the document in the shared db that stores key
- material.
-
- :return: the hash
- :rtype: str
- """
- return sha256(
- '%s%s' %
- (self._passphrase_as_string(), self.uuid)).hexdigest()
-
- def _get_secrets_from_shared_db(self):
- """
- Retrieve the document with encrypted key material from the shared
- database.
-
- :return: a document with encrypted key material in its contents
- :rtype: SoledadDocument
- """
- signal(SOLEDAD_DOWNLOADING_KEYS, self._uuid)
- db = self._shared_db
- if not db:
- logger.warning('No shared db found')
- return
- doc = db.get_doc(self._shared_db_doc_id())
- signal(SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid)
- return doc
-
- def _put_secrets_in_shared_db(self):
- """
- Assert local keys are the same as shared db's ones.
-
- Try to fetch keys from shared recovery database. If they already exist
- in the remote db, assert that that data is the same as local data.
- Otherwise, upload keys to shared recovery database.
- """
- soledad_assert(
- self._has_secret(),
- 'Tried to send keys to server but they don\'t exist in local '
- 'storage.')
- # try to get secrets doc from server, otherwise create it
- doc = self._get_secrets_from_shared_db()
- if doc is None:
- doc = SoledadDocument(
- doc_id=self._shared_db_doc_id())
- # fill doc with encrypted secrets
- doc.content = self.export_recovery_document()
- # upload secrets to server
- signal(SOLEDAD_UPLOADING_KEYS, self._uuid)
- db = self._shared_db
- if not db:
- logger.warning('No shared db found')
- return
- db.put_doc(doc)
- signal(SOLEDAD_DONE_UPLOADING_KEYS, self._uuid)
-
- #
- # Document storage, retrieval and sync.
- #
-
- def put_doc(self, doc):
- """
- Update a document in the local encrypted database.
-
- ============================== WARNING ==============================
- This method converts the document's contents to unicode in-place. This
- means that after calling C{put_doc(doc)}, the contents of the
- document, i.e. C{doc.content}, might be different from before the
- call.
- ============================== WARNING ==============================
-
- :param doc: the document to update
- :type doc: SoledadDocument
-
- :return: the new revision identifier for the document
- :rtype: str
- """
- doc.content = self._convert_to_unicode(doc.content)
- return self._db.put_doc(doc)
-
- def delete_doc(self, doc):
- """
- Delete a document from the local encrypted database.
-
- :param doc: the document to delete
- :type doc: SoledadDocument
-
- :return: the new revision identifier for the document
- :rtype: str
- """
- return self._db.delete_doc(doc)
-
- def get_doc(self, doc_id, include_deleted=False):
- """
- Retrieve a document from the local encrypted database.
-
- :param doc_id: the unique document identifier
- :type doc_id: str
- :param include_deleted: if True, deleted documents will be
- returned with empty content; otherwise asking
- for a deleted document will return None
- :type include_deleted: bool
-
- :return: the document object or None
- :rtype: SoledadDocument
- """
- return self._db.get_doc(doc_id, include_deleted=include_deleted)
-
- def get_docs(self, doc_ids, check_for_conflicts=True,
- include_deleted=False):
- """
- Get the content for many documents.
-
- :param doc_ids: a list of document identifiers
- :type doc_ids: list
- :param check_for_conflicts: if set False, then the conflict check will
- be skipped, and 'None' will be returned instead of True/False
- :type check_for_conflicts: bool
-
- :return: iterable giving the Document object for each document id
- in matching doc_ids order.
- :rtype: generator
- """
- return self._db.get_docs(
- doc_ids, check_for_conflicts=check_for_conflicts,
- include_deleted=include_deleted)
-
- def get_all_docs(self, include_deleted=False):
- """Get the JSON content for all documents in the database.
-
- :param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise deleted
- documents will not be included in the results.
- :return: (generation, [Document])
- The current generation of the database, followed by a list of
- all the documents in the database.
- """
- return self._db.get_all_docs(include_deleted)
-
- def _convert_to_unicode(self, content):
- """
- Converts content to unicode (or all the strings in content)
-
- NOTE: Even though this method supports any type, it will
- currently ignore contents of lists, tuple or any other
- iterable than dict. We don't need support for these at the
- moment
-
- :param content: content to convert
- :type content: object
-
- :rtype: object
- """
- if isinstance(content, unicode):
- return content
- elif isinstance(content, str):
- result = chardet.detect(content)
- default = "utf-8"
- encoding = result["encoding"] or default
- try:
- content = content.decode(encoding)
- except UnicodeError as e:
- logger.error("Unicode error: {0!r}. Using 'replace'".format(e))
- content = content.decode(encoding, 'replace')
- return content
- else:
- if isinstance(content, dict):
- for key in content.keys():
- content[key] = self._convert_to_unicode(content[key])
- return content
-
- def create_doc(self, content, doc_id=None):
- """
- Create a new document in the local encrypted database.
-
- :param content: the contents of the new document
- :type content: dict
- :param doc_id: an optional identifier specifying the document id
- :type doc_id: str
-
- :return: the new document
- :rtype: SoledadDocument
- """
- return self._db.create_doc(
- self._convert_to_unicode(content), doc_id=doc_id)
-
- def create_doc_from_json(self, json, doc_id=None):
- """
- Create a new document.
-
- You can optionally specify the document identifier, but the document
- must not already exist. See 'put_doc' if you want to override an
- existing document.
- If the database specifies a maximum document size and the document
- exceeds it, create will fail and raise a DocumentTooBig exception.
-
- :param json: The JSON document string
- :type json: str
- :param doc_id: An optional identifier specifying the document id.
- :type doc_id:
- :return: The new document
- :rtype: SoledadDocument
- """
- return self._db.create_doc_from_json(json, doc_id=doc_id)
-
- def create_index(self, index_name, *index_expressions):
- """
- Create an named index, which can then be queried for future lookups.
- Creating an index which already exists is not an error, and is cheap.
- Creating an index which does not match the index_expressions of the
- existing index is an error.
- Creating an index will block until the expressions have been evaluated
- and the index generated.
-
- :param index_name: A unique name which can be used as a key prefix
- :type index_name: str
- :param index_expressions: index expressions defining the index
- information.
- :type index_expressions: dict
-
- Examples:
-
- "fieldname", or "fieldname.subfieldname" to index alphabetically
- sorted on the contents of a field.
-
- "number(fieldname, width)", "lower(fieldname)"
- """
- if self._db:
- return self._db.create_index(
- index_name, *index_expressions)
-
- def delete_index(self, index_name):
- """
- Remove a named index.
-
- :param index_name: The name of the index we are removing
- :type index_name: str
- """
- if self._db:
- return self._db.delete_index(index_name)
-
- def list_indexes(self):
- """
- List the definitions of all known indexes.
-
- :return: A list of [('index-name', ['field', 'field2'])] definitions.
- :rtype: list
- """
- if self._db:
- return self._db.list_indexes()
-
- def get_from_index(self, index_name, *key_values):
- """
- Return documents that match the keys supplied.
-
- You must supply exactly the same number of values as have been defined
- in the index. It is possible to do a prefix match by using '*' to
- indicate a wildcard match. You can only supply '*' to trailing entries,
- (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.)
- It is also possible to append a '*' to the last supplied value (eg
- 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*')
-
- :param index_name: The index to query
- :type index_name: str
- :param key_values: values to match. eg, if you have
- an index with 3 fields then you would have:
- get_from_index(index_name, val1, val2, val3)
- :type key_values: tuple
- :return: List of [Document]
- :rtype: list
- """
- if self._db:
- return self._db.get_from_index(index_name, *key_values)
-
- def get_count_from_index(self, index_name, *key_values):
- """
- Return the count of the documents that match the keys and
- values supplied.
-
- :param index_name: The index to query
- :type index_name: str
- :param key_values: values to match. eg, if you have
- an index with 3 fields then you would have:
- get_from_index(index_name, val1, val2, val3)
- :type key_values: tuple
- :return: count.
- :rtype: int
- """
- if self._db:
- return self._db.get_count_from_index(index_name, *key_values)
-
- def get_range_from_index(self, index_name, start_value, end_value):
- """
- Return documents that fall within the specified range.
-
- Both ends of the range are inclusive. For both start_value and
- end_value, one must supply exactly the same number of values as have
- been defined in the index, or pass None. In case of a single column
- index, a string is accepted as an alternative for a tuple with a single
- value. It is possible to do a prefix match by using '*' to indicate
- a wildcard match. You can only supply '*' to trailing entries, (eg
- 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also
- possible to append a '*' to the last supplied value (eg 'val*', '*',
- '*' or 'val', 'val*', '*', but not 'val*', 'val', '*')
-
- :param index_name: The index to query
- :type index_name: str
- :param start_values: tuples of values that define the lower bound of
- the range. eg, if you have an index with 3 fields then you would
- have: (val1, val2, val3)
- :type start_values: tuple
- :param end_values: tuples of values that define the upper bound of the
- range. eg, if you have an index with 3 fields then you would have:
- (val1, val2, val3)
- :type end_values: tuple
- :return: List of [Document]
- :rtype: list
- """
- if self._db:
- return self._db.get_range_from_index(
- index_name, start_value, end_value)
-
- def get_index_keys(self, index_name):
- """
- Return all keys under which documents are indexed in this index.
-
- :param index_name: The index to query
- :type index_name: str
- :return: [] A list of tuples of indexed keys.
- :rtype: list
- """
- if self._db:
- return self._db.get_index_keys(index_name)
-
- def get_doc_conflicts(self, doc_id):
- """
- Get the list of conflicts for the given document.
-
- :param doc_id: the document id
- :type doc_id: str
-
- :return: a list of the document entries that are conflicted
- :rtype: list
- """
- if self._db:
- return self._db.get_doc_conflicts(doc_id)
-
- def resolve_doc(self, doc, conflicted_doc_revs):
- """
- Mark a document as no longer conflicted.
-
- :param doc: a document with the new content to be inserted.
- :type doc: SoledadDocument
- :param conflicted_doc_revs: a list of revisions that the new content
- supersedes.
- :type conflicted_doc_revs: list
- """
- if self._db:
- return self._db.resolve_doc(doc, conflicted_doc_revs)
-
- def sync(self, defer_decryption=True):
- """
- Synchronize the local encrypted replica with a remote replica.
-
- This method blocks until a syncing lock is acquired, so there are no
- attempts of concurrent syncs from the same client replica.
-
- :param url: the url of the target replica to sync with
- :type url: str
-
- :param defer_decryption: Whether to defer the decryption process using
- the intermediate database. If False,
- decryption will be done inline.
- :type defer_decryption: bool
-
- :return: The local generation before the synchronisation was
- performed.
- :rtype: str
- """
- if self._db:
- try:
- local_gen = self._db.sync(
- urlparse.urljoin(self.server_url, 'user-%s' % self._uuid),
- creds=self._creds, autocreate=False,
- defer_decryption=defer_decryption)
- signal(SOLEDAD_DONE_DATA_SYNC, self._uuid)
- return local_gen
- except Exception as e:
- logger.error("Soledad exception when syncing: %s" % str(e))
-
- def stop_sync(self):
- """
- Stop the current syncing process.
- """
- if self._db:
- self._db.stop_sync()
-
- def need_sync(self, url):
- """
- Return if local db replica differs from remote url's replica.
-
- :param url: The remote replica to compare with local replica.
- :type url: str
-
- :return: Whether remote replica and local replica differ.
- :rtype: bool
- """
- target = SoledadSyncTarget(
- url, self._db._get_replica_uid(), creds=self._creds,
- crypto=self._crypto)
- info = target.get_sync_info(self._db._get_replica_uid())
- # compare source generation with target's last known source generation
- if self._db._get_generation() != info[4]:
- signal(SOLEDAD_NEW_DATA_TO_SYNC, self._uuid)
- return True
- return False
-
- @property
- def syncing(self):
- """
- Property, True if the syncer is syncing.
- """
- return self._db.syncing
-
- def _set_token(self, token):
- """
- Set the authentication token for remote database access.
-
- Build the credentials dictionary with the following format:
-
- self._{
- 'token': {
- 'uuid': '<uuid>'
- 'token': '<token>'
- }
-
- :param token: The authentication token.
- :type token: str
- """
- self._creds = {
- 'token': {
- 'uuid': self._uuid,
- 'token': token,
- }
- }
-
- def _get_token(self):
- """
- Return current token from credentials dictionary.
- """
- return self._creds['token']['token']
-
- token = property(_get_token, _set_token, doc='The authentication Token.')
-
- #
- # Recovery document export and import methods
- #
-
- def export_recovery_document(self):
- """
- Export the storage secrets.
-
- A recovery document has the following structure:
-
- {
- 'storage_secrets': {
- '<storage_secret id>': {
- 'kdf': 'scrypt',
- 'kdf_salt': '<b64 repr of salt>'
- 'kdf_length': <key length>
- 'cipher': 'aes256',
- 'length': <secret length>,
- 'secret': '<encrypted storage_secret>',
- },
- },
- 'kdf': 'scrypt',
- 'kdf_salt': '<b64 repr of salt>',
- 'kdf_length: <key length>,
- '_mac_method': 'hmac',
- '_mac': '<mac>'
- }
-
- Note that multiple storage secrets might be stored in one recovery
- document. This method will also calculate a MAC of a string
- representation of the secrets dictionary.
-
- :return: The recovery document.
- :rtype: dict
- """
- # create salt and key for calculating MAC
- salt = os.urandom(self.SALT_LENGTH)
- key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
- data = {
- self.STORAGE_SECRETS_KEY: self._secrets,
- self.KDF_KEY: self.KDF_SCRYPT,
- self.KDF_SALT_KEY: binascii.b2a_base64(salt),
- self.KDF_LENGTH_KEY: len(key),
- MAC_METHOD_KEY: MacMethods.HMAC,
- MAC_KEY: hmac.new(
- key,
- json.dumps(self._secrets),
- sha256).hexdigest(),
- }
- return data
-
- def import_recovery_document(self, data):
- """
- Import storage secrets for symmetric encryption and uuid (if present)
- from a recovery document.
-
- Note that this method does not store the imported data on disk. For
- that, use C{self._store_secrets()}.
-
- :param data: The recovery document.
- :type data: dict
-
- :return: A tuple containing the number of imported secrets and whether
- there was MAC informationa available for authenticating.
- :rtype: (int, bool)
- """
- soledad_assert(self.STORAGE_SECRETS_KEY in data)
- # check mac of the recovery document
- mac = None
- if MAC_KEY in data:
- soledad_assert(data[MAC_KEY] is not None)
- soledad_assert(MAC_METHOD_KEY in data)
- soledad_assert(self.KDF_KEY in data)
- soledad_assert(self.KDF_SALT_KEY in data)
- soledad_assert(self.KDF_LENGTH_KEY in data)
- if data[MAC_METHOD_KEY] == MacMethods.HMAC:
- key = scrypt.hash(
- self._passphrase_as_string(),
- binascii.a2b_base64(data[self.KDF_SALT_KEY]),
- buflen=32)
- mac = hmac.new(
- key,
- json.dumps(data[self.STORAGE_SECRETS_KEY]),
- sha256).hexdigest()
- else:
- raise UnknownMacMethod('Unknown MAC method: %s.' %
- data[MAC_METHOD_KEY])
- if mac != data[MAC_KEY]:
- raise WrongMac('Could not authenticate recovery document\'s '
- 'contents.')
- # include secrets in the secret pool.
- secrets = 0
- for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items():
- if secret_id not in self._secrets:
- secrets += 1
- self._secrets[secret_id] = secret_data
- return secrets, mac
-
- #
- # Setters/getters
- #
-
- def _get_uuid(self):
- return self._uuid
-
- uuid = property(_get_uuid, doc='The user uuid.')
-
- def _get_secret_id(self):
- return self._secret_id
-
- secret_id = property(
- _get_secret_id,
- doc='The active secret id.')
-
- def _get_secrets_path(self):
- return self._secrets_path
-
- secrets_path = property(
- _get_secrets_path,
- doc='The path for the file containing the encrypted symmetric secret.')
-
- def _get_local_db_path(self):
- return self._local_db_path
-
- local_db_path = property(
- _get_local_db_path,
- doc='The path for the local database replica.')
-
- def _get_server_url(self):
- return self._server_url
-
- server_url = property(
- _get_server_url,
- doc='The URL of the Soledad server.')
-
- storage_secret = property(
- _get_storage_secret,
- doc='The secret used for symmetric encryption.')
-
- def _get_passphrase(self):
- return self._passphrase
-
- passphrase = property(
- _get_passphrase,
- doc='The passphrase for locking and unlocking encryption secrets for '
- 'local and remote storage.')
-
- def _passphrase_as_string(self):
- return self._passphrase.encode('utf-8')
-
-
-# ----------------------------------------------------------------------------
-# Monkey patching u1db to be able to provide a custom SSL cert
-# ----------------------------------------------------------------------------
-
-# We need a more reasonable timeout (in seconds)
-SOLEDAD_TIMEOUT = 120
-
-
-class VerifiedHTTPSConnection(httplib.HTTPSConnection):
- """
- HTTPSConnection verifying server side certificates.
- """
- # derived from httplib.py
-
- def connect(self):
- """
- Connect to a host on a given (SSL) port.
- """
- try:
- source = self.source_address
- sock = socket.create_connection((self.host, self.port),
- SOLEDAD_TIMEOUT, source)
- except AttributeError:
- # source_address was introduced in 2.7
- sock = socket.create_connection((self.host, self.port),
- SOLEDAD_TIMEOUT)
- if self._tunnel_host:
- self.sock = sock
- self._tunnel()
-
- highest_supported = ssl.PROTOCOL_SSLv23
-
- try:
- # needs python 2.7.9+
- # negotiate the best available version,
- # but explicitely disabled bad ones.
- ctx = ssl.SSLContext(highest_supported)
- ctx.options |= ssl.OP_NO_SSLv2
- ctx.options |= ssl.OP_NO_SSLv3
-
- ctx.load_verify_locations(cafile=SOLEDAD_CERT)
- ctx.verify_mode = ssl.CERT_REQUIRED
- self.sock = ctx.wrap_socket(sock)
-
- except AttributeError:
- self.sock = ssl.wrap_socket(
- sock, ca_certs=SOLEDAD_CERT, cert_reqs=ssl.CERT_REQUIRED,
- ssl_version=highest_supported)
-
- match_hostname(self.sock.getpeercert(), self.host)
-
-
-old__VerifiedHTTPSConnection = http_client._VerifiedHTTPSConnection
-http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection
-
-
-__all__ = ['soledad_assert', 'Soledad']
+from leap.soledad.client.api import Soledad
+from leap.soledad.common import soledad_assert
from ._version import get_versions
__version__ = get_versions()['version']
del get_versions
+
+__all__ = ['soledad_assert', 'Soledad', '__version__']
diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py
new file mode 100644
index 00000000..5b882bbe
--- /dev/null
+++ b/client/src/leap/soledad/client/adbapi.py
@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+# adbapi.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+An asyncrhonous interface to soledad using sqlcipher backend.
+It uses twisted.enterprise.adbapi.
+"""
+import re
+import os
+import sys
+import logging
+
+from functools import partial
+from threading import BoundedSemaphore
+
+from twisted.enterprise import adbapi
+from twisted.python import log
+from zope.proxy import ProxyBase, setProxiedObject
+from pysqlcipher.dbapi2 import OperationalError
+
+from leap.soledad.client import sqlcipher as soledad_sqlcipher
+from leap.soledad.client.pragmas import set_init_pragmas
+
+
+logger = logging.getLogger(name=__name__)
+
+
+DEBUG_SQL = os.environ.get("LEAP_DEBUG_SQL")
+if DEBUG_SQL:
+ log.startLogging(sys.stdout)
+
+"""
+How long the SQLCipher connection should wait for the lock to go away until
+raising an exception.
+"""
+SQLCIPHER_CONNECTION_TIMEOUT = 10
+
+"""
+How many times a SQLCipher query should be retried in case of timeout.
+"""
+SQLCIPHER_MAX_RETRIES = 10
+
+
+def getConnectionPool(opts, openfun=None, driver="pysqlcipher"):
+ """
+ Return a connection pool.
+
+ :param opts:
+ Options for the SQLCipher connection.
+ :type opts: SQLCipherOptions
+ :param openfun:
+ Callback invoked after every connect() on the underlying DB-API
+ object.
+ :type openfun: callable
+ :param driver:
+ The connection driver.
+ :type driver: str
+
+ :return: A U1DB connection pool.
+ :rtype: U1DBConnectionPool
+ """
+ if openfun is None and driver == "pysqlcipher":
+ openfun = partial(set_init_pragmas, opts=opts)
+ return U1DBConnectionPool(
+ "%s.dbapi2" % driver, database=opts.path,
+ check_same_thread=False, cp_openfun=openfun,
+ timeout=SQLCIPHER_CONNECTION_TIMEOUT)
+
+
+class U1DBConnection(adbapi.Connection):
+ """
+ A wrapper for a U1DB connection instance.
+ """
+
+ u1db_wrapper = soledad_sqlcipher.SoledadSQLCipherWrapper
+ """
+ The U1DB wrapper to use.
+ """
+
+ def __init__(self, pool, init_u1db=False):
+ """
+ :param pool: The pool of connections to that owns this connection.
+ :type pool: adbapi.ConnectionPool
+ :param init_u1db: Wether the u1db database should be initialized.
+ :type init_u1db: bool
+ """
+ self.init_u1db = init_u1db
+ adbapi.Connection.__init__(self, pool)
+
+ def reconnect(self):
+ """
+ Reconnect to the U1DB database.
+ """
+ if self._connection is not None:
+ self._pool.disconnect(self._connection)
+ self._connection = self._pool.connect()
+
+ if self.init_u1db:
+ self._u1db = self.u1db_wrapper(self._connection)
+
+ def __getattr__(self, name):
+ """
+ Route the requested attribute either to the U1DB wrapper or to the
+ connection.
+
+ :param name: The name of the attribute.
+ :type name: str
+ """
+ if name.startswith('u1db_'):
+ attr = re.sub('^u1db_', '', name)
+ return getattr(self._u1db, attr)
+ else:
+ return getattr(self._connection, name)
+
+
+class U1DBTransaction(adbapi.Transaction):
+ """
+ A wrapper for a U1DB 'cursor' object.
+ """
+
+ def __getattr__(self, name):
+ """
+ Route the requested attribute either to the U1DB wrapper of the
+ connection or to the actual connection cursor.
+
+ :param name: The name of the attribute.
+ :type name: str
+ """
+ if name.startswith('u1db_'):
+ attr = re.sub('^u1db_', '', name)
+ return getattr(self._connection._u1db, attr)
+ else:
+ return getattr(self._cursor, name)
+
+
+class U1DBConnectionPool(adbapi.ConnectionPool):
+ """
+ Represent a pool of connections to an U1DB database.
+ """
+
+ connectionFactory = U1DBConnection
+ transactionFactory = U1DBTransaction
+
+ def __init__(self, *args, **kwargs):
+ """
+ Initialize the connection pool.
+ """
+ adbapi.ConnectionPool.__init__(self, *args, **kwargs)
+ # all u1db connections, hashed by thread-id
+ self._u1dbconnections = {}
+
+ # The replica uid, primed by the connections on init.
+ self.replica_uid = ProxyBase(None)
+
+ conn = self.connectionFactory(self, init_u1db=True)
+ replica_uid = conn._u1db._real_replica_uid
+ setProxiedObject(self.replica_uid, replica_uid)
+
+ def runU1DBQuery(self, meth, *args, **kw):
+ """
+ Execute a U1DB query in a thread, using a pooled connection.
+
+ Concurrent threads trying to update the same database may timeout
+ because of other threads holding the database lock. Because of this,
+ we will retry SQLCIPHER_MAX_RETRIES times and fail after that.
+
+ :param meth: The U1DB wrapper method name.
+ :type meth: str
+
+ :return: a Deferred which will fire the return value of
+ 'self._runU1DBQuery(Transaction(...), *args, **kw)', or a Failure.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ meth = "u1db_%s" % meth
+ semaphore = BoundedSemaphore(SQLCIPHER_MAX_RETRIES - 1)
+
+ def _run_interaction():
+ return self.runInteraction(
+ self._runU1DBQuery, meth, *args, **kw)
+
+ def _errback(failure):
+ failure.trap(OperationalError)
+ if failure.getErrorMessage() == "database is locked":
+ should_retry = semaphore.acquire(False)
+ if should_retry:
+ logger.warning(
+ "Database operation timed out while waiting for "
+ "lock, trying again...")
+ return _run_interaction()
+ return failure
+
+ d = _run_interaction()
+ d.addErrback(_errback)
+ return d
+
+ def _runU1DBQuery(self, trans, meth, *args, **kw):
+ """
+ Execute a U1DB query.
+
+ :param trans: An U1DB transaction.
+ :type trans: adbapi.Transaction
+ :param meth: the U1DB wrapper method name.
+ :type meth: str
+ """
+ meth = getattr(trans, meth)
+ return meth(*args, **kw)
+
+ def _runInteraction(self, interaction, *args, **kw):
+ """
+ Interact with the database and return the result.
+
+ :param interaction:
+ A callable object whose first argument is an
+ L{adbapi.Transaction}.
+ :type interaction: callable
+ :return: a Deferred which will fire the return value of
+ 'interaction(Transaction(...), *args, **kw)', or a Failure.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ tid = self.threadID()
+ u1db = self._u1dbconnections.get(tid)
+ conn = self.connectionFactory(self, init_u1db=not bool(u1db))
+
+ if self.replica_uid is None:
+ replica_uid = conn._u1db._real_replica_uid
+ setProxiedObject(self.replica_uid, replica_uid)
+
+ if u1db is None:
+ self._u1dbconnections[tid] = conn._u1db
+ else:
+ conn._u1db = u1db
+
+ trans = self.transactionFactory(self, conn)
+ try:
+ result = interaction(trans, *args, **kw)
+ trans.close()
+ conn.commit()
+ return result
+ except:
+ excType, excValue, excTraceback = sys.exc_info()
+ try:
+ conn.rollback()
+ except:
+ log.err(None, "Rollback failed")
+ raise excType, excValue, excTraceback
+
+ def finalClose(self):
+ """
+ A final close, only called by the shutdown trigger.
+ """
+ self.shutdownID = None
+ self.threadpool.stop()
+ self.running = False
+ for conn in self.connections.values():
+ self._close(conn)
+ for u1db in self._u1dbconnections.values():
+ self._close(u1db)
+ self.connections.clear()
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
new file mode 100644
index 00000000..76d6acc3
--- /dev/null
+++ b/client/src/leap/soledad/client/api.py
@@ -0,0 +1,882 @@
+# -*- coding: utf-8 -*-
+# api.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Soledad - Synchronization Of Locally Encrypted Data Among Devices.
+
+This module holds the public api for Soledad.
+
+Soledad is the part of LEAP that manages storage and synchronization of
+application data. It is built on top of U1DB reference Python API and
+implements (1) a SQLCipher backend for local storage in the client, (2) a
+SyncTarget that encrypts data before syncing, and (3) a CouchDB backend for
+remote storage in the server side.
+"""
+import binascii
+import errno
+import httplib
+import logging
+import os
+import socket
+import ssl
+import urlparse
+
+try:
+ import cchardet as chardet
+except ImportError:
+ import chardet
+
+from StringIO import StringIO
+from u1db.remote import http_client
+from u1db.remote.ssl_match_hostname import match_hostname
+from zope.interface import implements
+
+from leap.common.config import get_path_prefix
+
+from leap.soledad.common import SHARED_DB_NAME
+from leap.soledad.common import soledad_assert
+from leap.soledad.common import soledad_assert_type
+
+from leap.soledad.client import adbapi
+from leap.soledad.client import events as soledad_events
+from leap.soledad.client import interfaces as soledad_interfaces
+from leap.soledad.client.crypto import SoledadCrypto
+from leap.soledad.client.secrets import SoledadSecrets
+from leap.soledad.client.shared_db import SoledadSharedDatabase
+from leap.soledad.client.sqlcipher import SQLCipherOptions, SQLCipherU1DBSync
+
+logger = logging.getLogger(name=__name__)
+
+#
+# Constants
+#
+
+"""
+Path to the certificate file used to certify the SSL connection between
+Soledad client and server.
+"""
+SOLEDAD_CERT = None
+
+
+class Soledad(object):
+ """
+ Soledad provides encrypted data storage and sync.
+
+ A Soledad instance is used to store and retrieve data in a local encrypted
+ database and synchronize this database with Soledad server.
+
+ This class is also responsible for bootstrapping users' account by
+ creating cryptographic secrets and/or storing/fetching them on Soledad
+ server.
+
+ Soledad uses ``leap.common.events`` to signal events. The possible events
+ to be signaled are:
+
+ SOLEDAD_CREATING_KEYS: emitted during bootstrap sequence when key
+ generation starts.
+ SOLEDAD_DONE_CREATING_KEYS: emitted during bootstrap sequence when key
+ generation finishes.
+ SOLEDAD_UPLOADING_KEYS: emitted during bootstrap sequence when soledad
+ starts sending keys to server.
+ SOLEDAD_DONE_UPLOADING_KEYS: emitted during bootstrap sequence when
+ soledad finishes sending keys to server.
+ SOLEDAD_DOWNLOADING_KEYS: emitted during bootstrap sequence when
+ soledad starts to retrieve keys from server.
+ SOLEDAD_DONE_DOWNLOADING_KEYS: emitted during bootstrap sequence when
+ soledad finishes downloading keys from server.
+ SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when
+ there's indeed new data to be synchronized between local database
+ replica and server's replica.
+ SOLEDAD_DONE_DATA_SYNC: emitted inside C{sync()} method when it has
+ finished synchronizing with remote replica.
+ """
+ implements(soledad_interfaces.ILocalStorage,
+ soledad_interfaces.ISyncableStorage,
+ soledad_interfaces.ISecretsStorage)
+
+ local_db_file_name = 'soledad.u1db'
+ secrets_file_name = "soledad.json"
+ default_prefix = os.path.join(get_path_prefix(), 'leap', 'soledad')
+
+ def __init__(self, uuid, passphrase, secrets_path, local_db_path,
+ server_url, cert_file, shared_db=None,
+ auth_token=None, defer_encryption=False, syncable=True):
+ """
+ Initialize configuration, cryptographic keys and dbs.
+
+ :param uuid: User's uuid.
+ :type uuid: str
+
+ :param passphrase:
+ The passphrase for locking and unlocking encryption secrets for
+ local and remote storage.
+ :type passphrase: unicode
+
+ :param secrets_path:
+ Path for storing encrypted key used for symmetric encryption.
+ :type secrets_path: str
+
+ :param local_db_path: Path for local encrypted storage db.
+ :type local_db_path: str
+
+ :param server_url:
+ URL for Soledad server. This is used either to sync with the user's
+ remote db and to interact with the shared recovery database.
+ :type server_url: str
+
+ :param cert_file:
+ Path to the certificate of the ca used to validate the SSL
+ certificate used by the remote soledad server.
+ :type cert_file: str
+
+ :param shared_db:
+ The shared database.
+ :type shared_db: HTTPDatabase
+
+ :param auth_token:
+ Authorization token for accessing remote databases.
+ :type auth_token: str
+
+ :param defer_encryption:
+ Whether to defer encryption/decryption of documents, or do it
+ inline while syncing.
+ :type defer_encryption: bool
+
+ :param syncable:
+ If set to ``False``, this database will not attempt to synchronize
+ with remote replicas (default is ``True``)
+ :type syncable: bool
+
+ :raise BootstrapSequenceError:
+ Raised when the secret initialization sequence (i.e. retrieval
+ from server or generation and storage on server) has failed for
+ some reason.
+ """
+ # store config params
+ self._uuid = uuid
+ self._passphrase = passphrase
+ self._local_db_path = local_db_path
+ self._server_url = server_url
+ self._defer_encryption = defer_encryption
+ self._secrets_path = None
+
+ self.shared_db = shared_db
+
+ # configure SSL certificate
+ global SOLEDAD_CERT
+ SOLEDAD_CERT = cert_file
+
+ # init crypto variables
+ self._set_token(auth_token)
+ self._crypto = SoledadCrypto(self)
+
+ self._init_config_with_defaults()
+ self._init_working_dirs()
+
+ self._secrets_path = secrets_path
+
+ # Initialize shared recovery database
+ self.init_shared_db(server_url, uuid, self._creds, syncable=syncable)
+
+ # The following can raise BootstrapSequenceError, that will be
+ # propagated upwards.
+ self._init_secrets()
+ self._init_u1db_sqlcipher_backend()
+
+ if syncable:
+ self._init_u1db_syncer()
+
+ #
+ # initialization/destruction methods
+ #
+ def _init_config_with_defaults(self):
+ """
+ Initialize configuration using default values for missing params.
+ """
+ soledad_assert_type(self._passphrase, unicode)
+ initialize = lambda attr, val: getattr(
+ self, attr, None) is None and setattr(self, attr, val)
+
+ initialize("_secrets_path", os.path.join(
+ self.default_prefix, self.secrets_file_name))
+ initialize("_local_db_path", os.path.join(
+ self.default_prefix, self.local_db_file_name))
+ # initialize server_url
+ soledad_assert(self._server_url is not None,
+ 'Missing URL for Soledad server.')
+
+ def _init_working_dirs(self):
+ """
+ Create work directories.
+
+ :raise OSError: in case file exists and is not a dir.
+ """
+ paths = map(lambda x: os.path.dirname(x), [
+ self._local_db_path, self._secrets_path])
+ for path in paths:
+ create_path_if_not_exists(path)
+
+ def _init_secrets(self):
+ """
+ Initialize Soledad secrets.
+ """
+ self._secrets = SoledadSecrets(
+ self.uuid, self._passphrase, self._secrets_path,
+ self.shared_db, self._crypto)
+ self._secrets.bootstrap()
+
+ def _init_u1db_sqlcipher_backend(self):
+ """
+ Initialize the U1DB SQLCipher database for local storage.
+
+ Instantiates a modified twisted adbapi that will maintain a threadpool
+ with a u1db-sqclipher connection for each thread, and will return
+ deferreds for each u1db query.
+
+ Currently, Soledad uses the default SQLCipher cipher, i.e.
+ 'aes-256-cbc'. We use scrypt to derive a 256-bit encryption key,
+ and internally the SQLCipherDatabase initialization uses the 'raw
+ PRAGMA key' format to handle the key to SQLCipher.
+ """
+ tohex = binascii.b2a_hex
+ # sqlcipher only accepts the hex version
+ key = tohex(self._secrets.get_local_storage_key())
+ sync_db_key = tohex(self._secrets.get_sync_db_key())
+
+ opts = SQLCipherOptions(
+ self._local_db_path, key,
+ is_raw_key=True, create=True,
+ defer_encryption=self._defer_encryption,
+ sync_db_key=sync_db_key,
+ )
+ self._sqlcipher_opts = opts
+ self._dbpool = adbapi.getConnectionPool(opts)
+
+ def _init_u1db_syncer(self):
+ """
+ Initialize the U1DB synchronizer.
+ """
+ replica_uid = self._dbpool.replica_uid
+ self._dbsyncer = SQLCipherU1DBSync(
+ self._sqlcipher_opts, self._crypto, replica_uid,
+ SOLEDAD_CERT,
+ defer_encryption=self._defer_encryption)
+
+ #
+ # Closing methods
+ #
+
+ def close(self):
+ """
+ Close underlying U1DB database.
+ """
+ logger.debug("Closing soledad")
+ self._dbpool.close()
+ if getattr(self, '_dbsyncer', None):
+ self._dbsyncer.close()
+
+ #
+ # ILocalStorage
+ #
+
+ def _defer(self, meth, *args, **kw):
+ """
+ Defer a method to be run on a U1DB connection pool.
+
+ :param meth: A method to defer to the U1DB connection pool.
+ :type meth: callable
+ :return: A deferred.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._dbpool.runU1DBQuery(meth, *args, **kw)
+
+ def put_doc(self, doc):
+ """
+ Update a document.
+
+ If the document currently has conflicts, put will fail.
+ If the database specifies a maximum document size and the document
+ exceeds it, put will fail and raise a DocumentTooBig exception.
+
+ ============================== WARNING ==============================
+ This method converts the document's contents to unicode in-place. This
+ means that after calling `put_doc(doc)`, the contents of the
+ document, i.e. `doc.content`, might be different from before the
+ call.
+ ============================== WARNING ==============================
+
+ :param doc: A document with new content.
+ :type doc: leap.soledad.common.document.SoledadDocument
+ :return: A deferred whose callback will be invoked with the new
+ revision identifier for the document. The document object will
+ also be updated.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ doc.content = _convert_to_unicode(doc.content)
+ return self._defer("put_doc", doc)
+
+ def delete_doc(self, doc):
+ """
+ Mark a document as deleted.
+
+ Will abort if the current revision doesn't match doc.rev.
+ This will also set doc.content to None.
+
+ :param doc: A document to be deleted.
+ :type doc: leap.soledad.common.document.SoledadDocument
+ :return: A deferred.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer("delete_doc", doc)
+
+ def get_doc(self, doc_id, include_deleted=False):
+ """
+ Get the JSON string for the given document.
+
+ :param doc_id: The unique document identifier
+ :type doc_id: str
+ :param include_deleted: If set to True, deleted documents will be
+ returned with empty content. Otherwise asking for a deleted
+ document will return None.
+ :type include_deleted: bool
+ :return: A deferred whose callback will be invoked with a document
+ object.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer(
+ "get_doc", doc_id, include_deleted=include_deleted)
+
+ def get_docs(
+ self, doc_ids, check_for_conflicts=True, include_deleted=False):
+ """
+ Get the JSON content for many documents.
+
+ :param doc_ids: A list of document identifiers.
+ :type doc_ids: list
+ :param check_for_conflicts: If set to False, then the conflict check
+ will be skipped, and 'None' will be returned instead of True/False.
+ :type check_for_conflicts: bool
+ :param include_deleted: If set to True, deleted documents will be
+ returned with empty content. Otherwise deleted documents will not
+ be included in the results.
+ :type include_deleted: bool
+ :return: A deferred whose callback will be invoked with an iterable
+ giving the document object for each document id in matching
+ doc_ids order.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer(
+ "get_docs", doc_ids, check_for_conflicts=check_for_conflicts,
+ include_deleted=include_deleted)
+
+ def get_all_docs(self, include_deleted=False):
+ """
+ Get the JSON content for all documents in the database.
+
+ :param include_deleted: If set to True, deleted documents will be
+ returned with empty content. Otherwise deleted documents will not
+ be included in the results.
+ :type include_deleted: bool
+
+ :return: A deferred which, when fired, will pass the a tuple
+ containing (generation, [Document]) to the callback, with the
+ current generation of the database, followed by a list of all the
+ documents in the database.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer("get_all_docs", include_deleted)
+
+ def create_doc(self, content, doc_id=None):
+ """
+ Create a new document.
+
+ You can optionally specify the document identifier, but the document
+ must not already exist. See 'put_doc' if you want to override an
+ existing document.
+ If the database specifies a maximum document size and the document
+ exceeds it, create will fail and raise a DocumentTooBig exception.
+
+ :param content: A Python dictionary.
+ :type content: dict
+ :param doc_id: An optional identifier specifying the document id.
+ :type doc_id: str
+ :return: A deferred whose callback will be invoked with a document.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ # TODO we probably should pass an optional "encoding" parameter to
+ # create_doc (and probably to put_doc too). There are cases (mail
+ # payloads for example) in which we already have the encoding in the
+ # headers, so we don't need to guess it.
+ return self._defer(
+ "create_doc", _convert_to_unicode(content), doc_id=doc_id)
+
+ def create_doc_from_json(self, json, doc_id=None):
+ """
+ Create a new document.
+
+ You can optionally specify the document identifier, but the document
+ must not already exist. See 'put_doc' if you want to override an
+ existing document.
+ If the database specifies a maximum document size and the document
+ exceeds it, create will fail and raise a DocumentTooBig exception.
+
+ :param json: The JSON document string
+ :type json: dict
+ :param doc_id: An optional identifier specifying the document id.
+ :type doc_id: str
+ :return: A deferred whose callback will be invoked with a document.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer("create_doc_from_json", json, doc_id=doc_id)
+
+ def create_index(self, index_name, *index_expressions):
+ """
+ Create a named index, which can then be queried for future lookups.
+
+ Creating an index which already exists is not an error, and is cheap.
+ Creating an index which does not match the index_expressions of the
+ existing index is an error.
+ Creating an index will block until the expressions have been evaluated
+ and the index generated.
+
+ :param index_name: A unique name which can be used as a key prefix
+ :type index_name: str
+ :param index_expressions: index expressions defining the index
+ information.
+
+ Examples:
+
+ "fieldname", or "fieldname.subfieldname" to index alphabetically
+ sorted on the contents of a field.
+
+ "number(fieldname, width)", "lower(fieldname)"
+ :type index_expresions: list of str
+ :return: A deferred.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer("create_index", index_name, *index_expressions)
+
+ def delete_index(self, index_name):
+ """
+ Remove a named index.
+
+ :param index_name: The name of the index we are removing
+ :type index_name: str
+ :return: A deferred.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer("delete_index", index_name)
+
+ def list_indexes(self):
+ """
+ List the definitions of all known indexes.
+
+ :return: A deferred whose callback will be invoked with a list of
+ [('index-name', ['field', 'field2'])] definitions.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer("list_indexes")
+
+ def get_from_index(self, index_name, *key_values):
+ """
+ Return documents that match the keys supplied.
+
+ You must supply exactly the same number of values as have been defined
+ in the index. It is possible to do a prefix match by using '*' to
+ indicate a wildcard match. You can only supply '*' to trailing entries,
+ (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.)
+ It is also possible to append a '*' to the last supplied value (eg
+ 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*')
+
+ :param index_name: The index to query
+ :type index_name: str
+ :param key_values: values to match. eg, if you have
+ an index with 3 fields then you would have:
+ get_from_index(index_name, val1, val2, val3)
+ :type key_values: list
+ :return: A deferred whose callback will be invoked with a list of
+ [Document].
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer("get_from_index", index_name, *key_values)
+
+ def get_count_from_index(self, index_name, *key_values):
+ """
+ Return the count for a given combination of index_name
+ and key values.
+
+ Extension method made from similar methods in u1db version 13.09
+
+ :param index_name: The index to query
+ :type index_name: str
+ :param key_values: values to match. eg, if you have
+ an index with 3 fields then you would have:
+ get_from_index(index_name, val1, val2, val3)
+ :type key_values: tuple
+ :return: A deferred whose callback will be invoked with the count.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer("get_count_from_index", index_name, *key_values)
+
+ def get_range_from_index(self, index_name, start_value, end_value):
+ """
+ Return documents that fall within the specified range.
+
+ Both ends of the range are inclusive. For both start_value and
+ end_value, one must supply exactly the same number of values as have
+ been defined in the index, or pass None. In case of a single column
+ index, a string is accepted as an alternative for a tuple with a single
+ value. It is possible to do a prefix match by using '*' to indicate
+ a wildcard match. You can only supply '*' to trailing entries, (eg
+ 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also
+ possible to append a '*' to the last supplied value (eg 'val*', '*',
+ '*' or 'val', 'val*', '*', but not 'val*', 'val', '*')
+
+ :param index_name: The index to query
+ :type index_name: str
+ :param start_values: tuples of values that define the lower bound of
+ the range. eg, if you have an index with 3 fields then you would
+ have: (val1, val2, val3)
+ :type start_values: tuple
+ :param end_values: tuples of values that define the upper bound of the
+ range. eg, if you have an index with 3 fields then you would have:
+ (val1, val2, val3)
+ :type end_values: tuple
+ :return: A deferred whose callback will be invoked with a list of
+ [Document].
+ :rtype: twisted.internet.defer.Deferred
+ """
+
+ return self._defer(
+ "get_range_from_index", index_name, start_value, end_value)
+
+ def get_index_keys(self, index_name):
+ """
+ Return all keys under which documents are indexed in this index.
+
+ :param index_name: The index to query
+ :type index_name: str
+ :return: A deferred whose callback will be invoked with a list of
+ tuples of indexed keys.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer("get_index_keys", index_name)
+
+ def get_doc_conflicts(self, doc_id):
+ """
+ Get the list of conflicts for the given document.
+
+ The order of the conflicts is such that the first entry is the value
+ that would be returned by "get_doc".
+
+ :param doc_id: The unique document identifier
+ :type doc_id: str
+ :return: A deferred whose callback will be invoked with a list of the
+ Document entries that are conflicted.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer("get_doc_conflicts", doc_id)
+
+ def resolve_doc(self, doc, conflicted_doc_revs):
+ """
+ Mark a document as no longer conflicted.
+
+ We take the list of revisions that the client knows about that it is
+ superseding. This may be a different list from the actual current
+ conflicts, in which case only those are removed as conflicted. This
+ may fail if the conflict list is significantly different from the
+ supplied information. (sync could have happened in the background from
+ the time you GET_DOC_CONFLICTS until the point where you RESOLVE)
+
+ :param doc: A Document with the new content to be inserted.
+ :type doc: SoledadDocument
+ :param conflicted_doc_revs: A list of revisions that the new content
+ supersedes.
+ :type conflicted_doc_revs: list(str)
+ :return: A deferred.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._defer("resolve_doc", doc, conflicted_doc_revs)
+
+ @property
+ def local_db_path(self):
+ return self._local_db_path
+
+ @property
+ def uuid(self):
+ return self._uuid
+
+ #
+ # ISyncableStorage
+ #
+
+ def sync(self, defer_decryption=True):
+ """
+ Synchronize documents with the server replica.
+
+ :param defer_decryption:
+ Whether to defer decryption of documents, or do it inline while
+ syncing.
+ :type defer_decryption: bool
+
+ :return: A deferred whose callback will be invoked with the local
+ generation before the synchronization was performed.
+ :rtype: twisted.internet.defer.Deferred
+ """
+
+ # -----------------------------------------------------------------
+ # TODO this needs work.
+ # Should review/write tests to check that this:
+
+ # (1) Defer to the syncer pool -- DONE (on dbsyncer)
+ # (2) Return the deferred
+ # (3) Add the callback for signaling the event (executed on reactor
+ # thread)
+ # (4) Check that the deferred is called with the local gen.
+
+ # -----------------------------------------------------------------
+
+ sync_url = urlparse.urljoin(self._server_url, 'user-%s' % self.uuid)
+ d = self._dbsyncer.sync(
+ sync_url,
+ creds=self._creds,
+ defer_decryption=defer_decryption)
+
+ def _sync_callback(local_gen):
+ soledad_events.emit(
+ soledad_events.SOLEDAD_DONE_DATA_SYNC, self.uuid)
+ return local_gen
+
+ def _sync_errback(failure):
+ s = StringIO()
+ failure.printDetailedTraceback(file=s)
+ msg = "Soledad exception when syncing!\n" + s.getvalue()
+ logger.error(msg)
+ return failure
+
+ d.addCallbacks(_sync_callback, _sync_errback)
+ return d
+
+ @property
+ def syncing(self):
+ """
+ Return wether Soledad is currently synchronizing with the server.
+
+ :return: Wether Soledad is currently synchronizing with the server.
+ :rtype: bool
+ """
+ return self._dbsyncer.syncing
+
+ def _set_token(self, token):
+ """
+ Set the authentication token for remote database access.
+
+ Internally, this builds the credentials dictionary with the following
+ format:
+
+ {
+ 'token': {
+ 'uuid': '<uuid>'
+ 'token': '<token>'
+ }
+ }
+
+ :param token: The authentication token.
+ :type token: str
+ """
+ self._creds = {
+ 'token': {
+ 'uuid': self.uuid,
+ 'token': token,
+ }
+ }
+
+ def _get_token(self):
+ """
+ Return current token from credentials dictionary.
+ """
+ return self._creds['token']['token']
+
+ token = property(_get_token, _set_token, doc='The authentication Token.')
+
+ #
+ # ISecretsStorage
+ #
+
+ def init_shared_db(self, server_url, uuid, creds, syncable=True):
+ """
+ Initialize the shared database.
+
+ :param server_url: URL of the remote database.
+ :type server_url: str
+ :param uuid: The user's unique id.
+ :type uuid: str
+ :param creds: A tuple containing the authentication method and
+ credentials.
+ :type creds: tuple
+ :param syncable:
+ If syncable is False, the database will not attempt to sync against
+ a remote replica.
+ :type syncable: bool
+ """
+ # only case this is False is for testing purposes
+ if self.shared_db is None:
+ shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME)
+ self.shared_db = SoledadSharedDatabase.open_database(
+ shared_db_url,
+ uuid,
+ creds=creds,
+ syncable=syncable)
+
+ @property
+ def storage_secret(self):
+ """
+ Return the secret used for local storage encryption.
+
+ :return: The secret used for local storage encryption.
+ :rtype: str
+ """
+ return self._secrets.storage_secret
+
+ @property
+ def remote_storage_secret(self):
+ """
+ Return the secret used for encryption of remotely stored data.
+
+ :return: The secret used for remote storage encryption.
+ :rtype: str
+ """
+ return self._secrets.remote_storage_secret
+
+ @property
+ def secrets(self):
+ """
+ Return the secrets object.
+
+ :return: The secrets object.
+ :rtype: SoledadSecrets
+ """
+ return self._secrets
+
+ def change_passphrase(self, new_passphrase):
+ """
+ Change the passphrase that encrypts the storage secret.
+
+ :param new_passphrase: The new passphrase.
+ :type new_passphrase: unicode
+
+ :raise NoStorageSecret: Raised if there's no storage secret available.
+ """
+ self._secrets.change_passphrase(new_passphrase)
+
+ #
+ # Raw SQLCIPHER Queries
+ #
+
+ def raw_sqlcipher_query(self, *args, **kw):
+ """
+ Run a raw sqlcipher query in the local database.
+ """
+ return self._dbpool.runQuery(*args, **kw)
+
+
+def _convert_to_unicode(content):
+ """
+ Convert content to unicode (or all the strings in content).
+
+ NOTE: Even though this method supports any type, it will
+ currently ignore contents of lists, tuple or any other
+ iterable than dict. We don't need support for these at the
+ moment
+
+ :param content: content to convert
+ :type content: object
+
+ :rtype: object
+ """
+ # Chardet doesn't guess very well with some smallish payloads.
+ # This parameter might need some empirical tweaking.
+ CUTOFF_CONFIDENCE = 0.90
+
+ if isinstance(content, unicode):
+ return content
+ elif isinstance(content, str):
+ encoding = "utf-8"
+ result = chardet.detect(content)
+ if result["confidence"] > CUTOFF_CONFIDENCE:
+ encoding = result["encoding"]
+ try:
+ content = content.decode(encoding)
+ except UnicodeError as e:
+ logger.error("Unicode error: {0!r}. Using 'replace'".format(e))
+ content = content.decode(encoding, 'replace')
+ return content
+ else:
+ if isinstance(content, dict):
+ for key in content.keys():
+ content[key] = _convert_to_unicode(content[key])
+ return content
+
+
+def create_path_if_not_exists(path):
+ try:
+ if not os.path.isdir(path):
+ logger.info('Creating directory: %s.' % path)
+ os.makedirs(path)
+ except OSError as exc:
+ if exc.errno == errno.EEXIST and os.path.isdir(path):
+ pass
+ else:
+ raise
+
+# ----------------------------------------------------------------------------
+# Monkey patching u1db to be able to provide a custom SSL cert
+# ----------------------------------------------------------------------------
+
+# We need a more reasonable timeout (in seconds)
+SOLEDAD_TIMEOUT = 120
+
+
+class VerifiedHTTPSConnection(httplib.HTTPSConnection):
+ """
+ HTTPSConnection verifying server side certificates.
+ """
+ # derived from httplib.py
+
+ def connect(self):
+ """
+ Connect to a host on a given (SSL) port.
+ """
+ try:
+ source = self.source_address
+ sock = socket.create_connection((self.host, self.port),
+ SOLEDAD_TIMEOUT, source)
+ except AttributeError:
+ # source_address was introduced in 2.7
+ sock = socket.create_connection((self.host, self.port),
+ SOLEDAD_TIMEOUT)
+ if self._tunnel_host:
+ self.sock = sock
+ self._tunnel()
+
+ self.sock = ssl.wrap_socket(sock,
+ ca_certs=SOLEDAD_CERT,
+ cert_reqs=ssl.CERT_REQUIRED)
+ match_hostname(self.sock.getpeercert(), self.host)
+
+
+old__VerifiedHTTPSConnection = http_client._VerifiedHTTPSConnection
+http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection
diff --git a/client/src/leap/soledad/client/auth.py b/client/src/leap/soledad/client/auth.py
index 72ab0008..6dfabeb4 100644
--- a/client/src/leap/soledad/client/auth.py
+++ b/client/src/leap/soledad/client/auth.py
@@ -14,15 +14,13 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
"""
Methods for token-based authentication.
These methods have to be included in all classes that extend HTTPClient so
they can do token-based auth requests to the Soledad server.
"""
-
+import base64
from u1db import errors
@@ -49,7 +47,7 @@ class TokenBasedAuth(object):
Return an authorization header to be included in the HTTP request, in
the form:
- [('Authorization', 'Token <base64 encoded creds')]
+ [('Authorization', 'Token <(base64 encoded) uuid:token>')]
:param method: The HTTP method.
:type method: str
@@ -64,7 +62,8 @@ class TokenBasedAuth(object):
if 'token' in self._creds:
uuid, token = self._creds['token']
auth = '%s:%s' % (uuid, token)
- return [('Authorization', 'Token %s' % auth.encode('base64')[:-1])]
+ b64_token = base64.b64encode(auth)
+ return [('Authorization', 'Token %s' % b64_token)]
else:
raise errors.UnknownAuthMethod(
'Wrong credentials: %s' % self._creds)
diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 7133f804..bdbaa8e0 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -23,31 +23,14 @@ import hmac
import hashlib
import json
import logging
-import multiprocessing
-import threading
from pycryptopp.cipher.aes import AES
from pycryptopp.cipher.xsalsa20 import XSalsa20
-from zope.proxy import sameProxiedObjects
from leap.soledad.common import soledad_assert
from leap.soledad.common import soledad_assert_type
-from leap.soledad.common.document import SoledadDocument
-
-
-from leap.soledad.common.crypto import (
- EncryptionSchemes,
- UnknownEncryptionScheme,
- MacMethods,
- UnknownMacMethod,
- WrongMac,
- ENC_JSON_KEY,
- ENC_SCHEME_KEY,
- ENC_METHOD_KEY,
- ENC_IV_KEY,
- MAC_KEY,
- MAC_METHOD_KEY,
-)
+from leap.soledad.common import crypto
+
logger = logging.getLogger(__name__)
@@ -55,37 +38,23 @@ logger = logging.getLogger(__name__)
MAC_KEY_LENGTH = 64
-class EncryptionMethods(object):
- """
- Representation of encryption methods that can be used.
+def _assert_known_encryption_method(method):
"""
+ Assert that we can encrypt/decrypt the given C{method}
- AES_256_CTR = 'aes-256-ctr'
- XSALSA20 = 'xsalsa20'
-
-#
-# Exceptions
-#
-
-
-class DocumentNotEncrypted(Exception):
- """
- Raised for failures in document encryption.
- """
- pass
-
-
-class UnknownEncryptionMethod(Exception):
- """
- Raised when trying to encrypt/decrypt with unknown method.
- """
- pass
-
+ :param method: The encryption method to assert.
+ :type method: str
-class NoSymmetricSecret(Exception):
- """
- Raised when trying to get a hashed passphrase.
+ :raise UnknownEncryptionMethodError: Raised when C{method} is unknown.
"""
+ valid_methods = [
+ crypto.EncryptionMethods.AES_256_CTR,
+ crypto.EncryptionMethods.XSALSA20,
+ ]
+ try:
+ soledad_assert(method in valid_methods)
+ except AssertionError:
+ raise crypto.UnknownEncryptionMethodError
def encrypt_sym(data, key, method):
@@ -104,25 +73,26 @@ def encrypt_sym(data, key, method):
:return: A tuple with the initial value and the encrypted data.
:rtype: (long, str)
+
+ :raise AssertionError: Raised if C{method} is unknown.
"""
soledad_assert_type(key, str)
-
soledad_assert(
len(key) == 32, # 32 x 8 = 256 bits.
'Wrong key size: %s bits (must be 256 bits long).' %
(len(key) * 8))
+ _assert_known_encryption_method(method)
+
iv = None
# AES-256 in CTR mode
- if method == EncryptionMethods.AES_256_CTR:
+ if method == crypto.EncryptionMethods.AES_256_CTR:
iv = os.urandom(16)
ciphertext = AES(key=key, iv=iv).process(data)
# XSalsa20
- elif method == EncryptionMethods.XSALSA20:
+ elif method == crypto.EncryptionMethods.XSALSA20:
iv = os.urandom(24)
ciphertext = XSalsa20(key=key, iv=iv).process(data)
- else:
- # raise if method is unknown
- raise UnknownEncryptionMethod('Unkwnown method: %s' % method)
+
return binascii.b2a_base64(iv), ciphertext
@@ -143,6 +113,8 @@ def decrypt_sym(data, key, method, **kwargs):
:return: The decrypted data.
:rtype: str
+
+ :raise UnknownEncryptionMethodError: Raised when C{method} is unknown.
"""
soledad_assert_type(key, str)
# assert params
@@ -152,17 +124,15 @@ def decrypt_sym(data, key, method, **kwargs):
soledad_assert(
'iv' in kwargs,
'%s needs an initial value.' % method)
+ _assert_known_encryption_method(method)
# AES-256 in CTR mode
- if method == EncryptionMethods.AES_256_CTR:
+ if method == crypto.EncryptionMethods.AES_256_CTR:
return AES(
key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data)
- elif method == EncryptionMethods.XSALSA20:
+ elif method == crypto.EncryptionMethods.XSALSA20:
return XSalsa20(
key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data)
- # raise if method is unknown
- raise UnknownEncryptionMethod('Unkwnown method: %s' % method)
-
def doc_mac_key(doc_id, secret):
"""
@@ -176,17 +146,13 @@ def doc_mac_key(doc_id, secret):
:param doc_id: The id of the document.
:type doc_id: str
- :param secret: soledad secret storage
- :type secret: Soledad.storage_secret
+ :param secret: The Soledad storage secret
+ :type secret: str
:return: The key.
:rtype: str
-
- :raise NoSymmetricSecret: if no symmetric secret was supplied.
"""
- if secret is None:
- raise NoSymmetricSecret()
-
+ soledad_assert(secret is not None)
return hmac.new(
secret[:MAC_KEY_LENGTH],
doc_id,
@@ -208,11 +174,11 @@ class SoledadCrypto(object):
self._soledad = soledad
def encrypt_sym(self, data, key,
- method=EncryptionMethods.AES_256_CTR):
+ method=crypto.EncryptionMethods.AES_256_CTR):
return encrypt_sym(data, key, method)
def decrypt_sym(self, data, key,
- method=EncryptionMethods.AES_256_CTR, **kwargs):
+ method=crypto.EncryptionMethods.AES_256_CTR, **kwargs):
return decrypt_sym(data, key, method, **kwargs)
def doc_mac_key(self, doc_id, secret):
@@ -224,7 +190,7 @@ class SoledadCrypto(object):
The password is derived using HMAC having sha256 as underlying hash
function. The key used for HMAC are the first
- C{soledad.REMOTE_STORAGE_SECRET_KENGTH} bytes of Soledad's storage
+ C{soledad.REMOTE_STORAGE_SECRET_LENGTH} bytes of Soledad's storage
secret stripped from the first MAC_KEY_LENGTH characters. The HMAC
message is C{doc_id}.
@@ -234,15 +200,10 @@ class SoledadCrypto(object):
:return: The passphrase.
:rtype: str
-
- :raise NoSymmetricSecret: if no symmetric secret was supplied.
"""
- if self.secret is None:
- raise NoSymmetricSecret()
+ soledad_assert(self.secret is not None)
return hmac.new(
- self.secret[
- MAC_KEY_LENGTH:
- self._soledad.REMOTE_STORAGE_SECRET_LENGTH],
+ self.secret[MAC_KEY_LENGTH:],
doc_id,
hashlib.sha256).digest()
@@ -251,17 +212,18 @@ class SoledadCrypto(object):
#
def _get_secret(self):
- return self._soledad.storage_secret
+ return self._soledad.secrets.remote_storage_secret
secret = property(
_get_secret, doc='The secret used for symmetric encryption')
+
#
# Crypto utilities for a SoledadDocument.
#
-
-def mac_doc(doc_id, doc_rev, ciphertext, mac_method, secret):
+def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
+ mac_method, secret):
"""
Calculate a MAC for C{doc} using C{ciphertext}.
@@ -277,21 +239,38 @@ def mac_doc(doc_id, doc_rev, ciphertext, mac_method, secret):
:type doc_rev: str
:param ciphertext: The content of the document.
:type ciphertext: str
+ :param enc_scheme: The encryption scheme.
+ :type enc_scheme: str
+ :param enc_method: The encryption method.
+ :type enc_method: str
+ :param enc_iv: The encryption initialization vector.
+ :type enc_iv: str
:param mac_method: The MAC method to use.
:type mac_method: str
- :param secret: soledad secret
- :type secret: Soledad.secret_storage
+ :param secret: The Soledad storage secret
+ :type secret: str
:return: The calculated MAC.
:rtype: str
- """
- if mac_method == MacMethods.HMAC:
- return hmac.new(
- doc_mac_key(doc_id, secret),
- str(doc_id) + str(doc_rev) + ciphertext,
- hashlib.sha256).digest()
- # raise if we do not know how to handle this MAC method
- raise UnknownMacMethod('Unknown MAC method: %s.' % mac_method)
+
+ :raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown.
+ """
+ try:
+ soledad_assert(mac_method == crypto.MacMethods.HMAC)
+ except AssertionError:
+ raise crypto.UnknownMacMethodError
+ template = "{doc_id}{doc_rev}{ciphertext}{enc_scheme}{enc_method}{enc_iv}"
+ content = template.format(
+ doc_id=doc_id,
+ doc_rev=doc_rev,
+ ciphertext=ciphertext,
+ enc_scheme=enc_scheme,
+ enc_method=enc_method,
+ enc_iv=enc_iv)
+ return hmac.new(
+ doc_mac_key(doc_id, secret),
+ content,
+ hashlib.sha256).digest()
def encrypt_doc(crypto, doc):
@@ -319,12 +298,12 @@ def encrypt_docstr(docstr, doc_id, doc_rev, key, secret):
string representing the following:
{
- ENC_JSON_KEY: '<encrypted doc JSON string>',
- ENC_SCHEME_KEY: 'symkey',
- ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR,
- ENC_IV_KEY: '<the initial value used to encrypt>',
+ crypto.ENC_JSON_KEY: '<encrypted doc JSON string>',
+ crypto.ENC_SCHEME_KEY: 'symkey',
+ crypto.ENC_METHOD_KEY: crypto.EncryptionMethods.AES_256_CTR,
+ crypto.ENC_IV_KEY: '<the initial value used to encrypt>',
MAC_KEY: '<mac>'
- MAC_METHOD_KEY: 'hmac'
+ crypto.MAC_METHOD_KEY: 'hmac'
}
:param docstr: A representation of the document to be encrypted.
@@ -339,30 +318,40 @@ def encrypt_docstr(docstr, doc_id, doc_rev, key, secret):
:param key: The key used to encrypt ``data`` (must be 256 bits long).
:type key: str
- :param secret: The Soledad secret (used for MAC auth).
+ :param secret: The Soledad storage secret (used for MAC auth).
:type secret: str
:return: The JSON serialization of the dict representing the encrypted
content.
:rtype: str
"""
- # encrypt content using AES-256 CTR mode
- iv, ciphertext = encrypt_sym(
+ enc_scheme = crypto.EncryptionSchemes.SYMKEY
+ enc_method = crypto.EncryptionMethods.AES_256_CTR
+ mac_method = crypto.MacMethods.HMAC
+ enc_iv, ciphertext = encrypt_sym(
str(docstr), # encryption/decryption routines expect str
- key, method=EncryptionMethods.AES_256_CTR)
+ key, method=enc_method)
+ mac = binascii.b2a_hex( # store the mac as hex.
+ mac_doc(
+ doc_id,
+ doc_rev,
+ ciphertext,
+ enc_scheme,
+ enc_method,
+ enc_iv,
+ mac_method,
+ secret))
# Return a representation for the encrypted content. In the following, we
# convert binary data to hexadecimal representation so the JSON
# serialization does not complain about what it tries to serialize.
hex_ciphertext = binascii.b2a_hex(ciphertext)
return json.dumps({
- ENC_JSON_KEY: hex_ciphertext,
- ENC_SCHEME_KEY: EncryptionSchemes.SYMKEY,
- ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR,
- ENC_IV_KEY: iv,
- MAC_KEY: binascii.b2a_hex(mac_doc( # store the mac as hex.
- doc_id, doc_rev, ciphertext,
- MacMethods.HMAC, secret)),
- MAC_METHOD_KEY: MacMethods.HMAC,
+ crypto.ENC_JSON_KEY: hex_ciphertext,
+ crypto.ENC_SCHEME_KEY: enc_scheme,
+ crypto.ENC_METHOD_KEY: enc_method,
+ crypto.ENC_IV_KEY: enc_iv,
+ crypto.MAC_KEY: mac,
+ crypto.MAC_METHOD_KEY: mac_method,
})
@@ -384,27 +373,77 @@ def decrypt_doc(crypto, doc):
return decrypt_doc_dict(doc.content, doc.doc_id, doc.rev, key, secret)
+def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
+ enc_iv, mac_method, secret, doc_mac):
+ """
+ Verify that C{doc_mac} is a correct MAC for the given document.
+
+ :param doc_id: The id of the document.
+ :type doc_id: str
+ :param doc_rev: The revision of the document.
+ :type doc_rev: str
+ :param ciphertext: The content of the document.
+ :type ciphertext: str
+ :param enc_scheme: The encryption scheme.
+ :type enc_scheme: str
+ :param enc_method: The encryption method.
+ :type enc_method: str
+ :param enc_iv: The encryption initialization vector.
+ :type enc_iv: str
+ :param mac_method: The MAC method to use.
+ :type mac_method: str
+ :param secret: The Soledad storage secret
+ :type secret: str
+ :param doc_mac: The MAC to be verified against.
+ :type doc_mac: str
+
+ :raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown.
+ :raise crypto.WrongMacError: Raised when MAC could not be verified.
+ """
+ calculated_mac = mac_doc(
+ doc_id,
+ doc_rev,
+ ciphertext,
+ enc_scheme,
+ enc_method,
+ enc_iv,
+ mac_method,
+ secret)
+ # we compare mac's hashes to avoid possible timing attacks that might
+ # exploit python's builtin comparison operator behaviour, which fails
+ # immediatelly when non-matching bytes are found.
+ doc_mac_hash = hashlib.sha256(
+ binascii.a2b_hex( # the mac is stored as hex
+ doc_mac)).digest()
+ calculated_mac_hash = hashlib.sha256(calculated_mac).digest()
+
+ if doc_mac_hash != calculated_mac_hash:
+ logger.warning("Wrong MAC while decrypting doc...")
+ raise crypto.WrongMacError("Could not authenticate document's "
+ "contents.")
+
+
def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret):
"""
- Decrypt C{doc}'s content.
+ Decrypt a symmetrically encrypted C{doc}'s content.
Return the JSON string representation of the document's decrypted content.
The passed doc_dict argument should have the following structure:
{
- ENC_JSON_KEY: '<enc_blob>',
- ENC_SCHEME_KEY: '<enc_scheme>',
- ENC_METHOD_KEY: '<enc_method>',
- ENC_IV_KEY: '<initial value used to encrypt>', # (optional)
+ crypto.ENC_JSON_KEY: '<enc_blob>',
+ crypto.ENC_SCHEME_KEY: '<enc_scheme>',
+ crypto.ENC_METHOD_KEY: '<enc_method>',
+ crypto.ENC_IV_KEY: '<initial value used to encrypt>', # (optional)
MAC_KEY: '<mac>'
- MAC_METHOD_KEY: 'hmac'
+ crypto.MAC_METHOD_KEY: 'hmac'
}
C{enc_blob} is the encryption of the JSON serialization of the document's
content. For now Soledad just deals with documents whose C{enc_scheme} is
- EncryptionSchemes.SYMKEY and C{enc_method} is
- EncryptionMethods.AES_256_CTR.
+ crypto.EncryptionSchemes.SYMKEY and C{enc_method} is
+ crypto.EncryptionMethods.AES_256_CTR.
:param doc_dict: The content of the document to be decrypted.
:type doc_dict: dict
@@ -423,48 +462,35 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret):
:return: The JSON serialization of the decrypted content.
:rtype: str
+
+ :raise UnknownEncryptionMethodError: Raised when trying to decrypt from an
+ unknown encryption method.
"""
- soledad_assert(ENC_JSON_KEY in doc_dict)
- soledad_assert(ENC_SCHEME_KEY in doc_dict)
- soledad_assert(ENC_METHOD_KEY in doc_dict)
- soledad_assert(MAC_KEY in doc_dict)
- soledad_assert(MAC_METHOD_KEY in doc_dict)
-
- # verify MAC
- ciphertext = binascii.a2b_hex( # content is stored as hex.
- doc_dict[ENC_JSON_KEY])
- mac = mac_doc(
- doc_id, doc_rev,
- ciphertext,
- doc_dict[MAC_METHOD_KEY], secret)
- # we compare mac's hashes to avoid possible timing attacks that might
- # exploit python's builtin comparison operator behaviour, which fails
- # immediatelly when non-matching bytes are found.
- doc_mac_hash = hashlib.sha256(
- binascii.a2b_hex( # the mac is stored as hex
- doc_dict[MAC_KEY])).digest()
- calculated_mac_hash = hashlib.sha256(mac).digest()
+ # assert document dictionary structure
+ expected_keys = set([
+ crypto.ENC_JSON_KEY,
+ crypto.ENC_SCHEME_KEY,
+ crypto.ENC_METHOD_KEY,
+ crypto.ENC_IV_KEY,
+ crypto.MAC_KEY,
+ crypto.MAC_METHOD_KEY,
+ ])
+ soledad_assert(expected_keys.issubset(set(doc_dict.keys())))
- if doc_mac_hash != calculated_mac_hash:
- logger.warning("Wrong MAC while decrypting doc...")
- raise WrongMac('Could not authenticate document\'s contents.')
- # decrypt doc's content
- enc_scheme = doc_dict[ENC_SCHEME_KEY]
- plainjson = None
- if enc_scheme == EncryptionSchemes.SYMKEY:
- enc_method = doc_dict[ENC_METHOD_KEY]
- if enc_method == EncryptionMethods.AES_256_CTR:
- soledad_assert(ENC_IV_KEY in doc_dict)
- plainjson = decrypt_sym(
- ciphertext, key,
- method=enc_method,
- iv=doc_dict[ENC_IV_KEY])
- else:
- raise UnknownEncryptionMethod(enc_method)
- else:
- raise UnknownEncryptionScheme(enc_scheme)
-
- return plainjson
+ ciphertext = binascii.a2b_hex(doc_dict[crypto.ENC_JSON_KEY])
+ enc_scheme = doc_dict[crypto.ENC_SCHEME_KEY]
+ enc_method = doc_dict[crypto.ENC_METHOD_KEY]
+ enc_iv = doc_dict[crypto.ENC_IV_KEY]
+ doc_mac = doc_dict[crypto.MAC_KEY]
+ mac_method = doc_dict[crypto.MAC_METHOD_KEY]
+
+ soledad_assert(enc_scheme == crypto.EncryptionSchemes.SYMKEY)
+
+ _verify_doc_mac(
+ doc_id, doc_rev, ciphertext, enc_scheme, enc_method,
+ enc_iv, mac_method, secret, doc_mac)
+
+ return decrypt_sym(ciphertext, key, method=enc_method, iv=enc_iv)
def is_symmetrically_encrypted(doc):
@@ -476,534 +502,8 @@ def is_symmetrically_encrypted(doc):
:rtype: bool
"""
- if doc.content and ENC_SCHEME_KEY in doc.content:
- if doc.content[ENC_SCHEME_KEY] == EncryptionSchemes.SYMKEY:
+ if doc.content and crypto.ENC_SCHEME_KEY in doc.content:
+ if doc.content[crypto.ENC_SCHEME_KEY] \
+ == crypto.EncryptionSchemes.SYMKEY:
return True
return False
-
-
-#
-# Encrypt/decrypt pools of workers
-#
-
-class SyncEncryptDecryptPool(object):
- """
- Base class for encrypter/decrypter pools.
- """
- WORKERS = 5
-
- def __init__(self, crypto, sync_db, write_lock):
- """
- Initialize the pool of encryption-workers.
-
- :param crypto: A SoledadCryto instance to perform the encryption.
- :type crypto: leap.soledad.crypto.SoledadCrypto
-
- :param sync_db: a database connection handle
- :type sync_db: handle
-
- :param write_lock: a write lock for controlling concurrent access
- to the sync_db
- :type write_lock: threading.Lock
- """
- self._pool = multiprocessing.Pool(self.WORKERS)
- self._crypto = crypto
- self._sync_db = sync_db
- self._sync_db_write_lock = write_lock
-
- def close(self):
- """
- Cleanly close the pool of workers.
- """
- logger.debug("Closing %s" % (self.__class__.__name__,))
- self._pool.close()
- try:
- self._pool.join()
- except Exception:
- pass
-
- def terminate(self):
- """
- Terminate the pool of workers.
- """
- logger.debug("Terminating %s" % (self.__class__.__name__,))
- self._pool.terminate()
-
-
-def encrypt_doc_task(doc_id, doc_rev, content, key, secret):
- """
- Encrypt the content of the given document.
-
- :param doc_id: The document id.
- :type doc_id: str
- :param doc_rev: The document revision.
- :type doc_rev: str
- :param content: The serialized content of the document.
- :type content: str
- :param key: The encryption key.
- :type key: str
- :param secret: The Soledad secret (used for MAC auth).
- :type secret: str
-
- :return: A tuple containing the doc id, revision and encrypted content.
- :rtype: tuple(str, str, str)
- """
- encrypted_content = encrypt_docstr(
- content, doc_id, doc_rev, key, secret)
- return doc_id, doc_rev, encrypted_content
-
-
-class SyncEncrypterPool(SyncEncryptDecryptPool):
- """
- Pool of workers that spawn subprocesses to execute the symmetric encryption
- of documents to be synced.
- """
- # TODO implement throttling to reduce cpu usage??
- WORKERS = 5
- TABLE_NAME = "docs_tosync"
- FIELD_NAMES = "doc_id, rev, content"
-
- def encrypt_doc(self, doc, workers=True):
- """
- Symmetrically encrypt a document.
-
- :param doc: The document with contents to be encrypted.
- :type doc: SoledadDocument
-
- :param workers: Whether to defer the decryption to the multiprocess
- pool of workers. Useful for debugging purposes.
- :type workers: bool
- """
- soledad_assert(self._crypto is not None, "need a crypto object")
- docstr = doc.get_json()
- key = self._crypto.doc_passphrase(doc.doc_id)
- secret = self._crypto.secret
- args = doc.doc_id, doc.rev, docstr, key, secret
-
- try:
- if workers:
- res = self._pool.apply_async(
- encrypt_doc_task, args,
- callback=self.encrypt_doc_cb)
- else:
- # encrypt inline
- res = encrypt_doc_task(*args)
- self.encrypt_doc_cb(res)
-
- except Exception as exc:
- logger.exception(exc)
-
- def encrypt_doc_cb(self, result):
- """
- Insert results of encryption routine into the local sync database.
-
- :param result: A tuple containing the doc id, revision and encrypted
- content.
- :type result: tuple(str, str, str)
- """
- doc_id, doc_rev, content = result
- self.insert_encrypted_local_doc(doc_id, doc_rev, content)
-
- def insert_encrypted_local_doc(self, doc_id, doc_rev, content):
- """
- Insert the contents of the encrypted doc into the local sync
- database.
-
- :param doc_id: The document id.
- :type doc_id: str
- :param doc_rev: The document revision.
- :type doc_rev: str
- :param content: The serialized content of the document.
- :type content: str
- :param content: The encrypted document.
- :type content: str
- """
- sql_del = "DELETE FROM '%s' WHERE doc_id=?" % (self.TABLE_NAME,)
- sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?)" % (self.TABLE_NAME,)
-
- con = self._sync_db
- with self._sync_db_write_lock:
- with con:
- con.execute(sql_del, (doc_id, ))
- con.execute(sql_ins, (doc_id, doc_rev, content))
-
-
-def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret):
- """
- Decrypt the content of the given document.
-
- :param doc_id: The document id.
- :type doc_id: str
- :param doc_rev: The document revision.
- :type doc_rev: str
- :param content: The encrypted content of the document.
- :type content: str
- :param gen: The generation corresponding to the modification of that
- document.
- :type gen: int
- :param trans_id: The transaction id corresponding to the modification of
- that document.
- :type trans_id: str
- :param key: The encryption key.
- :type key: str
- :param secret: The Soledad secret (used for MAC auth).
- :type secret: str
-
- :return: A tuple containing the doc id, revision and encrypted content.
- :rtype: tuple(str, str, str)
- """
- decrypted_content = decrypt_doc_dict(
- content, doc_id, doc_rev, key, secret)
- return doc_id, doc_rev, decrypted_content, gen, trans_id
-
-
-def get_insertable_docs_by_gen(expected, got):
- """
- Return a list of documents ready to be inserted. This list is computed
- by aligning the expected list with the already gotten docs, and returning
- the maximum number of docs that can be processed in the expected order
- before finding a gap.
-
- :param expected: A list of generations to be inserted.
- :type expected: list
-
- :param got: A dictionary whose values are the docs to be inserted.
- :type got: dict
- """
- ordered = [got.get(i) for i in expected]
- if None in ordered:
- return ordered[:ordered.index(None)]
- else:
- return ordered
-
-
-class SyncDecrypterPool(SyncEncryptDecryptPool):
- """
- Pool of workers that spawn subprocesses to execute the symmetric decryption
- of documents that were received.
-
- The decryption of the received documents is done in two steps:
-
- 1. All the encrypted docs are collected, together with their generation
- and transaction-id
- 2. The docs are enqueued for decryption. When completed, they are
- inserted following the generation order.
- """
- # TODO implement throttling to reduce cpu usage??
- TABLE_NAME = "docs_received"
- FIELD_NAMES = "doc_id, rev, content, gen, trans_id"
-
- write_encrypted_lock = threading.Lock()
-
- def __init__(self, *args, **kwargs):
- """
- Initialize the decrypter pool, and setup a dict for putting the
- results of the decrypted docs until they are picked by the insert
- routine that gets them in order.
- """
- self._insert_doc_cb = kwargs.pop("insert_doc_cb")
- SyncEncryptDecryptPool.__init__(self, *args, **kwargs)
- self.decrypted_docs = {}
- self.source_replica_uid = None
-
- def set_source_replica_uid(self, source_replica_uid):
- """
- Set the source replica uid for this decrypter pool instance.
-
- :param source_replica_uid: The uid of the source replica.
- :type source_replica_uid: str
- """
- self.source_replica_uid = source_replica_uid
-
- def insert_encrypted_received_doc(self, doc_id, doc_rev, content,
- gen, trans_id):
- """
- Insert a received message with encrypted content, to be decrypted later
- on.
-
- :param doc_id: The Document ID.
- :type doc_id: str
- :param doc_rev: The Document Revision
- :param doc_rev: str
- :param content: the Content of the document
- :type content: str
- :param gen: the Document Generation
- :type gen: int
- :param trans_id: Transaction ID
- :type trans_id: str
- """
- docstr = json.dumps(content)
- sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?)" % (
- self.TABLE_NAME,)
-
- con = self._sync_db
- with self._sync_db_write_lock:
- with con:
- con.execute(sql_ins, (doc_id, doc_rev, docstr, gen, trans_id))
-
- def insert_marker_for_received_doc(self, doc_id, doc_rev, gen):
- """
- Insert a marker with the document id, revision and generation on the
- sync db. This document does not have an encrypted payload, so the
- content has already been inserted into the decrypted_docs dictionary
- from where it can be picked following generation order.
- We need to leave here the marker to be able to calculate the expected
- insertion order for a synchronization batch.
-
- :param doc_id: The Document ID.
- :type doc_id: str
- :param doc_rev: The Document Revision
- :param doc_rev: str
- :param gen: the Document Generation
- :type gen: int
- """
- sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?)" % (
- self.TABLE_NAME,)
- con = self._sync_db
- with self._sync_db_write_lock:
- with con:
- con.execute(sql_ins, (doc_id, doc_rev, '', gen, ''))
-
- def insert_received_doc(self, doc_id, doc_rev, content, gen, trans_id):
- """
- Insert a document that is not symmetrically encrypted.
- We store it in the staging area (the decrypted_docs dictionary) to be
- picked up in order as the preceding documents are decrypted.
-
- :param doc_id: The Document ID.
- :type doc_id: str
- :param doc_rev: The Document Revision
- :param doc_rev: str
- :param content: the Content of the document
- :type content: str
- :param gen: the Document Generation
- :type gen: int
- :param trans_id: Transaction ID
- :type trans_id: str
- """
- # XXX this need a deeper review / testing.
- # I believe that what I'm doing here is prone to problems
- # if the sync is interrupted (ie, client crash) in the worst possible
- # moment. We would need a recover strategy in that case
- # (or, insert the document in the table all the same, but with a flag
- # saying if the document is sym-encrypted or not),
- content = json.dumps(content)
- result = doc_id, doc_rev, content, gen, trans_id
- self.decrypted_docs[gen] = result
- self.insert_marker_for_received_doc(doc_id, doc_rev, gen)
-
- def delete_encrypted_received_doc(self, doc_id, doc_rev):
- """
- Delete a encrypted received doc after it was inserted into the local
- db.
-
- :param doc_id: Document ID.
- :type doc_id: str
- :param doc_rev: Document revision.
- :type doc_rev: str
- """
- sql_del = "DELETE FROM '%s' WHERE doc_id=? AND rev=?" % (
- self.TABLE_NAME,)
- con = self._sync_db
- with self._sync_db_write_lock:
- with con:
- con.execute(sql_del, (doc_id, doc_rev))
-
- def decrypt_doc(self, doc_id, rev, source_replica_uid, workers=True):
- """
- Symmetrically decrypt a document.
-
- :param doc_id: The ID for the document with contents to be encrypted.
- :type doc: str
- :param rev: The revision of the document.
- :type rev: str
- :param source_replica_uid:
- :type source_replica_uid: str
-
- :param workers: Whether to defer the decryption to the multiprocess
- pool of workers. Useful for debugging purposes.
- :type workers: bool
- """
- self.source_replica_uid = source_replica_uid
-
- # insert_doc_cb is a proxy object that gets updated with the right
- # insert function only when the sync_target invokes the sync_exchange
- # method. so, if we don't still have a non-empty callback, we refuse
- # to proceed.
- if sameProxiedObjects(self._insert_doc_cb.get(source_replica_uid),
- None):
- logger.debug("Sync decrypter pool: no insert_doc_cb() yet.")
- return
-
- # XXX move to get_doc function...
- c = self._sync_db.cursor()
- sql = "SELECT * FROM '%s' WHERE doc_id=? AND rev=?" % (
- self.TABLE_NAME,)
- try:
- c.execute(sql, (doc_id, rev))
- res = c.fetchone()
- except Exception as exc:
- logger.warning("Error getting docs from syncdb: %r" % (exc,))
- return
- if res is None:
- logger.debug("Doc %s:%s does not exist in sync db" % (doc_id, rev))
- return
-
- soledad_assert(self._crypto is not None, "need a crypto object")
- try:
- doc_id, rev, docstr, gen, trans_id = res
- except ValueError:
- logger.warning("Wrong entry in sync db")
- return
-
- if len(docstr) == 0:
- # not encrypted payload
- return
-
- try:
- content = json.loads(docstr)
- except TypeError:
- logger.warning("Wrong type while decoding json: %s" % repr(docstr))
- return
-
- key = self._crypto.doc_passphrase(doc_id)
- secret = self._crypto.secret
- args = doc_id, rev, content, gen, trans_id, key, secret
-
- try:
- if workers:
- # Ouch. This is sent to the workers asynchronously, so
- # we have no way of logging errors. We'd have to inspect
- # lingering results by querying successful / get() over them...
- # Or move the heck out of it to twisted.
- res = self._pool.apply_async(
- decrypt_doc_task, args,
- callback=self.decrypt_doc_cb)
- else:
- # decrypt inline
- res = decrypt_doc_task(*args)
- self.decrypt_doc_cb(res)
-
- except Exception as exc:
- logger.exception(exc)
-
- def decrypt_doc_cb(self, result):
- """
- Temporarily store the decryption result in a dictionary where it will
- be picked by process_decrypted.
-
- :param result: A tuple containing the doc id, revision and encrypted
- content.
- :type result: tuple(str, str, str)
- """
- doc_id, rev, content, gen, trans_id = result
- logger.debug("Sync decrypter pool: decrypted doc %s: %s %s" % (doc_id, rev, gen))
- self.decrypted_docs[gen] = result
-
- def get_docs_by_generation(self):
- """
- Get all documents in the received table from the sync db,
- ordered by generation.
-
- :return: list of doc_id, rev, generation
- """
- c = self._sync_db.cursor()
- sql = "SELECT doc_id, rev, gen FROM %s ORDER BY gen" % (
- self.TABLE_NAME,)
- c.execute(sql)
- return c.fetchall()
-
- def count_received_encrypted_docs(self):
- """
- Count how many documents we have in the table for received and
- encrypted docs.
-
- :return: The count of documents.
- :rtype: int
- """
- if self._sync_db is None:
- logger.warning("cannot return count with null sync_db")
- return
- c = self._sync_db.cursor()
- sql = "SELECT COUNT(*) FROM %s" % (self.TABLE_NAME,)
- c.execute(sql)
- res = c.fetchone()
- if res is not None:
- return res[0]
- else:
- return 0
-
- def decrypt_received_docs(self):
- """
- Get all the encrypted documents from the sync database and dispatch a
- decrypt worker to decrypt each one of them.
- """
- docs_by_generation = self.get_docs_by_generation()
- logger.debug("Sync decrypter pool: There are %d documents to " \
- "decrypt." % len(docs_by_generation))
- for doc_id, rev, gen in filter(None, docs_by_generation):
- self.decrypt_doc(doc_id, rev, self.source_replica_uid)
-
- def process_decrypted(self):
- """
- Process the already decrypted documents, and insert as many documents
- as can be taken from the expected order without finding a gap.
-
- :return: Whether we have processed all the pending docs.
- :rtype: bool
- """
- # Acquire the lock to avoid processing while we're still
- # getting data from the syncing stream, to avoid InvalidGeneration
- # problems.
- with self.write_encrypted_lock:
- already_decrypted = self.decrypted_docs
- docs = self.get_docs_by_generation()
- docs = filter(lambda entry: len(entry) > 0, docs)
- expected = [gen for doc_id, rev, gen in docs]
- docs_to_insert = get_insertable_docs_by_gen(
- expected, already_decrypted)
- for doc_fields in docs_to_insert:
- self.insert_decrypted_local_doc(*doc_fields)
- remaining = self.count_received_encrypted_docs()
- return remaining == 0
-
- def insert_decrypted_local_doc(self, doc_id, doc_rev, content,
- gen, trans_id):
- """
- Insert the decrypted document into the local sqlcipher database.
- Makes use of the passed callback `return_doc_cb` passed to the caller
- by u1db sync.
-
- :param doc_id: The document id.
- :type doc_id: str
- :param doc_rev: The document revision.
- :type doc_rev: str
- :param content: The serialized content of the document.
- :type content: str
- :param gen: The generation corresponding to the modification of that
- document.
- :type gen: int
- :param trans_id: The transaction id corresponding to the modification
- of that document.
- :type trans_id: str
- """
- # could pass source_replica in params for callback chain
- insert_fun = self._insert_doc_cb[self.source_replica_uid]
- logger.debug("Sync decrypter pool: inserting doc in local db: " \
- "%s:%s %s" % (doc_id, doc_rev, gen))
- try:
- # convert deleted documents to avoid error on document creation
- if content == 'null':
- content = None
- doc = SoledadDocument(doc_id, doc_rev, content)
- insert_fun(doc, int(gen), trans_id)
- except Exception as exc:
- logger.error("Sync decrypter pool: error while inserting "
- "decrypted doc into local db.")
- logger.exception(exc)
-
- else:
- # If no errors found, remove it from the local temporary dict
- # and from the received database.
- self.decrypted_docs.pop(gen)
- self.delete_encrypted_received_doc(doc_id, doc_rev)
diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py
new file mode 100644
index 00000000..d9a72b25
--- /dev/null
+++ b/client/src/leap/soledad/client/encdecpool.py
@@ -0,0 +1,746 @@
+# -*- coding: utf-8 -*-
+# encdecpool.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+A pool of encryption/decryption concurrent and parallel workers for using
+during synchronization.
+"""
+
+
+import multiprocessing
+import Queue
+import json
+import logging
+
+from twisted.internet import reactor
+from twisted.internet import defer
+from twisted.internet.threads import deferToThread
+
+from leap.soledad.common.document import SoledadDocument
+from leap.soledad.common import soledad_assert
+
+from leap.soledad.client.crypto import encrypt_docstr
+from leap.soledad.client.crypto import decrypt_doc_dict
+
+
+logger = logging.getLogger(__name__)
+
+
+#
+# Encrypt/decrypt pools of workers
+#
+
+class SyncEncryptDecryptPool(object):
+ """
+ Base class for encrypter/decrypter pools.
+ """
+
+ # TODO implement throttling to reduce cpu usage??
+ WORKERS = multiprocessing.cpu_count()
+
+ def __init__(self, crypto, sync_db):
+ """
+ Initialize the pool of encryption-workers.
+
+ :param crypto: A SoledadCryto instance to perform the encryption.
+ :type crypto: leap.soledad.crypto.SoledadCrypto
+
+ :param sync_db: A database connection handle
+ :type sync_db: pysqlcipher.dbapi2.Connection
+ """
+ self._crypto = crypto
+ self._sync_db = sync_db
+ self._pool = multiprocessing.Pool(self.WORKERS)
+
+ def close(self):
+ """
+ Cleanly close the pool of workers.
+ """
+ logger.debug("Closing %s" % (self.__class__.__name__,))
+ self._pool.close()
+ try:
+ self._pool.join()
+ except Exception:
+ pass
+
+ def terminate(self):
+ """
+ Terminate the pool of workers.
+ """
+ logger.debug("Terminating %s" % (self.__class__.__name__,))
+ self._pool.terminate()
+
+ def _runOperation(self, query, *args):
+ """
+ Run an operation on the sync db.
+
+ :param query: The query to be executed.
+ :type query: str
+ :param args: A list of query arguments.
+ :type args: list
+
+ :return: A deferred that will fire when the operation in the database
+ has finished.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._sync_db.runOperation(query, *args)
+
+ def _runQuery(self, query, *args):
+ """
+ Run a query on the sync db.
+
+ :param query: The query to be executed.
+ :type query: str
+ :param args: A list of query arguments.
+ :type args: list
+
+ :return: A deferred that will fire with the results of the database
+ query.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ return self._sync_db.runQuery(query, *args)
+
+
+def encrypt_doc_task(doc_id, doc_rev, content, key, secret):
+ """
+ Encrypt the content of the given document.
+
+ :param doc_id: The document id.
+ :type doc_id: str
+ :param doc_rev: The document revision.
+ :type doc_rev: str
+ :param content: The serialized content of the document.
+ :type content: str
+ :param key: The encryption key.
+ :type key: str
+ :param secret: The Soledad storage secret (used for MAC auth).
+ :type secret: str
+
+ :return: A tuple containing the doc id, revision and encrypted content.
+ :rtype: tuple(str, str, str)
+ """
+ encrypted_content = encrypt_docstr(
+ content, doc_id, doc_rev, key, secret)
+ return doc_id, doc_rev, encrypted_content
+
+
+class SyncEncrypterPool(SyncEncryptDecryptPool):
+ """
+ Pool of workers that spawn subprocesses to execute the symmetric encryption
+ of documents to be synced.
+ """
+ TABLE_NAME = "docs_tosync"
+ FIELD_NAMES = "doc_id PRIMARY KEY, rev, content"
+
+ ENCRYPT_LOOP_PERIOD = 0.5
+
+ def __init__(self, *args, **kwargs):
+ """
+ Initialize the sync encrypter pool.
+ """
+ SyncEncryptDecryptPool.__init__(self, *args, **kwargs)
+
+ self._stopped = False
+ self._sync_queue = multiprocessing.Queue()
+
+ # start the encryption loop
+ self._deferred_loop = deferToThread(self._encrypt_docs_loop)
+ self._deferred_loop.addCallback(
+ lambda _: logger.debug("Finished encrypter thread."))
+
+ def enqueue_doc_for_encryption(self, doc):
+ """
+ Enqueue a document for encryption.
+
+ :param doc: The document to be encrypted.
+ :type doc: SoledadDocument
+ """
+ try:
+ self.sync_queue.put_nowait(doc)
+ except multiprocessing.Queue.Full:
+ # do not asynchronously encrypt this file if the queue is full
+ pass
+
+ def _encrypt_docs_loop(self):
+ """
+ Process the syncing queue and send the documents there to be encrypted
+ in the sync db. They will be read by the SoledadSyncTarget during the
+ sync_exchange.
+ """
+ logger.debug("Starting encrypter thread.")
+ while not self._stopped:
+ try:
+ doc = self._sync_queue.get(True, self.ENCRYPT_LOOP_PERIOD)
+ self._encrypt_doc(doc)
+ except Queue.Empty:
+ pass
+
+ def _encrypt_doc(self, doc):
+ """
+ Symmetrically encrypt a document.
+
+ :param doc: The document with contents to be encrypted.
+ :type doc: SoledadDocument
+
+ :param workers: Whether to defer the decryption to the multiprocess
+ pool of workers. Useful for debugging purposes.
+ :type workers: bool
+ """
+ soledad_assert(self._crypto is not None, "need a crypto object")
+ docstr = doc.get_json()
+ key = self._crypto.doc_passphrase(doc.doc_id)
+ secret = self._crypto.secret
+ args = doc.doc_id, doc.rev, docstr, key, secret
+ # encrypt asynchronously
+ self._pool.apply_async(
+ encrypt_doc_task, args,
+ callback=self._encrypt_doc_cb)
+
+ def _encrypt_doc_cb(self, result):
+ """
+ Insert results of encryption routine into the local sync database.
+
+ :param result: A tuple containing the doc id, revision and encrypted
+ content.
+ :type result: tuple(str, str, str)
+ """
+ doc_id, doc_rev, content = result
+ return self._insert_encrypted_local_doc(doc_id, doc_rev, content)
+
+ def _insert_encrypted_local_doc(self, doc_id, doc_rev, content):
+ """
+ Insert the contents of the encrypted doc into the local sync
+ database.
+
+ :param doc_id: The document id.
+ :type doc_id: str
+ :param doc_rev: The document revision.
+ :type doc_rev: str
+ :param content: The serialized content of the document.
+ :type content: str
+ """
+ query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?)" \
+ % (self.TABLE_NAME,)
+ return self._runOperation(query, (doc_id, doc_rev, content))
+
+ @defer.inlineCallbacks
+ def get_encrypted_doc(self, doc_id, doc_rev):
+ """
+ Get an encrypted document from the sync db.
+
+ :param doc_id: The id of the document.
+ :type doc_id: str
+ :param doc_rev: The revision of the document.
+ :type doc_rev: str
+
+ :return: A deferred that will fire with the encrypted content of the
+ document or None if the document was not found in the sync
+ db.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ logger.debug("Trying to get encrypted doc from sync db: %s" % doc_id)
+ query = "SELECT content FROM %s WHERE doc_id=? and rev=?" \
+ % self.TABLE_NAME
+ result = yield self._runQuery(query, (doc_id, doc_rev))
+ if result:
+ val = result.pop()
+ defer.returnValue(val[0])
+ defer.returnValue(None)
+
+ def delete_encrypted_doc(self, doc_id, doc_rev):
+ """
+ Delete an encrypted document from the sync db.
+
+ :param doc_id: The id of the document.
+ :type doc_id: str
+ :param doc_rev: The revision of the document.
+ :type doc_rev: str
+
+ :return: A deferred that will fire when the operation in the database
+ has finished.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ query = "DELETE FROM %s WHERE doc_id=? and rev=?" \
+ % self.TABLE_NAME
+ self._runOperation(query, (doc_id, doc_rev))
+
+ def close(self):
+ """
+ Close the encrypter pool.
+ """
+ self._stopped = True
+ self._sync_queue.close()
+ q = self._sync_queue
+ del q
+ self._sync_queue = None
+
+
+def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret,
+ idx):
+ """
+ Decrypt the content of the given document.
+
+ :param doc_id: The document id.
+ :type doc_id: str
+ :param doc_rev: The document revision.
+ :type doc_rev: str
+ :param content: The encrypted content of the document.
+ :type content: str
+ :param gen: The generation corresponding to the modification of that
+ document.
+ :type gen: int
+ :param trans_id: The transaction id corresponding to the modification of
+ that document.
+ :type trans_id: str
+ :param key: The encryption key.
+ :type key: str
+ :param secret: The Soledad storage secret (used for MAC auth).
+ :type secret: str
+ :param idx: The index of this document in the current sync process.
+ :type idx: int
+
+ :return: A tuple containing the doc id, revision and encrypted content.
+ :rtype: tuple(str, str, str)
+ """
+ decrypted_content = decrypt_doc_dict(content, doc_id, doc_rev, key, secret)
+ return doc_id, doc_rev, decrypted_content, gen, trans_id, idx
+
+
+class SyncDecrypterPool(SyncEncryptDecryptPool):
+ """
+ Pool of workers that spawn subprocesses to execute the symmetric decryption
+ of documents that were received.
+
+ The decryption of the received documents is done in two steps:
+
+ 1. Encrypted documents are stored in the sync db by the actual soledad
+ sync loop.
+ 2. The soledad sync loop tells us how many documents we should expect
+ to process.
+ 3. We start a decrypt-and-process loop:
+
+ a. Encrypted documents are fetched.
+ b. Encrypted documents are decrypted.
+ c. The longest possible list of decrypted documents are inserted
+ in the soledad db (this depends on which documents have already
+ arrived and which documents have already been decrypte, because
+ the order of insertion in the local soledad db matters).
+ d. Processed documents are deleted from the database.
+
+ 4. When we have processed as many documents as we should, the loop
+ finishes.
+ """
+ # TODO implement throttling to reduce cpu usage??
+ TABLE_NAME = "docs_received"
+ FIELD_NAMES = "doc_id PRIMARY KEY, rev, content, gen, " \
+ "trans_id, encrypted, idx"
+
+ """
+ Period of recurrence of the periodic decrypting task, in seconds.
+ """
+ DECRYPT_LOOP_PERIOD = 0.5
+
+ def __init__(self, *args, **kwargs):
+ """
+ Initialize the decrypter pool, and setup a dict for putting the
+ results of the decrypted docs until they are picked by the insert
+ routine that gets them in order.
+
+ :param insert_doc_cb: A callback for inserting received documents from
+ target. If not overriden, this will call u1db
+ insert_doc_from_target in synchronizer, which
+ implements the TAKE OTHER semantics.
+ :type insert_doc_cb: function
+ :param source_replica_uid: The source replica uid, used to find the
+ correct callback for inserting documents.
+ :type source_replica_uid: str
+ """
+ self._insert_doc_cb = kwargs.pop("insert_doc_cb")
+ self.source_replica_uid = kwargs.pop("source_replica_uid")
+ SyncEncryptDecryptPool.__init__(self, *args, **kwargs)
+
+ self._last_inserted_idx = 0
+ self._docs_to_process = None
+ self._processed_docs = 0
+
+ self._async_results = []
+ self._failure = None
+ self._finished = False
+
+ # XXX we want to empty the database before starting, but this is an
+ # asynchronous call, so we have to somehow make sure that it is
+ # executed before any other call to the database, without
+ # blocking.
+ self._empty()
+
+ def _launch_decrypt_and_process(self):
+ d = self._decrypt_and_process_docs()
+ d.addErrback(lambda f: self._set_failure(f))
+
+ def _schedule_decrypt_and_process(self):
+ reactor.callLater(
+ self.DECRYPT_LOOP_PERIOD,
+ self._launch_decrypt_and_process)
+
+ @property
+ def failure(self):
+ return self._failure
+
+ def _set_failure(self, failure):
+ self._failure = failure
+ self._finished = True
+
+ def failed(self):
+ return bool(self._failure)
+
+ def start(self, docs_to_process):
+ """
+ Set the number of documents we expect to process.
+
+ This should be called by the during the sync exchange process as soon
+ as we know how many documents are arriving from the server.
+
+ :param docs_to_process: The number of documents to process.
+ :type docs_to_process: int
+ """
+ self._docs_to_process = docs_to_process
+ self._schedule_decrypt_and_process()
+
+ def insert_encrypted_received_doc(
+ self, doc_id, doc_rev, content, gen, trans_id, idx):
+ """
+ Insert a received message with encrypted content, to be decrypted later
+ on.
+
+ :param doc_id: The Document ID.
+ :type doc_id: str
+ :param doc_rev: The Document Revision
+ :param doc_rev: str
+ :param content: the Content of the document
+ :type content: str
+ :param gen: the Document Generation
+ :type gen: int
+ :param trans_id: Transaction ID
+ :type trans_id: str
+ :param idx: The index of this document in the current sync process.
+ :type idx: int
+
+ :return: A deferred that will fire when the operation in the database
+ has finished.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ docstr = json.dumps(content)
+ query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?, ?, ?, ?, ?)" \
+ % self.TABLE_NAME
+ return self._runOperation(
+ query, (doc_id, doc_rev, docstr, gen, trans_id, 1, idx))
+
+ def insert_received_doc(
+ self, doc_id, doc_rev, content, gen, trans_id, idx):
+ """
+ Insert a document that is not symmetrically encrypted.
+ We store it in the staging area (the decrypted_docs dictionary) to be
+ picked up in order as the preceding documents are decrypted.
+
+ :param doc_id: The Document ID.
+ :type doc_id: str
+ :param doc_rev: The Document Revision
+ :param doc_rev: str
+ :param content: the Content of the document
+ :type content: str
+ :param gen: the Document Generation
+ :type gen: int
+ :param trans_id: Transaction ID
+ :type trans_id: str
+ :param idx: The index of this document in the current sync process.
+ :type idx: int
+
+ :return: A deferred that will fire when the operation in the database
+ has finished.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ if not isinstance(content, str):
+ content = json.dumps(content)
+ query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?, ?, ?, ?, ?)" \
+ % self.TABLE_NAME
+ return self._runOperation(
+ query, (doc_id, doc_rev, content, gen, trans_id, 0, idx))
+
+ def _delete_received_doc(self, doc_id):
+ """
+ Delete a received doc after it was inserted into the local db.
+
+ :param doc_id: Document ID.
+ :type doc_id: str
+
+ :return: A deferred that will fire when the operation in the database
+ has finished.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ query = "DELETE FROM '%s' WHERE doc_id=?" \
+ % self.TABLE_NAME
+ return self._runOperation(query, (doc_id,))
+
+ def _async_decrypt_doc(self, doc_id, rev, content, gen, trans_id, idx):
+ """
+ Dispatch an asynchronous document decrypting routine and save the
+ result object.
+
+ :param doc_id: The ID for the document with contents to be encrypted.
+ :type doc: str
+ :param rev: The revision of the document.
+ :type rev: str
+ :param content: The serialized content of the document.
+ :type content: str
+ :param gen: The generation corresponding to the modification of that
+ document.
+ :type gen: int
+ :param trans_id: The transaction id corresponding to the modification
+ of that document.
+ :type trans_id: str
+ :param idx: The index of this document in the current sync process.
+ :type idx: int
+
+ :return: A deferred that will fire after the document hasa been
+ decrypted and inserted in the sync db.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ soledad_assert(self._crypto is not None, "need a crypto object")
+
+ content = json.loads(content)
+ key = self._crypto.doc_passphrase(doc_id)
+ secret = self._crypto.secret
+ args = doc_id, rev, content, gen, trans_id, key, secret, idx
+ # decrypt asynchronously
+ self._async_results.append(
+ self._pool.apply_async(
+ decrypt_doc_task, args))
+
+ def _decrypt_doc_cb(self, result):
+ """
+ Store the decryption result in the sync db from where it will later be
+ picked by _process_decrypted_docs.
+
+ :param result: A tuple containing the document's id, revision,
+ content, generation, transaction id and sync index.
+ :type result: tuple(str, str, str, int, str, int)
+
+ :return: A deferred that will fire after the document has been
+ inserted in the sync db.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ doc_id, rev, content, gen, trans_id, idx = result
+ logger.debug("Sync decrypter pool: decrypted doc %s: %s %s %s"
+ % (doc_id, rev, gen, trans_id))
+ return self.insert_received_doc(
+ doc_id, rev, content, gen, trans_id, idx)
+
+ def _get_docs(self, encrypted=None, order_by='idx', order='ASC'):
+ """
+ Get documents from the received docs table in the sync db.
+
+ :param encrypted: If not None, only return documents with encrypted
+ field equal to given parameter.
+ :type encrypted: bool or None
+ :param order_by: The name of the field to order results.
+ :type order_by: str
+ :param order: Whether the order should be ASC or DESC.
+ :type order: str
+
+ :return: A deferred that will fire with the results of the database
+ query.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ query = "SELECT doc_id, rev, content, gen, trans_id, encrypted, " \
+ "idx FROM %s" % self.TABLE_NAME
+ if encrypted is not None:
+ query += " WHERE encrypted = %d" % int(encrypted)
+ query += " ORDER BY %s %s" % (order_by, order)
+ return self._runQuery(query)
+
+ @defer.inlineCallbacks
+ def _get_insertable_docs(self):
+ """
+ Return a list of non-encrypted documents ready to be inserted.
+
+ :return: A deferred that will fire with the list of insertable
+ documents.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ # here, we fetch the list of decrypted documents and compare with the
+ # index of the last succesfully processed document.
+ decrypted_docs = yield self._get_docs(encrypted=False)
+ insertable = []
+ last_idx = self._last_inserted_idx
+ for doc_id, rev, content, gen, trans_id, encrypted, idx in \
+ decrypted_docs:
+ # XXX for some reason, a document might not have been deleted from
+ # the database. This is a bug. In this point, already
+ # processed documents should have been removed from the sync
+ # database and we should not have to skip them here. We need
+ # to find out why this is happening, fix, and remove the
+ # skipping below.
+ if (idx < last_idx + 1):
+ continue
+ if (idx != last_idx + 1):
+ break
+ insertable.append((doc_id, rev, content, gen, trans_id, idx))
+ last_idx += 1
+ defer.returnValue(insertable)
+
+ @defer.inlineCallbacks
+ def _async_decrypt_received_docs(self):
+ """
+ Get all the encrypted documents from the sync database and dispatch a
+ decrypt worker to decrypt each one of them.
+
+ :return: A deferred that will fire after all documents have been
+ decrypted and inserted back in the sync db.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ docs = yield self._get_docs(encrypted=True)
+ for doc_id, rev, content, gen, trans_id, _, idx in docs:
+ self._async_decrypt_doc(
+ doc_id, rev, content, gen, trans_id, idx)
+
+ @defer.inlineCallbacks
+ def _process_decrypted_docs(self):
+ """
+ Fetch as many decrypted documents as can be taken from the expected
+ order and insert them in the local replica.
+
+ :return: A deferred that will fire with the list of inserted
+ documents.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ insertable = yield self._get_insertable_docs()
+ for doc_fields in insertable:
+ self._insert_decrypted_local_doc(*doc_fields)
+ defer.returnValue(insertable)
+
+ def _delete_processed_docs(self, inserted):
+ """
+ Delete from the sync db documents that have been processed.
+
+ :param inserted: List of documents inserted in the previous process
+ step.
+ :type inserted: list
+
+ :return: A list of deferreds that will fire when each operation in the
+ database has finished.
+ :rtype: twisted.internet.defer.DeferredList
+ """
+ deferreds = []
+ for doc_id, doc_rev, _, _, _, _ in inserted:
+ deferreds.append(
+ self._delete_received_doc(doc_id))
+ if not deferreds:
+ return defer.succeed(None)
+ return defer.gatherResults(deferreds)
+
+ def _insert_decrypted_local_doc(self, doc_id, doc_rev, content,
+ gen, trans_id, idx):
+ """
+ Insert the decrypted document into the local replica.
+
+ Make use of the passed callback `insert_doc_cb` passed to the caller
+ by u1db sync.
+
+ :param doc_id: The document id.
+ :type doc_id: str
+ :param doc_rev: The document revision.
+ :type doc_rev: str
+ :param content: The serialized content of the document.
+ :type content: str
+ :param gen: The generation corresponding to the modification of that
+ document.
+ :type gen: int
+ :param trans_id: The transaction id corresponding to the modification
+ of that document.
+ :type trans_id: str
+ """
+ # could pass source_replica in params for callback chain
+ logger.debug("Sync decrypter pool: inserting doc in local db: "
+ "%s:%s %s" % (doc_id, doc_rev, gen))
+
+ # convert deleted documents to avoid error on document creation
+ if content == 'null':
+ content = None
+ doc = SoledadDocument(doc_id, doc_rev, content)
+ gen = int(gen)
+ self._insert_doc_cb(doc, gen, trans_id)
+
+ # store info about processed docs
+ self._last_inserted_idx = idx
+ self._processed_docs += 1
+
+ def _empty(self):
+ """
+ Empty the received docs table of the sync database.
+
+ :return: A deferred that will fire when the operation in the database
+ has finished.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ query = "DELETE FROM %s WHERE 1" % (self.TABLE_NAME,)
+ return self._runOperation(query)
+
+ def _collect_async_decryption_results(self):
+ """
+ Collect the results of the asynchronous doc decryptions and re-raise
+ any exception raised by a multiprocessing async decryption call.
+
+ :raise Exception: Raised if an async call has raised an exception.
+ """
+ async_results = self._async_results[:]
+ for res in async_results:
+ if res.ready():
+ self._decrypt_doc_cb(res.get()) # might raise an exception!
+ self._async_results.remove(res)
+
+ @defer.inlineCallbacks
+ def _decrypt_and_process_docs(self):
+ """
+ Decrypt the documents received from remote replica and insert them
+ into the local one.
+
+ This method implicitelly returns a defferred (see the decorator
+ above). It should only be called by _launch_decrypt_and_process().
+ because this way any exceptions raised here will be stored by the
+ errback attached to the deferred returned.
+
+ :return: A deferred which will fire after all decrypt, process and
+ delete operations have been executed.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ if not self.failed():
+ if self._processed_docs < self._docs_to_process:
+ yield self._async_decrypt_received_docs()
+ yield self._collect_async_decryption_results()
+ docs = yield self._process_decrypted_docs()
+ yield self._delete_processed_docs(docs)
+ # recurse
+ self._schedule_decrypt_and_process()
+ else:
+ self._finished = True
+
+ def has_finished(self):
+ """
+ Return whether the decrypter has finished its work.
+ """
+ return self._finished
diff --git a/client/src/leap/soledad/client/events.py b/client/src/leap/soledad/client/events.py
index c4c09ac5..b1379521 100644
--- a/client/src/leap/soledad/client/events.py
+++ b/client/src/leap/soledad/client/events.py
@@ -20,39 +20,35 @@
Signaling functions.
"""
+from leap.common.events import emit
+from leap.common.events import catalog
-SOLEDAD_CREATING_KEYS = 'Creating keys...'
-SOLEDAD_DONE_CREATING_KEYS = 'Done creating keys.'
-SOLEDAD_DOWNLOADING_KEYS = 'Downloading keys...'
-SOLEDAD_DONE_DOWNLOADING_KEYS = 'Done downloading keys.'
-SOLEDAD_UPLOADING_KEYS = 'Uploading keys...'
-SOLEDAD_DONE_UPLOADING_KEYS = 'Done uploading keys.'
-SOLEDAD_NEW_DATA_TO_SYNC = 'New data available.'
-SOLEDAD_DONE_DATA_SYNC = 'Done data sync.'
-SOLEDAD_SYNC_SEND_STATUS = 'Sync: sent one document.'
-SOLEDAD_SYNC_RECEIVE_STATUS = 'Sync: received one document.'
-# we want to use leap.common.events to emits signals, if it is available.
-try:
- from leap.common import events
- from leap.common.events import signal
- SOLEDAD_CREATING_KEYS = events.proto.SOLEDAD_CREATING_KEYS
- SOLEDAD_DONE_CREATING_KEYS = events.proto.SOLEDAD_DONE_CREATING_KEYS
- SOLEDAD_DOWNLOADING_KEYS = events.proto.SOLEDAD_DOWNLOADING_KEYS
- SOLEDAD_DONE_DOWNLOADING_KEYS = \
- events.proto.SOLEDAD_DONE_DOWNLOADING_KEYS
- SOLEDAD_UPLOADING_KEYS = events.proto.SOLEDAD_UPLOADING_KEYS
- SOLEDAD_DONE_UPLOADING_KEYS = \
- events.proto.SOLEDAD_DONE_UPLOADING_KEYS
- SOLEDAD_NEW_DATA_TO_SYNC = events.proto.SOLEDAD_NEW_DATA_TO_SYNC
- SOLEDAD_DONE_DATA_SYNC = events.proto.SOLEDAD_DONE_DATA_SYNC
- SOLEDAD_SYNC_SEND_STATUS = events.proto.SOLEDAD_SYNC_SEND_STATUS
- SOLEDAD_SYNC_RECEIVE_STATUS = events.proto.SOLEDAD_SYNC_RECEIVE_STATUS
+SOLEDAD_CREATING_KEYS = catalog.SOLEDAD_CREATING_KEYS
+SOLEDAD_DONE_CREATING_KEYS = catalog.SOLEDAD_DONE_CREATING_KEYS
+SOLEDAD_DOWNLOADING_KEYS = catalog.SOLEDAD_DOWNLOADING_KEYS
+SOLEDAD_DONE_DOWNLOADING_KEYS = \
+ catalog.SOLEDAD_DONE_DOWNLOADING_KEYS
+SOLEDAD_UPLOADING_KEYS = catalog.SOLEDAD_UPLOADING_KEYS
+SOLEDAD_DONE_UPLOADING_KEYS = \
+ catalog.SOLEDAD_DONE_UPLOADING_KEYS
+SOLEDAD_NEW_DATA_TO_SYNC = catalog.SOLEDAD_NEW_DATA_TO_SYNC
+SOLEDAD_DONE_DATA_SYNC = catalog.SOLEDAD_DONE_DATA_SYNC
+SOLEDAD_SYNC_SEND_STATUS = catalog.SOLEDAD_SYNC_SEND_STATUS
+SOLEDAD_SYNC_RECEIVE_STATUS = catalog.SOLEDAD_SYNC_RECEIVE_STATUS
-except ImportError:
- # we define a fake signaling function and fake signal constants that will
- # allow for logging signaling attempts in case leap.common.events is not
- # available.
- def signal(signal, content=""):
- logger.info("Would signal: %s - %s." % (str(signal), content))
+__all__ = [
+ "catalog",
+ "emit",
+ "SOLEDAD_CREATING_KEYS",
+ "SOLEDAD_DONE_CREATING_KEYS",
+ "SOLEDAD_DOWNLOADING_KEYS",
+ "SOLEDAD_DONE_DOWNLOADING_KEYS",
+ "SOLEDAD_UPLOADING_KEYS",
+ "SOLEDAD_DONE_UPLOADING_KEYS",
+ "SOLEDAD_NEW_DATA_TO_SYNC",
+ "SOLEDAD_DONE_DATA_SYNC",
+ "SOLEDAD_SYNC_SEND_STATUS",
+ "SOLEDAD_SYNC_RECEIVE_STATUS",
+]
diff --git a/client/src/leap/soledad/client/examples/README b/client/src/leap/soledad/client/examples/README
new file mode 100644
index 00000000..3aed8377
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/README
@@ -0,0 +1,4 @@
+Right now, you can find here both an example of use
+and the benchmarking scripts.
+TODO move benchmark scripts to root scripts/ folder,
+and leave here only a minimal example.
diff --git a/client/src/leap/soledad/client/examples/benchmarks/.gitignore b/client/src/leap/soledad/client/examples/benchmarks/.gitignore
new file mode 100644
index 00000000..2211df63
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/benchmarks/.gitignore
@@ -0,0 +1 @@
+*.txt
diff --git a/client/src/leap/soledad/client/examples/benchmarks/get_sample.sh b/client/src/leap/soledad/client/examples/benchmarks/get_sample.sh
new file mode 100755
index 00000000..1995eee1
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/benchmarks/get_sample.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+mkdir tmp
+wget http://www.gutenberg.org/cache/epub/101/pg101.txt -O hacker_crackdown.txt
diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py
new file mode 100644
index 00000000..7fa1e38f
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py
@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+# measure_index_times.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Measure u1db retrieval times for different u1db index situations.
+"""
+from __future__ import print_function
+from functools import partial
+import datetime
+import hashlib
+import os
+import sys
+
+import u1db
+from twisted.internet import defer, reactor
+
+from leap.soledad.client import adbapi
+from leap.soledad.client.sqlcipher import SQLCipherOptions
+
+
+folder = os.environ.get("TMPDIR", "tmp")
+numdocs = int(os.environ.get("DOCS", "1000"))
+silent = os.environ.get("SILENT", False)
+tmpdb = os.path.join(folder, "test.soledad")
+
+
+sample_file = os.environ.get("SAMPLE", "hacker_crackdown.txt")
+sample_path = os.path.join(os.curdir, sample_file)
+
+try:
+ with open(sample_file) as f:
+ SAMPLE = f.readlines()
+except Exception:
+ print("[!] Problem opening sample file. Did you download "
+ "the sample, or correctly set 'SAMPLE' env var?")
+ sys.exit(1)
+
+if numdocs > len(SAMPLE):
+ print("[!] Sorry! The requested DOCS number is larger than "
+ "the num of lines in our sample file")
+ sys.exit(1)
+
+
+def debug(*args):
+ if not silent:
+ print(*args)
+
+debug("[+] db path:", tmpdb)
+debug("[+] num docs", numdocs)
+
+if os.path.isfile(tmpdb):
+ debug("[+] Removing existing db file...")
+ os.remove(tmpdb)
+
+start_time = datetime.datetime.now()
+
+opts = SQLCipherOptions(tmpdb, "secret", create=True)
+dbpool = adbapi.getConnectionPool(opts)
+
+
+def createDoc(doc):
+ return dbpool.runU1DBQuery("create_doc", doc)
+
+db_indexes = {
+ 'by-chash': ['chash'],
+ 'by-number': ['number']}
+
+
+def create_indexes(_):
+ deferreds = []
+ for index, definition in db_indexes.items():
+ d = dbpool.runU1DBQuery("create_index", index, *definition)
+ deferreds.append(d)
+ return defer.gatherResults(deferreds)
+
+
+class TimeWitness(object):
+ def __init__(self, init_time):
+ self.init_time = init_time
+
+ def get_time_count(self):
+ return datetime.datetime.now() - self.init_time
+
+
+def get_from_index(_):
+ init_time = datetime.datetime.now()
+ debug("GETTING FROM INDEX...", init_time)
+
+ def printValue(res, time):
+ print("RESULT->", res)
+ print("Index Query Took: ", time.get_time_count())
+ return res
+
+ d = dbpool.runU1DBQuery(
+ "get_from_index", "by-chash",
+ #"1150c7f10fabce0a57ce13071349fc5064f15bdb0cc1bf2852f74ef3f103aff5")
+ # XXX this is line 89 from the hacker crackdown...
+ # Should accept any other optional hash as an enviroment variable.
+ "57793320d4997a673fc7062652da0596c36a4e9fbe31310d2281e67d56d82469")
+ d.addCallback(printValue, TimeWitness(init_time))
+ return d
+
+
+def getAllDocs():
+ return dbpool.runU1DBQuery("get_all_docs")
+
+
+def errBack(e):
+ debug("[!] ERROR FOUND!!!")
+ e.printTraceback()
+ reactor.stop()
+
+
+def countDocs(_):
+ debug("counting docs...")
+ d = getAllDocs()
+ d.addCallbacks(printResult, errBack)
+ d.addCallbacks(allDone, errBack)
+ return d
+
+
+def printResult(r, **kwargs):
+ if kwargs:
+ debug(*kwargs.values())
+ elif isinstance(r, u1db.Document):
+ debug(r.doc_id, r.content['number'])
+ else:
+ len_results = len(r[1])
+ debug("GOT %s results" % len(r[1]))
+
+ if len_results == numdocs:
+ debug("ALL GOOD")
+ else:
+ debug("[!] MISSING DOCS!!!!!")
+ raise ValueError("We didn't expect this result len")
+
+
+def allDone(_):
+ debug("ALL DONE!")
+
+ #if silent:
+ end_time = datetime.datetime.now()
+ print((end_time - start_time).total_seconds())
+ reactor.stop()
+
+
+def insert_docs(_):
+ deferreds = []
+ for i in range(numdocs):
+ payload = SAMPLE[i]
+ chash = hashlib.sha256(payload).hexdigest()
+ doc = {"number": i, "payload": payload, 'chash': chash}
+ d = createDoc(doc)
+ d.addCallbacks(partial(printResult, i=i, chash=chash, payload=payload),
+ lambda e: e.printTraceback())
+ deferreds.append(d)
+ return defer.gatherResults(deferreds, consumeErrors=True)
+
+d = create_indexes(None)
+d.addCallback(insert_docs)
+d.addCallback(get_from_index)
+d.addCallback(countDocs)
+
+reactor.run()
diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py
new file mode 100644
index 00000000..c6d76e6b
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py
@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+# measure_index_times.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Measure u1db retrieval times for different u1db index situations.
+"""
+from __future__ import print_function
+from functools import partial
+import datetime
+import hashlib
+import os
+import sys
+
+import u1db
+from twisted.internet import defer, reactor
+
+from leap.soledad.client import adbapi
+from leap.soledad.client.sqlcipher import SQLCipherOptions
+
+
+folder = os.environ.get("TMPDIR", "tmp")
+numdocs = int(os.environ.get("DOCS", "1000"))
+silent = os.environ.get("SILENT", False)
+tmpdb = os.path.join(folder, "test.soledad")
+
+
+sample_file = os.environ.get("SAMPLE", "hacker_crackdown.txt")
+sample_path = os.path.join(os.curdir, sample_file)
+
+try:
+ with open(sample_file) as f:
+ SAMPLE = f.readlines()
+except Exception:
+ print("[!] Problem opening sample file. Did you download "
+ "the sample, or correctly set 'SAMPLE' env var?")
+ sys.exit(1)
+
+if numdocs > len(SAMPLE):
+ print("[!] Sorry! The requested DOCS number is larger than "
+ "the num of lines in our sample file")
+ sys.exit(1)
+
+
+def debug(*args):
+ if not silent:
+ print(*args)
+
+debug("[+] db path:", tmpdb)
+debug("[+] num docs", numdocs)
+
+if os.path.isfile(tmpdb):
+ debug("[+] Removing existing db file...")
+ os.remove(tmpdb)
+
+start_time = datetime.datetime.now()
+
+opts = SQLCipherOptions(tmpdb, "secret", create=True)
+dbpool = adbapi.getConnectionPool(opts)
+
+
+def createDoc(doc, doc_id):
+ return dbpool.runU1DBQuery("create_doc", doc, doc_id=doc_id)
+
+db_indexes = {
+ 'by-chash': ['chash'],
+ 'by-number': ['number']}
+
+
+def create_indexes(_):
+ deferreds = []
+ for index, definition in db_indexes.items():
+ d = dbpool.runU1DBQuery("create_index", index, *definition)
+ deferreds.append(d)
+ return defer.gatherResults(deferreds)
+
+
+class TimeWitness(object):
+ def __init__(self, init_time):
+ self.init_time = init_time
+
+ def get_time_count(self):
+ return datetime.datetime.now() - self.init_time
+
+
+def get_from_index(_):
+ init_time = datetime.datetime.now()
+ debug("GETTING FROM INDEX...", init_time)
+
+ def printValue(res, time):
+ print("RESULT->", res)
+ print("Index Query Took: ", time.get_time_count())
+ return res
+
+ d = dbpool.runU1DBQuery(
+ "get_doc",
+ #"1150c7f10fabce0a57ce13071349fc5064f15bdb0cc1bf2852f74ef3f103aff5")
+ # XXX this is line 89 from the hacker crackdown...
+ # Should accept any other optional hash as an enviroment variable.
+ "57793320d4997a673fc7062652da0596c36a4e9fbe31310d2281e67d56d82469")
+ d.addCallback(printValue, TimeWitness(init_time))
+ return d
+
+
+def getAllDocs():
+ return dbpool.runU1DBQuery("get_all_docs")
+
+
+def errBack(e):
+ debug("[!] ERROR FOUND!!!")
+ e.printTraceback()
+ reactor.stop()
+
+
+def countDocs(_):
+ debug("counting docs...")
+ d = getAllDocs()
+ d.addCallbacks(printResult, errBack)
+ d.addCallbacks(allDone, errBack)
+ return d
+
+
+def printResult(r, **kwargs):
+ if kwargs:
+ debug(*kwargs.values())
+ elif isinstance(r, u1db.Document):
+ debug(r.doc_id, r.content['number'])
+ else:
+ len_results = len(r[1])
+ debug("GOT %s results" % len(r[1]))
+
+ if len_results == numdocs:
+ debug("ALL GOOD")
+ else:
+ debug("[!] MISSING DOCS!!!!!")
+ raise ValueError("We didn't expect this result len")
+
+
+def allDone(_):
+ debug("ALL DONE!")
+
+ #if silent:
+ end_time = datetime.datetime.now()
+ print((end_time - start_time).total_seconds())
+ reactor.stop()
+
+
+def insert_docs(_):
+ deferreds = []
+ for i in range(numdocs):
+ payload = SAMPLE[i]
+ chash = hashlib.sha256(payload).hexdigest()
+ doc = {"number": i, "payload": payload, 'chash': chash}
+ d = createDoc(doc, doc_id=chash)
+ d.addCallbacks(partial(printResult, i=i, chash=chash, payload=payload),
+ lambda e: e.printTraceback())
+ deferreds.append(d)
+ return defer.gatherResults(deferreds, consumeErrors=True)
+
+d = create_indexes(None)
+d.addCallback(insert_docs)
+d.addCallback(get_from_index)
+d.addCallback(countDocs)
+
+reactor.run()
diff --git a/client/src/leap/soledad/client/examples/compare.txt b/client/src/leap/soledad/client/examples/compare.txt
new file mode 100644
index 00000000..19a1325a
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/compare.txt
@@ -0,0 +1,8 @@
+TIMES=100 TMPDIR=/media/sdb5/leap python use_adbapi.py 1.34s user 0.16s system 53% cpu 2.832 total
+TIMES=100 TMPDIR=/media/sdb5/leap python use_api.py 1.22s user 0.14s system 62% cpu 2.181 total
+
+TIMES=1000 TMPDIR=/media/sdb5/leap python use_api.py 2.18s user 0.34s system 27% cpu 9.213 total
+TIMES=1000 TMPDIR=/media/sdb5/leap python use_adbapi.py 2.40s user 0.34s system 39% cpu 7.004 total
+
+TIMES=5000 TMPDIR=/media/sdb5/leap python use_api.py 6.63s user 1.27s system 13% cpu 57.882 total
+TIMES=5000 TMPDIR=/media/sdb5/leap python use_adbapi.py 6.84s user 1.26s system 36% cpu 22.367 total
diff --git a/client/src/leap/soledad/client/examples/manifest.phk b/client/src/leap/soledad/client/examples/manifest.phk
new file mode 100644
index 00000000..2c86c07d
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/manifest.phk
@@ -0,0 +1,50 @@
+The Hacker's Manifesto
+
+The Hacker's Manifesto
+by: The Mentor
+
+Another one got caught today, it's all over the papers. "Teenager
+Arrested in Computer Crime Scandal", "Hacker Arrested after Bank
+Tampering." "Damn kids. They're all alike." But did you, in your
+three-piece psychology and 1950's technobrain, ever take a look behind
+the eyes of the hacker? Did you ever wonder what made him tick, what
+forces shaped him, what may have molded him? I am a hacker, enter my
+world. Mine is a world that begins with school. I'm smarter than most of
+the other kids, this crap they teach us bores me. "Damn underachiever.
+They're all alike." I'm in junior high or high school. I've listened to
+teachers explain for the fifteenth time how to reduce a fraction. I
+understand it. "No, Ms. Smith, I didn't show my work. I did it in
+my head." "Damn kid. Probably copied it. They're all alike." I made a
+discovery today. I found a computer. Wait a second, this is cool. It does
+what I want it to. If it makes a mistake, it's because I screwed it up.
+Not because it doesn't like me, or feels threatened by me, or thinks I'm
+a smart ass, or doesn't like teaching and shouldn't be here. Damn kid.
+All he does is play games. They're all alike. And then it happened... a
+door opened to a world... rushing through the phone line like heroin
+through an addict's veins, an electronic pulse is sent out, a refuge from
+the day-to-day incompetencies is sought... a board is found. "This is
+it... this is where I belong..." I know everyone here... even if I've
+never met them, never talked to them, may never hear from them again... I
+know you all... Damn kid. Tying up the phone line again. They're all
+alike... You bet your ass we're all alike... we've been spoon-fed baby
+food at school when we hungered for steak... the bits of meat that you
+did let slip through were pre-chewed and tasteless. We've been dominated
+by sadists, or ignored by the apathetic. The few that had something to
+teach found us willing pupils, but those few are like drops of water in
+the desert. This is our world now... the world of the electron and the
+switch, the beauty of the baud. We make use of a service already existing
+without paying for what could be dirt-cheap if it wasn't run by
+profiteering gluttons, and you call us criminals. We explore... and you
+call us criminals. We seek after knowledge... and you call us criminals.
+We exist without skin color, without nationality, without religious
+bias... and you call us criminals. You build atomic bombs, you wage wars,
+you murder, cheat, and lie to us and try to make us believe it's for our
+own good, yet we're the criminals. Yes, I am a criminal. My crime is that
+of curiosity. My crime is that of judging people by what they say and
+think, not what they look like. My crime is that of outsmarting you,
+something that you will never forgive me for. I am a hacker, and this is
+my manifesto. You may stop this individual, but you can't stop us all...
+after all, we're all alike.
+
+This was the last published file written by The Mentor. Shortly after
+releasing it, he was busted by the FBI. The Mentor, sadly missed.
diff --git a/client/src/leap/soledad/client/examples/plot-async-db.py b/client/src/leap/soledad/client/examples/plot-async-db.py
new file mode 100644
index 00000000..018a1a1d
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/plot-async-db.py
@@ -0,0 +1,45 @@
+import csv
+from matplotlib import pyplot as plt
+
+FILE = "bench.csv"
+
+# config the plot
+plt.xlabel('number of inserts')
+plt.ylabel('time (seconds)')
+plt.title('SQLCipher parallelization')
+
+kwargs = {
+ 'linewidth': 1.0,
+ 'linestyle': '-',
+}
+
+series = (('sync', 'r'),
+ ('async', 'g'))
+
+data = {'mark': [],
+ 'sync': [],
+ 'async': []}
+
+with open(FILE, 'rb') as csvfile:
+ series_reader = csv.reader(csvfile, delimiter=',')
+ for m, s, a in series_reader:
+ data['mark'].append(int(m))
+ data['sync'].append(float(s))
+ data['async'].append(float(a))
+
+xmax = max(data['mark'])
+xmin = min(data['mark'])
+ymax = max(data['sync'] + data['async'])
+ymin = min(data['sync'] + data['async'])
+
+for run in series:
+ name = run[0]
+ color = run[1]
+ plt.plot(data['mark'], data[name], label=name, color=color, **kwargs)
+
+plt.axes().annotate("", xy=(xmax, ymax))
+plt.axes().annotate("", xy=(xmin, ymin))
+
+plt.grid()
+plt.legend()
+plt.show()
diff --git a/client/src/leap/soledad/client/examples/run_benchmark.py b/client/src/leap/soledad/client/examples/run_benchmark.py
new file mode 100644
index 00000000..a112cf45
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/run_benchmark.py
@@ -0,0 +1,28 @@
+"""
+Run a mini-benchmark between regular api and dbapi
+"""
+import commands
+import os
+import time
+
+TMPDIR = os.environ.get("TMPDIR", "/tmp")
+CSVFILE = 'bench.csv'
+
+cmd = "SILENT=1 TIMES={times} TMPDIR={tmpdir} python ./use_{version}api.py"
+
+parse_time = lambda r: r.split('\n')[-1]
+
+
+with open(CSVFILE, 'w') as log:
+
+ for times in range(0, 10000, 500):
+ cmd1 = cmd.format(times=times, tmpdir=TMPDIR, version="")
+ sync_time = parse_time(commands.getoutput(cmd1))
+
+ cmd2 = cmd.format(times=times, tmpdir=TMPDIR, version="adb")
+ async_time = parse_time(commands.getoutput(cmd2))
+
+ print times, sync_time, async_time
+ log.write("%s, %s, %s\n" % (times, sync_time, async_time))
+ log.flush()
+ time.sleep(2)
diff --git a/client/src/leap/soledad/client/examples/soledad_sync.py b/client/src/leap/soledad/client/examples/soledad_sync.py
new file mode 100644
index 00000000..6d0f6595
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/soledad_sync.py
@@ -0,0 +1,65 @@
+from leap.bitmask.config.providerconfig import ProviderConfig
+from leap.bitmask.crypto.srpauth import SRPAuth
+from leap.soledad.client import Soledad
+
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+
+# EDIT THIS --------------------------------------------
+user = u"USERNAME"
+uuid = u"USERUUID"
+_pass = u"USERPASS"
+server_url = "https://soledad.server.example.org:2323"
+# EDIT THIS --------------------------------------------
+
+secrets_path = "/tmp/%s.secrets" % uuid
+local_db_path = "/tmp/%s.soledad" % uuid
+cert_file = "/tmp/cacert.pem"
+provider_config = '/tmp/cdev.json'
+
+
+provider = ProviderConfig()
+provider.load(provider_config)
+
+soledad = None
+
+
+def printStuff(r):
+ print r
+
+
+def printErr(err):
+ logging.exception(err.value)
+
+
+def init_soledad(_):
+ token = srpauth.get_token()
+ print "token", token
+
+ global soledad
+ soledad = Soledad(uuid, _pass, secrets_path, local_db_path,
+ server_url, cert_file,
+ auth_token=token, defer_encryption=False)
+
+ def getall(_):
+ d = soledad.get_all_docs()
+ return d
+
+ d1 = soledad.create_doc({"test": 42})
+ d1.addCallback(getall)
+ d1.addCallbacks(printStuff, printErr)
+
+ d2 = soledad.sync()
+ d2.addCallbacks(printStuff, printErr)
+ d2.addBoth(lambda r: reactor.stop())
+
+
+srpauth = SRPAuth(provider)
+
+d = srpauth.authenticate(user, _pass)
+d.addCallbacks(init_soledad, printErr)
+
+
+from twisted.internet import reactor
+reactor.run()
diff --git a/client/src/leap/soledad/client/examples/use_adbapi.py b/client/src/leap/soledad/client/examples/use_adbapi.py
new file mode 100644
index 00000000..d7bd21f2
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/use_adbapi.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+# use_adbapi.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Example of use of the asynchronous soledad api.
+"""
+from __future__ import print_function
+import datetime
+import os
+
+import u1db
+from twisted.internet import defer, reactor
+
+from leap.soledad.client import adbapi
+from leap.soledad.client.sqlcipher import SQLCipherOptions
+
+
+folder = os.environ.get("TMPDIR", "tmp")
+times = int(os.environ.get("TIMES", "1000"))
+silent = os.environ.get("SILENT", False)
+
+tmpdb = os.path.join(folder, "test.soledad")
+
+
+def debug(*args):
+ if not silent:
+ print(*args)
+
+debug("[+] db path:", tmpdb)
+debug("[+] times", times)
+
+if os.path.isfile(tmpdb):
+ debug("[+] Removing existing db file...")
+ os.remove(tmpdb)
+
+start_time = datetime.datetime.now()
+
+opts = SQLCipherOptions(tmpdb, "secret", create=True)
+dbpool = adbapi.getConnectionPool(opts)
+
+
+def createDoc(doc):
+ return dbpool.runU1DBQuery("create_doc", doc)
+
+
+def getAllDocs():
+ return dbpool.runU1DBQuery("get_all_docs")
+
+
+def countDocs(_):
+ debug("counting docs...")
+ d = getAllDocs()
+ d.addCallbacks(printResult, lambda e: e.printTraceback())
+ d.addBoth(allDone)
+
+
+def printResult(r):
+ if isinstance(r, u1db.Document):
+ debug(r.doc_id, r.content['number'])
+ else:
+ len_results = len(r[1])
+ debug("GOT %s results" % len(r[1]))
+
+ if len_results == times:
+ debug("ALL GOOD")
+ else:
+ raise ValueError("We didn't expect this result len")
+
+
+def allDone(_):
+ debug("ALL DONE!")
+ if silent:
+ end_time = datetime.datetime.now()
+ print((end_time - start_time).total_seconds())
+ reactor.stop()
+
+deferreds = []
+payload = open('manifest.phk').read()
+
+for i in range(times):
+ doc = {"number": i, "payload": payload}
+ d = createDoc(doc)
+ d.addCallbacks(printResult, lambda e: e.printTraceback())
+ deferreds.append(d)
+
+
+all_done = defer.gatherResults(deferreds, consumeErrors=True)
+all_done.addCallback(countDocs)
+
+reactor.run()
diff --git a/client/src/leap/soledad/client/examples/use_api.py b/client/src/leap/soledad/client/examples/use_api.py
new file mode 100644
index 00000000..e2501c98
--- /dev/null
+++ b/client/src/leap/soledad/client/examples/use_api.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+# use_api.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Example of use of the soledad api.
+"""
+from __future__ import print_function
+import datetime
+import os
+
+from leap.soledad.client import sqlcipher
+from leap.soledad.client.sqlcipher import SQLCipherOptions
+
+
+folder = os.environ.get("TMPDIR", "tmp")
+times = int(os.environ.get("TIMES", "1000"))
+silent = os.environ.get("SILENT", False)
+
+tmpdb = os.path.join(folder, "test.soledad")
+
+
+def debug(*args):
+ if not silent:
+ print(*args)
+
+debug("[+] db path:", tmpdb)
+debug("[+] times", times)
+
+if os.path.isfile(tmpdb):
+ debug("[+] Removing existing db file...")
+ os.remove(tmpdb)
+
+start_time = datetime.datetime.now()
+
+opts = SQLCipherOptions(tmpdb, "secret", create=True)
+db = sqlcipher.SQLCipherDatabase(opts)
+
+
+def allDone():
+ debug("ALL DONE!")
+
+payload = open('manifest.phk').read()
+
+for i in range(times):
+ doc = {"number": i, "payload": payload}
+ d = db.create_doc(doc)
+ debug(d.doc_id, d.content['number'])
+
+debug("Count", len(db.get_all_docs()[1]))
+if silent:
+ end_time = datetime.datetime.now()
+ print((end_time - start_time).total_seconds())
+
+allDone()
diff --git a/client/src/leap/soledad/client/http_target.py b/client/src/leap/soledad/client/http_target.py
new file mode 100644
index 00000000..30590ae1
--- /dev/null
+++ b/client/src/leap/soledad/client/http_target.py
@@ -0,0 +1,622 @@
+# -*- coding: utf-8 -*-
+# http_target.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+A U1DB backend for encrypting data before sending to server and decrypting
+after receiving.
+"""
+
+
+import json
+import base64
+import logging
+
+from uuid import uuid4
+from functools import partial
+
+from twisted.internet import defer
+from twisted.internet import reactor
+from twisted.web.error import Error
+
+from u1db import errors
+from u1db import SyncTarget
+from u1db.remote import utils
+
+from leap.common.http import HTTPClient
+
+from leap.soledad.common.document import SoledadDocument
+from leap.soledad.common.errors import InvalidAuthTokenError
+
+from leap.soledad.client.crypto import is_symmetrically_encrypted
+from leap.soledad.client.crypto import encrypt_doc
+from leap.soledad.client.crypto import decrypt_doc
+from leap.soledad.client.events import SOLEDAD_SYNC_SEND_STATUS
+from leap.soledad.client.events import SOLEDAD_SYNC_RECEIVE_STATUS
+from leap.soledad.client.events import emit
+from leap.soledad.client.encdecpool import SyncDecrypterPool
+
+
+logger = logging.getLogger(__name__)
+
+
+class SoledadHTTPSyncTarget(SyncTarget):
+ """
+ A SyncTarget that encrypts data before sending and decrypts data after
+ receiving.
+
+ Normally encryption will have been written to the sync database upon
+ document modification. The sync database is also used to write temporarily
+ the parsed documents that the remote send us, before being decrypted and
+ written to the main database.
+ """
+
+ def __init__(self, url, source_replica_uid, creds, crypto, cert_file,
+ sync_db=None, sync_enc_pool=None):
+ """
+ Initialize the sync target.
+
+ :param url: The server sync url.
+ :type url: str
+ :param source_replica_uid: The source replica uid which we use when
+ deferring decryption.
+ :type source_replica_uid: str
+ :param url: The url of the target replica to sync with.
+ :type url: str
+ :param creds: A dictionary containing the uuid and token.
+ :type creds: creds
+ :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt
+ document contents when syncing.
+ :type crypto: soledad.crypto.SoledadCrypto
+ :param cert_file: Path to the certificate of the ca used to validate
+ the SSL certificate used by the remote soledad
+ server.
+ :type cert_file: str
+ :param sync_db: Optional. handler for the db with the symmetric
+ encryption of the syncing documents. If
+ None, encryption will be done in-place,
+ instead of retreiving it from the dedicated
+ database.
+ :type sync_db: Sqlite handler
+ :param verify_ssl: Whether we should perform SSL server certificate
+ verification.
+ :type verify_ssl: bool
+ """
+ if url.endswith("/"):
+ url = url[:-1]
+ self._url = str(url) + "/sync-from/" + source_replica_uid
+ self.source_replica_uid = source_replica_uid
+ self._auth_header = None
+ self.set_creds(creds)
+ self._crypto = crypto
+ self._sync_db = sync_db
+ self._sync_enc_pool = sync_enc_pool
+ self._insert_doc_cb = None
+ # asynchronous encryption/decryption attributes
+ self._decryption_callback = None
+ self._sync_decr_pool = None
+ self._http = HTTPClient(cert_file)
+
+ def set_creds(self, creds):
+ """
+ Update credentials.
+
+ :param creds: A dictionary containing the uuid and token.
+ :type creds: dict
+ """
+ uuid = creds['token']['uuid']
+ token = creds['token']['token']
+ auth = '%s:%s' % (uuid, token)
+ b64_token = base64.b64encode(auth)
+ self._auth_header = {'Authorization': ['Token %s' % b64_token]}
+
+ @property
+ def _defer_encryption(self):
+ return self._sync_enc_pool is not None
+
+ #
+ # SyncTarget API
+ #
+
+ @defer.inlineCallbacks
+ def get_sync_info(self, source_replica_uid):
+ """
+ Return information about known state of remote database.
+
+ Return the replica_uid and the current database generation of the
+ remote database, and its last-seen database generation for the client
+ replica.
+
+ :param source_replica_uid: The client-size replica uid.
+ :type source_replica_uid: str
+
+ :return: A deferred which fires with (target_replica_uid,
+ target_replica_generation, target_trans_id,
+ source_replica_last_known_generation,
+ source_replica_last_known_transaction_id)
+ :rtype: twisted.internet.defer.Deferred
+ """
+ raw = yield self._http_request(self._url, headers=self._auth_header)
+ res = json.loads(raw)
+ defer.returnValue([
+ res['target_replica_uid'],
+ res['target_replica_generation'],
+ res['target_replica_transaction_id'],
+ res['source_replica_generation'],
+ res['source_transaction_id']
+ ])
+
+ def record_sync_info(
+ self, source_replica_uid, source_replica_generation,
+ source_replica_transaction_id):
+ """
+ Record tip information for another replica.
+
+ After sync_exchange has been processed, the caller will have
+ received new content from this replica. This call allows the
+ source replica instigating the sync to inform us what their
+ generation became after applying the documents we returned.
+
+ This is used to allow future sync operations to not need to repeat data
+ that we just talked about. It also means that if this is called at the
+ wrong time, there can be database records that will never be
+ synchronized.
+
+ :param source_replica_uid: The identifier for the source replica.
+ :type source_replica_uid: str
+ :param source_replica_generation: The database generation for the
+ source replica.
+ :type source_replica_generation: int
+ :param source_replica_transaction_id: The transaction id associated
+ with the source replica
+ generation.
+ :type source_replica_transaction_id: str
+
+ :return: A deferred which fires with the result of the query.
+ :rtype: twisted.internet.defer.Deferred
+ """
+ data = json.dumps({
+ 'generation': source_replica_generation,
+ 'transaction_id': source_replica_transaction_id
+ })
+ headers = self._auth_header.copy()
+ headers.update({'content-type': ['application/json']})
+ return self._http_request(
+ self._url,
+ method='PUT',
+ headers=headers,
+ body=data)
+
+ @defer.inlineCallbacks
+ def sync_exchange(self, docs_by_generation, source_replica_uid,
+ last_known_generation, last_known_trans_id,
+ insert_doc_cb, ensure_callback=None,
+ defer_decryption=True, sync_id=None):
+ """
+ Find out which documents the remote database does not know about,
+ encrypt and send them. After that, receive documents from the remote
+ database.
+
+ :param docs_by_generations: A list of (doc_id, generation, trans_id)
+ of local documents that were changed since
+ the last local generation the remote
+ replica knows about.
+ :type docs_by_generations: list of tuples
+
+ :param source_replica_uid: The uid of the source replica.
+ :type source_replica_uid: str
+
+ :param last_known_generation: Target's last known generation.
+ :type last_known_generation: int
+
+ :param last_known_trans_id: Target's last known transaction id.
+ :type last_known_trans_id: str
+
+ :param insert_doc_cb: A callback for inserting received documents from
+ target. If not overriden, this will call u1db
+ insert_doc_from_target in synchronizer, which
+ implements the TAKE OTHER semantics.
+ :type insert_doc_cb: function
+
+ :param ensure_callback: A callback that ensures we know the target
+ replica uid if the target replica was just
+ created.
+ :type ensure_callback: function
+
+ :param defer_decryption: Whether to defer the decryption process using
+ the intermediate database. If False,
+ decryption will be done inline.
+ :type defer_decryption: bool
+
+ :return: A deferred which fires with the new generation and
+ transaction id of the target replica.
+ :rtype: twisted.internet.defer.Deferred
+ """
+
+ self._ensure_callback = ensure_callback
+
+ if sync_id is None:
+ sync_id = str(uuid4())
+ self.source_replica_uid = source_replica_uid
+
+ # save a reference to the callback so we can use it after decrypting
+ self._insert_doc_cb = insert_doc_cb
+
+ gen_after_send, trans_id_after_send = yield self._send_docs(
+ docs_by_generation,
+ last_known_generation,
+ last_known_trans_id,
+ sync_id)
+
+ cur_target_gen, cur_target_trans_id = yield self._receive_docs(
+ last_known_generation, last_known_trans_id,
+ ensure_callback, sync_id,
+ defer_decryption=defer_decryption)
+
+ # update gen and trans id info in case we just sent and did not
+ # receive docs.
+ if gen_after_send is not None and gen_after_send > cur_target_gen:
+ cur_target_gen = gen_after_send
+ cur_target_trans_id = trans_id_after_send
+
+ defer.returnValue([cur_target_gen, cur_target_trans_id])
+
+ #
+ # methods to send docs
+ #
+
+ def _prepare(self, comma, entries, **dic):
+ entry = comma + '\r\n' + json.dumps(dic)
+ entries.append(entry)
+ return len(entry)
+
+ @defer.inlineCallbacks
+ def _send_docs(self, docs_by_generation, last_known_generation,
+ last_known_trans_id, sync_id):
+
+ if not docs_by_generation:
+ defer.returnValue([None, None])
+
+ headers = self._auth_header.copy()
+ headers.update({'content-type': ['application/x-soledad-sync-put']})
+ # add remote replica metadata to the request
+ first_entries = ['[']
+ self._prepare(
+ '', first_entries,
+ last_known_generation=last_known_generation,
+ last_known_trans_id=last_known_trans_id,
+ sync_id=sync_id,
+ ensure=self._ensure_callback is not None)
+ idx = 0
+ total = len(docs_by_generation)
+ for doc, gen, trans_id in docs_by_generation:
+ idx += 1
+ result = yield self._send_one_doc(
+ headers, first_entries, doc,
+ gen, trans_id, total, idx)
+ if self._defer_encryption:
+ self._sync_enc_pool.delete_encrypted_doc(
+ doc.doc_id, doc.rev)
+ emit(SOLEDAD_SYNC_SEND_STATUS,
+ "Soledad sync send status: %d/%d"
+ % (idx, total))
+ response_dict = json.loads(result)[0]
+ gen_after_send = response_dict['new_generation']
+ trans_id_after_send = response_dict['new_transaction_id']
+ defer.returnValue([gen_after_send, trans_id_after_send])
+
+ @defer.inlineCallbacks
+ def _send_one_doc(self, headers, first_entries, doc, gen, trans_id,
+ number_of_docs, doc_idx):
+ entries = first_entries[:]
+ # add the document to the request
+ content = yield self._encrypt_doc(doc)
+ self._prepare(
+ ',', entries,
+ id=doc.doc_id, rev=doc.rev, content=content, gen=gen,
+ trans_id=trans_id, number_of_docs=number_of_docs,
+ doc_idx=doc_idx)
+ entries.append('\r\n]')
+ data = ''.join(entries)
+ result = yield self._http_request(
+ self._url,
+ method='POST',
+ headers=headers,
+ body=data)
+ defer.returnValue(result)
+
+ def _encrypt_doc(self, doc):
+ d = None
+ if doc.is_tombstone():
+ d = defer.succeed(None)
+ elif not self._defer_encryption:
+ # fallback case, for tests
+ d = defer.succeed(encrypt_doc(self._crypto, doc))
+ else:
+
+ def _maybe_encrypt_doc_inline(doc_json):
+ if doc_json is None:
+ # the document is not marked as tombstone, but we got
+ # nothing from the sync db. As it is not encrypted
+ # yet, we force inline encryption.
+ return encrypt_doc(self._crypto, doc)
+ return doc_json
+
+ d = self._sync_enc_pool.get_encrypted_doc(doc.doc_id, doc.rev)
+ d.addCallback(_maybe_encrypt_doc_inline)
+ return d
+
+ #
+ # methods to receive doc
+ #
+
+ @defer.inlineCallbacks
+ def _receive_docs(self, last_known_generation, last_known_trans_id,
+ ensure_callback, sync_id, defer_decryption):
+
+ self._queue_for_decrypt = defer_decryption \
+ and self._sync_db is not None
+
+ new_generation = last_known_generation
+ new_transaction_id = last_known_trans_id
+
+ if self._queue_for_decrypt:
+ logger.debug(
+ "Soledad sync: will queue received docs for decrypting.")
+
+ if defer_decryption:
+ self._setup_sync_decr_pool()
+
+ headers = self._auth_header.copy()
+ headers.update({'content-type': ['application/x-soledad-sync-get']})
+
+ #---------------------------------------------------------------------
+ # maybe receive the first document
+ #---------------------------------------------------------------------
+
+ # we fetch the first document before fetching the rest because we need
+ # to know the total number of documents to be received, and this
+ # information comes as metadata to each request.
+
+ d = self._receive_one_doc(
+ headers, last_known_generation, last_known_trans_id,
+ sync_id, 0)
+ d.addCallback(partial(self._insert_received_doc, 1, 1))
+ number_of_changes, ngen, ntrans = yield d
+
+ if defer_decryption:
+ self._sync_decr_pool.start(number_of_changes)
+
+ #---------------------------------------------------------------------
+ # maybe receive the rest of the documents
+ #---------------------------------------------------------------------
+
+ # launch many asynchronous fetches and inserts of received documents
+ # in the temporary sync db. Will wait for all results before
+ # continuing.
+
+ received = 1
+ deferreds = []
+ while received < number_of_changes:
+ d = self._receive_one_doc(
+ headers, last_known_generation,
+ last_known_trans_id, sync_id, received)
+ d.addCallback(
+ partial(
+ self._insert_received_doc,
+ received + 1, # the index of the current received doc
+ number_of_changes))
+ deferreds.append(d)
+ received += 1
+ results = yield defer.gatherResults(deferreds)
+
+ # get generation and transaction id of target after insertions
+ if deferreds:
+ _, new_generation, new_transaction_id = results.pop()
+
+ #---------------------------------------------------------------------
+ # wait for async decryption to finish
+ #---------------------------------------------------------------------
+
+ # below we do a trick so we can wait for the SyncDecrypterPool to
+ # finish its work before finally returning the new generation and
+ # transaction id of the remote replica. To achieve that, we create a
+ # Deferred that will return the results of the sync and, if we are
+ # decrypting asynchronously, we use reactor.callLater() to
+ # periodically poll the decrypter and check if it has finished its
+ # work. When it has finished, we either call the callback or errback
+ # of that deferred. In case we are not asynchronously decrypting, we
+ # just fire the deferred.
+
+ def _shutdown_and_finish(res):
+ self._sync_decr_pool.close()
+ return new_generation, new_transaction_id
+
+ d = defer.Deferred()
+ d.addCallback(_shutdown_and_finish)
+
+ def _wait_or_finish():
+ if not self._sync_decr_pool.has_finished():
+ reactor.callLater(
+ SyncDecrypterPool.DECRYPT_LOOP_PERIOD,
+ _wait_or_finish)
+ else:
+ if not self._sync_decr_pool.failed():
+ d.callback(None)
+ else:
+ d.errback(self._sync_decr_pool.failure)
+
+ if defer_decryption:
+ _wait_or_finish()
+ else:
+ d.callback(None)
+
+ new_generation, new_transaction_id = yield d
+ defer.returnValue([new_generation, new_transaction_id])
+
+ def _receive_one_doc(self, headers, last_known_generation,
+ last_known_trans_id, sync_id, received):
+ entries = ['[']
+ # add remote replica metadata to the request
+ self._prepare(
+ '', entries,
+ last_known_generation=last_known_generation,
+ last_known_trans_id=last_known_trans_id,
+ sync_id=sync_id,
+ ensure=self._ensure_callback is not None)
+ # inform server of how many documents have already been received
+ self._prepare(
+ ',', entries, received=received)
+ entries.append('\r\n]')
+ # send headers
+ return self._http_request(
+ self._url,
+ method='POST',
+ headers=headers,
+ body=''.join(entries))
+
+ def _insert_received_doc(self, idx, total, response):
+ """
+ Insert a received document into the local replica.
+
+ :param idx: The index count of the current operation.
+ :type idx: int
+ :param total: The total number of operations.
+ :type total: int
+ :param response: The body and headers of the response.
+ :type response: tuple(str, dict)
+ """
+ new_generation, new_transaction_id, number_of_changes, doc_id, \
+ rev, content, gen, trans_id = \
+ self._parse_received_doc_response(response)
+ if doc_id is not None:
+ # decrypt incoming document and insert into local database
+ # -------------------------------------------------------------
+ # symmetric decryption of document's contents
+ # -------------------------------------------------------------
+ # If arriving content was symmetrically encrypted, we decrypt it.
+ # We do it inline if defer_decryption flag is False or no sync_db
+ # was defined, otherwise we defer it writing it to the received
+ # docs table.
+ doc = SoledadDocument(doc_id, rev, content)
+ if is_symmetrically_encrypted(doc):
+ if self._queue_for_decrypt:
+ self._sync_decr_pool.insert_encrypted_received_doc(
+ doc.doc_id, doc.rev, doc.content, gen, trans_id,
+ idx)
+ else:
+ # defer_decryption is False or no-sync-db fallback
+ doc.set_json(decrypt_doc(self._crypto, doc))
+ self._insert_doc_cb(doc, gen, trans_id)
+ else:
+ # not symmetrically encrypted doc, insert it directly
+ # or save it in the decrypted stage.
+ if self._queue_for_decrypt:
+ self._sync_decr_pool.insert_received_doc(
+ doc.doc_id, doc.rev, doc.content, gen, trans_id,
+ idx)
+ else:
+ self._insert_doc_cb(doc, gen, trans_id)
+ # -------------------------------------------------------------
+ # end of symmetric decryption
+ # -------------------------------------------------------------
+ msg = "%d/%d" % (idx, total)
+ emit(SOLEDAD_SYNC_RECEIVE_STATUS, msg)
+ logger.debug("Soledad sync receive status: %s" % msg)
+ return number_of_changes, new_generation, new_transaction_id
+
+ def _parse_received_doc_response(self, response):
+ """
+ Parse the response from the server containing the received document.
+
+ :param response: The body and headers of the response.
+ :type response: tuple(str, dict)
+
+ :return: (new_gen, new_trans_id, number_of_changes, doc_id, rev,
+ content, gen, trans_id)
+ :rtype: tuple
+ """
+ # decode incoming stream
+ parts = response.splitlines()
+ if not parts or parts[0] != '[' or parts[-1] != ']':
+ raise errors.BrokenSyncStream
+ data = parts[1:-1]
+ # decode metadata
+ line, comma = utils.check_and_strip_comma(data[0])
+ metadata = None
+ try:
+ metadata = json.loads(line)
+ new_generation = metadata['new_generation']
+ new_transaction_id = metadata['new_transaction_id']
+ number_of_changes = metadata['number_of_changes']
+ except (json.JSONDecodeError, KeyError):
+ raise errors.BrokenSyncStream
+ # make sure we have replica_uid from fresh new dbs
+ if self._ensure_callback and 'replica_uid' in metadata:
+ self._ensure_callback(metadata['replica_uid'])
+ # parse incoming document info
+ doc_id = None
+ rev = None
+ content = None
+ gen = None
+ trans_id = None
+ if number_of_changes > 0:
+ try:
+ entry = json.loads(data[1])
+ doc_id = entry['id']
+ rev = entry['rev']
+ content = entry['content']
+ gen = entry['gen']
+ trans_id = entry['trans_id']
+ except (IndexError, KeyError):
+ raise errors.BrokenSyncStream
+ return new_generation, new_transaction_id, number_of_changes, \
+ doc_id, rev, content, gen, trans_id
+
+ def _setup_sync_decr_pool(self):
+ """
+ Set up the SyncDecrypterPool for deferred decryption.
+ """
+ if self._sync_decr_pool is None and self._sync_db is not None:
+ # initialize syncing queue decryption pool
+ self._sync_decr_pool = SyncDecrypterPool(
+ self._crypto,
+ self._sync_db,
+ insert_doc_cb=self._insert_doc_cb,
+ source_replica_uid=self.source_replica_uid)
+
+ def _http_request(self, url, method='GET', body=None, headers={}):
+ d = self._http.request(url, method, body, headers)
+ d.addErrback(_unauth_to_invalid_token_error)
+ return d
+
+
+def _unauth_to_invalid_token_error(failure):
+ """
+ An errback to translate unauthorized errors to our own invalid token
+ class.
+
+ :param failure: The original failure.
+ :type failure: twisted.python.failure.Failure
+
+ :return: Either the original failure or an invalid auth token error.
+ :rtype: twisted.python.failure.Failure
+ """
+ failure.trap(Error)
+ if failure.getErrorMessage() == "401 Unauthorized":
+ raise InvalidAuthTokenError
+ return failure
diff --git a/client/src/leap/soledad/client/interfaces.py b/client/src/leap/soledad/client/interfaces.py
new file mode 100644
index 00000000..4f7b0779
--- /dev/null
+++ b/client/src/leap/soledad/client/interfaces.py
@@ -0,0 +1,362 @@
+# -*- coding: utf-8 -*-
+# interfaces.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Interfaces used by the Soledad Client.
+"""
+from zope.interface import Interface, Attribute
+
+
+class ILocalStorage(Interface):
+ """
+ I implement core methods for the u1db local storage of documents and
+ indexes.
+ """
+ local_db_path = Attribute(
+ "The path for the local database replica")
+ local_db_file_name = Attribute(
+ "The name of the local SQLCipher U1DB database file")
+ uuid = Attribute("The user uuid")
+ default_prefix = Attribute(
+ "Prefix for default values for path")
+
+ def put_doc(self, doc):
+ """
+ Update a document in the local encrypted database.
+
+ :param doc: the document to update
+ :type doc: SoledadDocument
+
+ :return:
+ a deferred that will fire with the new revision identifier for
+ the document
+ :rtype: Deferred
+ """
+
+ def delete_doc(self, doc):
+ """
+ Delete a document from the local encrypted database.
+
+ :param doc: the document to delete
+ :type doc: SoledadDocument
+
+ :return:
+ a deferred that will fire with ...
+ :rtype: Deferred
+ """
+
+ def get_doc(self, doc_id, include_deleted=False):
+ """
+ Retrieve a document from the local encrypted database.
+
+ :param doc_id: the unique document identifier
+ :type doc_id: str
+ :param include_deleted:
+ if True, deleted documents will be returned with empty content;
+ otherwise asking for a deleted document will return None
+ :type include_deleted: bool
+
+ :return:
+ A deferred that will fire with the document object, containing a
+ SoledadDocument, or None if it could not be found
+ :rtype: Deferred
+ """
+
+ def get_docs(self, doc_ids, check_for_conflicts=True,
+ include_deleted=False):
+ """
+ Get the content for many documents.
+
+ :param doc_ids: a list of document identifiers
+ :type doc_ids: list
+ :param check_for_conflicts: if set False, then the conflict check will
+ be skipped, and 'None' will be returned instead of True/False
+ :type check_for_conflicts: bool
+
+ :return:
+ A deferred that will fire with an iterable giving the Document
+ object for each document id in matching doc_ids order.
+ :rtype: Deferred
+ """
+
+ def get_all_docs(self, include_deleted=False):
+ """
+ Get the JSON content for all documents in the database.
+
+ :param include_deleted: If set to True, deleted documents will be
+ returned with empty content. Otherwise deleted
+ documents will not be included in the results.
+ :return:
+ A deferred that will fire with (generation, [Document]): that is,
+ the current generation of the database, followed by a list of all
+ the documents in the database.
+ :rtype: Deferred
+ """
+
+ def create_doc(self, content, doc_id=None):
+ """
+ Create a new document in the local encrypted database.
+
+ :param content: the contents of the new document
+ :type content: dict
+ :param doc_id: an optional identifier specifying the document id
+ :type doc_id: str
+
+ :return:
+ A deferred tht will fire with the new document (SoledadDocument
+ instance).
+ :rtype: Deferred
+ """
+
+ def create_doc_from_json(self, json, doc_id=None):
+ """
+ Create a new document.
+
+ You can optionally specify the document identifier, but the document
+ must not already exist. See 'put_doc' if you want to override an
+ existing document.
+ If the database specifies a maximum document size and the document
+ exceeds it, create will fail and raise a DocumentTooBig exception.
+
+ :param json: The JSON document string
+ :type json: str
+ :param doc_id: An optional identifier specifying the document id.
+ :type doc_id:
+ :return:
+ A deferred that will fire with the new document (A SoledadDocument
+ instance)
+ :rtype: Deferred
+ """
+
+ def create_index(self, index_name, *index_expressions):
+ """
+ Create an named index, which can then be queried for future lookups.
+ Creating an index which already exists is not an error, and is cheap.
+ Creating an index which does not match the index_expressions of the
+ existing index is an error.
+ Creating an index will block until the expressions have been evaluated
+ and the index generated.
+
+ :param index_name: A unique name which can be used as a key prefix
+ :type index_name: str
+ :param index_expressions:
+ index expressions defining the index information.
+ :type index_expressions: dict
+
+ Examples:
+
+ "fieldname", or "fieldname.subfieldname" to index alphabetically
+ sorted on the contents of a field.
+
+ "number(fieldname, width)", "lower(fieldname)"
+ """
+
+ def delete_index(self, index_name):
+ """
+ Remove a named index.
+
+ :param index_name: The name of the index we are removing
+ :type index_name: str
+ """
+
+ def list_indexes(self):
+ """
+ List the definitions of all known indexes.
+
+ :return: A list of [('index-name', ['field', 'field2'])] definitions.
+ :rtype: Deferred
+ """
+
+ def get_from_index(self, index_name, *key_values):
+ """
+ Return documents that match the keys supplied.
+
+ You must supply exactly the same number of values as have been defined
+ in the index. It is possible to do a prefix match by using '*' to
+ indicate a wildcard match. You can only supply '*' to trailing entries,
+ (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.)
+ It is also possible to append a '*' to the last supplied value (eg
+ 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*')
+
+ :param index_name: The index to query
+ :type index_name: str
+ :param key_values: values to match. eg, if you have
+ an index with 3 fields then you would have:
+ get_from_index(index_name, val1, val2, val3)
+ :type key_values: tuple
+ :return: List of [Document]
+ :rtype: list
+ """
+
+ def get_count_from_index(self, index_name, *key_values):
+ """
+ Return the count of the documents that match the keys and
+ values supplied.
+
+ :param index_name: The index to query
+ :type index_name: str
+ :param key_values: values to match. eg, if you have
+ an index with 3 fields then you would have:
+ get_from_index(index_name, val1, val2, val3)
+ :type key_values: tuple
+ :return: count.
+ :rtype: int
+ """
+
+ def get_range_from_index(self, index_name, start_value, end_value):
+ """
+ Return documents that fall within the specified range.
+
+ Both ends of the range are inclusive. For both start_value and
+ end_value, one must supply exactly the same number of values as have
+ been defined in the index, or pass None. In case of a single column
+ index, a string is accepted as an alternative for a tuple with a single
+ value. It is possible to do a prefix match by using '*' to indicate
+ a wildcard match. You can only supply '*' to trailing entries, (eg
+ 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also
+ possible to append a '*' to the last supplied value (eg 'val*', '*',
+ '*' or 'val', 'val*', '*', but not 'val*', 'val', '*')
+
+ :param index_name: The index to query
+ :type index_name: str
+ :param start_values: tuples of values that define the lower bound of
+ the range. eg, if you have an index with 3 fields then you would
+ have: (val1, val2, val3)
+ :type start_values: tuple
+ :param end_values: tuples of values that define the upper bound of the
+ range. eg, if you have an index with 3 fields then you would have:
+ (val1, val2, val3)
+ :type end_values: tuple
+ :return: A deferred that will fire with a list of [Document]
+ :rtype: Deferred
+ """
+
+ def get_index_keys(self, index_name):
+ """
+ Return all keys under which documents are indexed in this index.
+
+ :param index_name: The index to query
+ :type index_name: str
+ :return:
+ A deferred that will fire with a list of tuples of indexed keys.
+ :rtype: Deferred
+ """
+
+ def get_doc_conflicts(self, doc_id):
+ """
+ Get the list of conflicts for the given document.
+
+ :param doc_id: the document id
+ :type doc_id: str
+
+ :return:
+ A deferred that will fire with a list of the document entries that
+ are conflicted.
+ :rtype: Deferred
+ """
+
+ def resolve_doc(self, doc, conflicted_doc_revs):
+ """
+ Mark a document as no longer conflicted.
+
+ :param doc: a document with the new content to be inserted.
+ :type doc: SoledadDocument
+ :param conflicted_doc_revs:
+ A deferred that will fire with a list of revisions that the new
+ content supersedes.
+ :type conflicted_doc_revs: list
+ """
+
+
+class ISyncableStorage(Interface):
+ """
+ I implement methods to synchronize with a remote replica.
+ """
+ replica_uid = Attribute("The uid of the local replica")
+ syncing = Attribute(
+ "Property, True if the syncer is syncing.")
+ token = Attribute("The authentication Token.")
+
+ def sync(self, defer_decryption=True):
+ """
+ Synchronize the local encrypted replica with a remote replica.
+
+ This method blocks until a syncing lock is acquired, so there are no
+ attempts of concurrent syncs from the same client replica.
+
+ :param url: the url of the target replica to sync with
+ :type url: str
+
+ :param defer_decryption:
+ Whether to defer the decryption process using the intermediate
+ database. If False, decryption will be done inline.
+ :type defer_decryption: bool
+
+ :return:
+ A deferred that will fire with the local generation before the
+ synchronisation was performed.
+ :rtype: str
+ """
+
+ def stop_sync(self):
+ """
+ Stop the current syncing process.
+ """
+
+
+class ISecretsStorage(Interface):
+ """
+ I implement methods needed for initializing and accessing secrets, that are
+ synced against the Shared Recovery Database.
+ """
+ secrets_file_name = Attribute(
+ "The name of the file where the storage secrets will be stored")
+
+ storage_secret = Attribute("")
+ remote_storage_secret = Attribute("")
+ shared_db = Attribute("The shared db object")
+
+ # XXX this used internally from secrets, so it might be good to preserve
+ # as a public boundary with other components.
+
+ # We should also probably document its interface.
+ secrets = Attribute("A SoledadSecrets object containing access to secrets")
+
+ def init_shared_db(self, server_url, uuid, creds):
+ """
+ Initialize the shared recovery database.
+
+ :param server_url:
+ :type server_url:
+ :param uuid:
+ :type uuid:
+ :param creds:
+ :type creds:
+ """
+
+ def change_passphrase(self, new_passphrase):
+ """
+ Change the passphrase that encrypts the storage secret.
+
+ :param new_passphrase: The new passphrase.
+ :type new_passphrase: unicode
+
+ :raise NoStorageSecret: Raised if there's no storage secret available.
+ """
+
+ # XXX not in use. Uncomment if we ever decide to allow
+ # multiple secrets.
+ # secret_id = Attribute("The id of the storage secret to be used")
diff --git a/client/src/leap/soledad/client/pragmas.py b/client/src/leap/soledad/client/pragmas.py
new file mode 100644
index 00000000..55397d10
--- /dev/null
+++ b/client/src/leap/soledad/client/pragmas.py
@@ -0,0 +1,379 @@
+# -*- coding: utf-8 -*-
+# pragmas.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Different pragmas used in the initialization of the SQLCipher database.
+"""
+import logging
+import string
+import threading
+import os
+
+from leap.soledad.common import soledad_assert
+
+
+logger = logging.getLogger(__name__)
+
+
+_db_init_lock = threading.Lock()
+
+
+def set_init_pragmas(conn, opts=None, extra_queries=None):
+ """
+ Set the initialization pragmas.
+
+ This includes the crypto pragmas, and any other options that must
+ be passed early to sqlcipher db.
+ """
+ soledad_assert(opts is not None)
+ extra_queries = [] if extra_queries is None else extra_queries
+ with _db_init_lock:
+ # only one execution path should initialize the db
+ _set_init_pragmas(conn, opts, extra_queries)
+
+
+def _set_init_pragmas(conn, opts, extra_queries):
+
+ sync_off = os.environ.get('LEAP_SQLITE_NOSYNC')
+ memstore = os.environ.get('LEAP_SQLITE_MEMSTORE')
+ nowal = os.environ.get('LEAP_SQLITE_NOWAL')
+
+ set_crypto_pragmas(conn, opts)
+
+ if not nowal:
+ set_write_ahead_logging(conn)
+ if sync_off:
+ set_synchronous_off(conn)
+ else:
+ set_synchronous_normal(conn)
+ if memstore:
+ set_mem_temp_store(conn)
+
+ for query in extra_queries:
+ conn.cursor().execute(query)
+
+
+def set_crypto_pragmas(db_handle, sqlcipher_opts):
+ """
+ Set cryptographic params (key, cipher, KDF number of iterations and
+ cipher page size).
+
+ :param db_handle:
+ :type db_handle:
+ :param sqlcipher_opts: options for the SQLCipherDatabase
+ :type sqlcipher_opts: SQLCipherOpts instance
+ """
+ # XXX assert CryptoOptions
+ opts = sqlcipher_opts
+ _set_key(db_handle, opts.key, opts.is_raw_key)
+ _set_cipher(db_handle, opts.cipher)
+ _set_kdf_iter(db_handle, opts.kdf_iter)
+ _set_cipher_page_size(db_handle, opts.cipher_page_size)
+
+
+def _set_key(db_handle, key, is_raw_key):
+ """
+ Set the ``key`` for use with the database.
+
+ The process of creating a new, encrypted database is called 'keying'
+ the database. SQLCipher uses just-in-time key derivation at the point
+ it is first needed for an operation. This means that the key (and any
+ options) must be set before the first operation on the database. As
+ soon as the database is touched (e.g. SELECT, CREATE TABLE, UPDATE,
+ etc.) and pages need to be read or written, the key is prepared for
+ use.
+
+ Implementation Notes:
+
+ * PRAGMA key should generally be called as the first operation on a
+ database.
+
+ :param key: The key for use with the database.
+ :type key: str
+ :param is_raw_key:
+ Whether C{key} is a raw 64-char hex string or a passphrase that should
+ be hashed to obtain the encyrption key.
+ :type is_raw_key: bool
+ """
+ if is_raw_key:
+ _set_key_raw(db_handle, key)
+ else:
+ _set_key_passphrase(db_handle, key)
+
+
+def _set_key_passphrase(db_handle, passphrase):
+ """
+ Set a passphrase for encryption key derivation.
+
+ The key itself can be a passphrase, which is converted to a key using
+ PBKDF2 key derivation. The result is used as the encryption key for
+ the database. By using this method, there is no way to alter the KDF;
+ if you want to do so you should use a raw key instead and derive the
+ key using your own KDF.
+
+ :param db_handle: A handle to the SQLCipher database.
+ :type db_handle: pysqlcipher.Connection
+ :param passphrase: The passphrase used to derive the encryption key.
+ :type passphrase: str
+ """
+ db_handle.cursor().execute("PRAGMA key = '%s'" % passphrase)
+
+
+def _set_key_raw(db_handle, key):
+ """
+ Set a raw hexadecimal encryption key.
+
+ It is possible to specify an exact byte sequence using a blob literal.
+ With this method, it is the calling application's responsibility to
+ ensure that the data provided is a 64 character hex string, which will
+ be converted directly to 32 bytes (256 bits) of key data.
+
+ :param db_handle: A handle to the SQLCipher database.
+ :type db_handle: pysqlcipher.Connection
+ :param key: A 64 character hex string.
+ :type key: str
+ """
+ if not all(c in string.hexdigits for c in key):
+ raise NotAnHexString(key)
+ db_handle.cursor().execute('PRAGMA key = "x\'%s"' % key)
+
+
+def _set_cipher(db_handle, cipher='aes-256-cbc'):
+ """
+ Set the cipher and mode to use for symmetric encryption.
+
+ SQLCipher uses aes-256-cbc as the default cipher and mode of
+ operation. It is possible to change this, though not generally
+ recommended, using PRAGMA cipher.
+
+ SQLCipher makes direct use of libssl, so all cipher options available
+ to libssl are also available for use with SQLCipher. See `man enc` for
+ OpenSSL's supported ciphers.
+
+ Implementation Notes:
+
+ * PRAGMA cipher must be called after PRAGMA key and before the first
+ actual database operation or it will have no effect.
+
+ * If a non-default value is used PRAGMA cipher to create a database,
+ it must also be called every time that database is opened.
+
+ * SQLCipher does not implement its own encryption. Instead it uses the
+ widely available and peer-reviewed OpenSSL libcrypto for all
+ cryptographic functions.
+
+ :param db_handle: A handle to the SQLCipher database.
+ :type db_handle: pysqlcipher.Connection
+ :param cipher: The cipher and mode to use.
+ :type cipher: str
+ """
+ db_handle.cursor().execute("PRAGMA cipher = '%s'" % cipher)
+
+
+def _set_kdf_iter(db_handle, kdf_iter=4000):
+ """
+ Set the number of iterations for the key derivation function.
+
+ SQLCipher uses PBKDF2 key derivation to strengthen the key and make it
+ resistent to brute force and dictionary attacks. The default
+ configuration uses 4000 PBKDF2 iterations (effectively 16,000 SHA1
+ operations). PRAGMA kdf_iter can be used to increase or decrease the
+ number of iterations used.
+
+ Implementation Notes:
+
+ * PRAGMA kdf_iter must be called after PRAGMA key and before the first
+ actual database operation or it will have no effect.
+
+ * If a non-default value is used PRAGMA kdf_iter to create a database,
+ it must also be called every time that database is opened.
+
+ * It is not recommended to reduce the number of iterations if a
+ passphrase is in use.
+
+ :param db_handle: A handle to the SQLCipher database.
+ :type db_handle: pysqlcipher.Connection
+ :param kdf_iter: The number of iterations to use.
+ :type kdf_iter: int
+ """
+ db_handle.cursor().execute("PRAGMA kdf_iter = '%d'" % kdf_iter)
+
+
+def _set_cipher_page_size(db_handle, cipher_page_size=1024):
+ """
+ Set the page size of the encrypted database.
+
+ SQLCipher 2 introduced the new PRAGMA cipher_page_size that can be
+ used to adjust the page size for the encrypted database. The default
+ page size is 1024 bytes, but it can be desirable for some applications
+ to use a larger page size for increased performance. For instance,
+ some recent testing shows that increasing the page size can noticeably
+ improve performance (5-30%) for certain queries that manipulate a
+ large number of pages (e.g. selects without an index, large inserts in
+ a transaction, big deletes).
+
+ To adjust the page size, call the pragma immediately after setting the
+ key for the first time and each subsequent time that you open the
+ database.
+
+ Implementation Notes:
+
+ * PRAGMA cipher_page_size must be called after PRAGMA key and before
+ the first actual database operation or it will have no effect.
+
+ * If a non-default value is used PRAGMA cipher_page_size to create a
+ database, it must also be called every time that database is opened.
+
+ :param db_handle: A handle to the SQLCipher database.
+ :type db_handle: pysqlcipher.Connection
+ :param cipher_page_size: The page size.
+ :type cipher_page_size: int
+ """
+ db_handle.cursor().execute(
+ "PRAGMA cipher_page_size = '%d'" % cipher_page_size)
+
+
+# XXX UNUSED ?
+def set_rekey(db_handle, new_key, is_raw_key):
+ """
+ Change the key of an existing encrypted database.
+
+ To change the key on an existing encrypted database, it must first be
+ unlocked with the current encryption key. Once the database is
+ readable and writeable, PRAGMA rekey can be used to re-encrypt every
+ page in the database with a new key.
+
+ * PRAGMA rekey must be called after PRAGMA key. It can be called at any
+ time once the database is readable.
+
+ * PRAGMA rekey can not be used to encrypted a standard SQLite
+ database! It is only useful for changing the key on an existing
+ database.
+
+ * Previous versions of SQLCipher provided a PRAGMA rekey_cipher and
+ code>PRAGMA rekey_kdf_iter. These are deprecated and should not be
+ used. Instead, use sqlcipher_export().
+
+ :param db_handle: A handle to the SQLCipher database.
+ :type db_handle: pysqlcipher.Connection
+ :param new_key: The new key.
+ :type new_key: str
+ :param is_raw_key: Whether C{password} is a raw 64-char hex string or a
+ passphrase that should be hashed to obtain the encyrption
+ key.
+ :type is_raw_key: bool
+ """
+ if is_raw_key:
+ _set_rekey_raw(db_handle, new_key)
+ else:
+ _set_rekey_passphrase(db_handle, new_key)
+
+
+def _set_rekey_passphrase(db_handle, passphrase):
+ """
+ Change the passphrase for encryption key derivation.
+
+ The key itself can be a passphrase, which is converted to a key using
+ PBKDF2 key derivation. The result is used as the encryption key for
+ the database.
+
+ :param db_handle: A handle to the SQLCipher database.
+ :type db_handle: pysqlcipher.Connection
+ :param passphrase: The passphrase used to derive the encryption key.
+ :type passphrase: str
+ """
+ db_handle.cursor().execute("PRAGMA rekey = '%s'" % passphrase)
+
+
+def _set_rekey_raw(db_handle, key):
+ """
+ Change the raw hexadecimal encryption key.
+
+ It is possible to specify an exact byte sequence using a blob literal.
+ With this method, it is the calling application's responsibility to
+ ensure that the data provided is a 64 character hex string, which will
+ be converted directly to 32 bytes (256 bits) of key data.
+
+ :param db_handle: A handle to the SQLCipher database.
+ :type db_handle: pysqlcipher.Connection
+ :param key: A 64 character hex string.
+ :type key: str
+ """
+ if not all(c in string.hexdigits for c in key):
+ raise NotAnHexString(key)
+ db_handle.cursor().execute('PRAGMA rekey = "x\'%s"' % key)
+
+
+def set_synchronous_off(db_handle):
+ """
+ Change the setting of the "synchronous" flag to OFF.
+ """
+ logger.debug("SQLCIPHER: SETTING SYNCHRONOUS OFF")
+ db_handle.cursor().execute('PRAGMA synchronous=OFF')
+
+
+def set_synchronous_normal(db_handle):
+ """
+ Change the setting of the "synchronous" flag to NORMAL.
+ """
+ logger.debug("SQLCIPHER: SETTING SYNCHRONOUS NORMAL")
+ db_handle.cursor().execute('PRAGMA synchronous=NORMAL')
+
+
+def set_mem_temp_store(db_handle):
+ """
+ Use a in-memory store for temporary tables.
+ """
+ logger.debug("SQLCIPHER: SETTING TEMP_STORE MEMORY")
+ db_handle.cursor().execute('PRAGMA temp_store=MEMORY')
+
+
+def set_write_ahead_logging(db_handle):
+ """
+ Enable write-ahead logging, and set the autocheckpoint to 50 pages.
+
+ Setting the autocheckpoint to a small value, we make the reads not
+ suffer too much performance degradation.
+
+ From the sqlite docs:
+
+ "There is a tradeoff between average read performance and average write
+ performance. To maximize the read performance, one wants to keep the
+ WAL as small as possible and hence run checkpoints frequently, perhaps
+ as often as every COMMIT. To maximize write performance, one wants to
+ amortize the cost of each checkpoint over as many writes as possible,
+ meaning that one wants to run checkpoints infrequently and let the WAL
+ grow as large as possible before each checkpoint. The decision of how
+ often to run checkpoints may therefore vary from one application to
+ another depending on the relative read and write performance
+ requirements of the application. The default strategy is to run a
+ checkpoint once the WAL reaches 1000 pages"
+ """
+ logger.debug("SQLCIPHER: SETTING WRITE-AHEAD LOGGING")
+ db_handle.cursor().execute('PRAGMA journal_mode=WAL')
+
+ # The optimum value can still use a little bit of tuning, but we favor
+ # small sizes of the WAL file to get fast reads, since we assume that
+ # the writes will be quick enough to not block too much.
+
+ db_handle.cursor().execute('PRAGMA wal_autocheckpoint=50')
+
+
+class NotAnHexString(Exception):
+ """
+ Raised when trying to (raw) key the database with a non-hex string.
+ """
+ pass
diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py
new file mode 100644
index 00000000..e89e21aa
--- /dev/null
+++ b/client/src/leap/soledad/client/secrets.py
@@ -0,0 +1,787 @@
+# -*- coding: utf-8 -*-
+# secrets.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Soledad secrets handling.
+"""
+
+
+import os
+import scrypt
+import hmac
+import logging
+import binascii
+import errno
+
+
+from hashlib import sha256
+import simplejson as json
+
+
+from leap.soledad.common import soledad_assert
+from leap.soledad.common import soledad_assert_type
+from leap.soledad.common import document
+from leap.soledad.common import errors
+from leap.soledad.common import crypto
+from leap.soledad.client import events
+
+
+logger = logging.getLogger(name=__name__)
+
+
+#
+# Exceptions
+#
+
+
+class SecretsException(Exception):
+ """
+ Generic exception type raised by this module.
+ """
+
+
+class NoStorageSecret(SecretsException):
+ """
+ Raised when trying to use a storage secret but none is available.
+ """
+ pass
+
+
+class PassphraseTooShort(SecretsException):
+ """
+ Raised when trying to change the passphrase but the provided passphrase is
+ too short.
+ """
+
+
+class BootstrapSequenceError(SecretsException):
+ """
+ Raised when an attempt to generate a secret and store it in a recovery
+ document on server failed.
+ """
+
+
+#
+# Secrets handler
+#
+
+class SoledadSecrets(object):
+ """
+ Soledad secrets handler.
+
+ The first C{self.REMOTE_STORAGE_SECRET_LENGTH} bytes of the storage
+ secret are used for remote storage encryption. We use the next
+ C{self.LOCAL_STORAGE_SECRET} bytes to derive a key for local storage.
+ From these bytes, the first C{self.SALT_LENGTH} bytes are used as the
+ salt and the rest as the password for the scrypt hashing.
+ """
+
+ LOCAL_STORAGE_SECRET_LENGTH = 512
+ """
+ The length, in bytes, of the secret used to derive a passphrase for the
+ SQLCipher database.
+ """
+
+ REMOTE_STORAGE_SECRET_LENGTH = 512
+ """
+ The length, in bytes, of the secret used to derive an encryption key and a
+ MAC auth key for remote storage.
+ """
+
+ SALT_LENGTH = 64
+ """
+ The length, in bytes, of the salt used to derive the key for the storage
+ secret encryption.
+ """
+
+ GEN_SECRET_LENGTH = LOCAL_STORAGE_SECRET_LENGTH \
+ + REMOTE_STORAGE_SECRET_LENGTH \
+ + SALT_LENGTH # for sync db
+ """
+ The length, in bytes, of the secret to be generated. This includes local
+ and remote secrets, and the salt for deriving the sync db secret.
+ """
+
+ MINIMUM_PASSPHRASE_LENGTH = 6
+ """
+ The minimum length, in bytes, for a passphrase. The passphrase length is
+ only checked when the user changes her passphrase, not when she
+ instantiates Soledad.
+ """
+
+ IV_SEPARATOR = ":"
+ """
+ A separator used for storing the encryption initial value prepended to the
+ ciphertext.
+ """
+
+ UUID_KEY = 'uuid'
+ STORAGE_SECRETS_KEY = 'storage_secrets'
+ ACTIVE_SECRET_KEY = 'active_secret'
+ SECRET_KEY = 'secret'
+ CIPHER_KEY = 'cipher'
+ LENGTH_KEY = 'length'
+ KDF_KEY = 'kdf'
+ KDF_SALT_KEY = 'kdf_salt'
+ KDF_LENGTH_KEY = 'kdf_length'
+ KDF_SCRYPT = 'scrypt'
+ CIPHER_AES256 = 'aes256'
+ """
+ Keys used to access storage secrets in recovery documents.
+ """
+
+ def __init__(self, uuid, passphrase, secrets_path, shared_db, crypto):
+ """
+ Initialize the secrets manager.
+
+ :param uuid: User's unique id.
+ :type uuid: str
+ :param passphrase: The passphrase for locking and unlocking encryption
+ secrets for local and remote storage.
+ :type passphrase: unicode
+ :param secrets_path: Path for storing encrypted key used for
+ symmetric encryption.
+ :type secrets_path: str
+ :param shared_db: The shared database that stores user secrets.
+ :type shared_db: leap.soledad.client.shared_db.SoledadSharedDatabase
+ :param crypto: A soledad crypto object.
+ :type crypto: SoledadCrypto
+ """
+ # XXX removed since not in use
+ # We will pick the first secret available.
+ # param secret_id: The id of the storage secret to be used.
+
+ self._uuid = uuid
+ self._passphrase = passphrase
+ self._secrets_path = secrets_path
+ self._shared_db = shared_db
+ self._crypto = crypto
+ self._secrets = {}
+
+ self._secret_id = None
+
+ def bootstrap(self):
+ """
+ Bootstrap secrets.
+
+ Soledad secrets bootstrap is the following sequence of stages:
+
+ * stage 1 - local secret loading:
+ - if secrets exist locally, load them.
+ * stage 2 - remote secret loading:
+ - else, if secrets exist in server, download them.
+ * stage 3 - secret generation:
+ - else, generate a new secret and store in server.
+
+ This method decides which bootstrap stages have already been performed
+ and performs the missing ones in order.
+
+ :raise BootstrapSequenceError: Raised when the secret generation and
+ storage on server sequence has failed for some reason.
+ """
+ # STAGE 1 - verify if secrets exist locally
+ if not self._has_secret(): # try to load from local storage.
+
+ # STAGE 2 - there are no secrets in local storage, so try to fetch
+ # encrypted secrets from server.
+ logger.info(
+ 'Trying to fetch cryptographic secrets from shared recovery '
+ 'database...')
+
+ # --- start of atomic operation in shared db ---
+
+ # obtain lock on shared db
+ token = timeout = None
+ try:
+ token, timeout = self._shared_db.lock()
+ except errors.AlreadyLockedError:
+ raise BootstrapSequenceError('Database is already locked.')
+ except errors.LockTimedOutError:
+ raise BootstrapSequenceError('Lock operation timed out.')
+
+ self._get_or_gen_crypto_secrets()
+
+ # release the lock on shared db
+ try:
+ self._shared_db.unlock(token)
+ self._shared_db.close()
+ except errors.NotLockedError:
+ # for some reason the lock expired. Despite that, secret
+ # loading or generation/storage must have been executed
+ # successfully, so we pass.
+ pass
+ except errors.InvalidTokenError:
+ # here, our lock has not only expired but also some other
+ # client application has obtained a new lock and is currently
+ # doing its thing in the shared database. Using the same
+ # reasoning as above, we assume everything went smooth and
+ # pass.
+ pass
+ except Exception as e:
+ logger.error("Unhandled exception when unlocking shared "
+ "database.")
+ logger.exception(e)
+
+ # --- end of atomic operation in shared db ---
+
+ def _has_secret(self):
+ """
+ Return whether there is a storage secret available for use or not.
+
+ :return: Whether there's a storage secret for symmetric encryption.
+ :rtype: bool
+ """
+ logger.info("Checking if there's a secret in local storage...")
+ if (self._secret_id is None or self._secret_id not in self._secrets) \
+ and os.path.isfile(self._secrets_path):
+ try:
+ self._load_secrets() # try to load from disk
+ except IOError as e:
+ logger.warning(
+ 'IOError while loading secrets from disk: %s' % str(e))
+
+ if self.storage_secret is not None:
+ logger.info("Found a secret in local storage.")
+ return True
+
+ logger.info("Could not find a secret in local storage.")
+ return False
+
+ def _load_secrets(self):
+ """
+ Load storage secrets from local file.
+ """
+ # read storage secrets from file
+ content = None
+ with open(self._secrets_path, 'r') as f:
+ content = json.loads(f.read())
+ _, mac, active_secret = self._import_recovery_document(content)
+ # choose first secret if no secret_id was given
+ if self._secret_id is None:
+ if active_secret is None:
+ self.set_secret_id(self._secrets.items()[0][0])
+ else:
+ self.set_secret_id(active_secret)
+ # enlarge secret if needed
+ enlarged = False
+ if len(self._secrets[self._secret_id]) < self.GEN_SECRET_LENGTH:
+ gen_len = self.GEN_SECRET_LENGTH \
+ - len(self._secrets[self._secret_id])
+ new_piece = os.urandom(gen_len)
+ self._secrets[self._secret_id] += new_piece
+ enlarged = True
+ # store and save in shared db if needed
+ if not mac or enlarged:
+ self._store_secrets()
+ self._put_secrets_in_shared_db()
+
+ def _get_or_gen_crypto_secrets(self):
+ """
+ Retrieves or generates the crypto secrets.
+
+ :raises BootstrapSequenceError: Raised when unable to store secrets in
+ shared database.
+ """
+ if self._shared_db.syncable:
+ doc = self._get_secrets_from_shared_db()
+ else:
+ doc = None
+
+ if doc is not None:
+ logger.info(
+ 'Found cryptographic secrets in shared recovery '
+ 'database.')
+ _, mac, active_secret = self._import_recovery_document(doc.content)
+ if mac is False:
+ self.put_secrets_in_shared_db()
+ self._store_secrets() # save new secrets in local file
+ if self._secret_id is None:
+ if active_secret is None:
+ self.set_secret_id(self._secrets.items()[0][0])
+ else:
+ self.set_secret_id(active_secret)
+ else:
+ # STAGE 3 - there are no secrets in server also, so
+ # generate a secret and store it in remote db.
+ logger.info(
+ 'No cryptographic secrets found, creating new '
+ ' secrets...')
+ self.set_secret_id(self._gen_secret())
+
+ if self._shared_db.syncable:
+ try:
+ self._put_secrets_in_shared_db()
+ except Exception as ex:
+ # storing generated secret in shared db failed for
+ # some reason, so we erase the generated secret and
+ # raise.
+ try:
+ os.unlink(self._secrets_path)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ # no such file or directory
+ logger.exception(e)
+ logger.exception(ex)
+ raise BootstrapSequenceError(
+ 'Could not store generated secret in the shared '
+ 'database, bailing out...')
+
+ #
+ # Shared DB related methods
+ #
+
+ def _shared_db_doc_id(self):
+ """
+ Calculate the doc_id of the document in the shared db that stores key
+ material.
+
+ :return: the hash
+ :rtype: str
+ """
+ return sha256(
+ '%s%s' %
+ (self._passphrase_as_string(), self._uuid)).hexdigest()
+
+ def _export_recovery_document(self):
+ """
+ Export the storage secrets.
+
+ A recovery document has the following structure:
+
+ {
+ 'storage_secrets': {
+ '<storage_secret id>': {
+ 'kdf': 'scrypt',
+ 'kdf_salt': '<b64 repr of salt>'
+ 'kdf_length': <key length>
+ 'cipher': 'aes256',
+ 'length': <secret length>,
+ 'secret': '<encrypted storage_secret>',
+ },
+ },
+ 'active_secret': '<secret_id>',
+ 'kdf': 'scrypt',
+ 'kdf_salt': '<b64 repr of salt>',
+ 'kdf_length: <key length>,
+ '_mac_method': 'hmac',
+ '_mac': '<mac>'
+ }
+
+ Note that multiple storage secrets might be stored in one recovery
+ document. This method will also calculate a MAC of a string
+ representation of the secrets dictionary.
+
+ :return: The recovery document.
+ :rtype: dict
+ """
+ # create salt and key for calculating MAC
+ salt = os.urandom(self.SALT_LENGTH)
+ key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
+ # encrypt secrets
+ encrypted_secrets = {}
+ for secret_id in self._secrets:
+ encrypted_secrets[secret_id] = self._encrypt_storage_secret(
+ self._secrets[secret_id])
+ # create the recovery document
+ data = {
+ self.STORAGE_SECRETS_KEY: encrypted_secrets,
+ self.ACTIVE_SECRET_KEY: self._secret_id,
+ self.KDF_KEY: self.KDF_SCRYPT,
+ self.KDF_SALT_KEY: binascii.b2a_base64(salt),
+ self.KDF_LENGTH_KEY: len(key),
+ crypto.MAC_METHOD_KEY: crypto.MacMethods.HMAC,
+ crypto.MAC_KEY: hmac.new(
+ key,
+ json.dumps(encrypted_secrets, sort_keys=True),
+ sha256).hexdigest(),
+ }
+ return data
+
+ def _import_recovery_document(self, data):
+ """
+ Import storage secrets for symmetric encryption and uuid (if present)
+ from a recovery document.
+
+ Note that this method does not store the imported data on disk. For
+ that, use C{self._store_secrets()}.
+
+ :param data: The recovery document.
+ :type data: dict
+
+ :return: A tuple containing the number of imported secrets, whether
+ there was MAC information available for authenticating, and
+ the secret_id of the last active secret.
+ :rtype: (int, bool)
+ """
+ soledad_assert(self.STORAGE_SECRETS_KEY in data)
+ # check mac of the recovery document
+ mac = None
+ if crypto.MAC_KEY in data:
+ soledad_assert(data[crypto.MAC_KEY] is not None)
+ soledad_assert(crypto.MAC_METHOD_KEY in data)
+ soledad_assert(self.KDF_KEY in data)
+ soledad_assert(self.KDF_SALT_KEY in data)
+ soledad_assert(self.KDF_LENGTH_KEY in data)
+ if data[crypto.MAC_METHOD_KEY] == crypto.MacMethods.HMAC:
+ key = scrypt.hash(
+ self._passphrase_as_string(),
+ binascii.a2b_base64(data[self.KDF_SALT_KEY]),
+ buflen=32)
+ mac = hmac.new(
+ key,
+ json.dumps(
+ data[self.STORAGE_SECRETS_KEY], sort_keys=True),
+ sha256).hexdigest()
+ else:
+ raise crypto.UnknownMacMethodError('Unknown MAC method: %s.' %
+ data[crypto.MAC_METHOD_KEY])
+ if mac != data[crypto.MAC_KEY]:
+ raise crypto.WrongMacError('Could not authenticate recovery document\'s '
+ 'contents.')
+ # include secrets in the secret pool.
+ secret_count = 0
+ secrets = data[self.STORAGE_SECRETS_KEY].items()
+ active_secret = None
+ # XXX remove check for existence of key (included for backwards
+ # compatibility)
+ if self.ACTIVE_SECRET_KEY in data:
+ active_secret = data[self.ACTIVE_SECRET_KEY]
+ for secret_id, encrypted_secret in secrets:
+ if secret_id not in self._secrets:
+ try:
+ self._secrets[secret_id] = \
+ self._decrypt_storage_secret(encrypted_secret)
+ secret_count += 1
+ except SecretsException as e:
+ logger.error("Failed to decrypt storage secret: %s"
+ % str(e))
+ return secret_count, mac, active_secret
+
+ def _get_secrets_from_shared_db(self):
+ """
+ Retrieve the document with encrypted key material from the shared
+ database.
+
+ :return: a document with encrypted key material in its contents
+ :rtype: document.SoledadDocument
+ """
+ events.emit(events.SOLEDAD_DOWNLOADING_KEYS, self._uuid)
+ db = self._shared_db
+ if not db:
+ logger.warning('No shared db found')
+ return
+ doc = db.get_doc(self._shared_db_doc_id())
+ events.emit(events.SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid)
+ return doc
+
+ def _put_secrets_in_shared_db(self):
+ """
+ Assert local keys are the same as shared db's ones.
+
+ Try to fetch keys from shared recovery database. If they already exist
+ in the remote db, assert that that data is the same as local data.
+ Otherwise, upload keys to shared recovery database.
+ """
+ soledad_assert(
+ self._has_secret(),
+ 'Tried to send keys to server but they don\'t exist in local '
+ 'storage.')
+ # try to get secrets doc from server, otherwise create it
+ doc = self._get_secrets_from_shared_db()
+ if doc is None:
+ doc = document.SoledadDocument(
+ doc_id=self._shared_db_doc_id())
+ # fill doc with encrypted secrets
+ doc.content = self._export_recovery_document()
+ # upload secrets to server
+ events.emit(events.SOLEDAD_UPLOADING_KEYS, self._uuid)
+ db = self._shared_db
+ if not db:
+ logger.warning('No shared db found')
+ return
+ db.put_doc(doc)
+ events.emit(events.SOLEDAD_DONE_UPLOADING_KEYS, self._uuid)
+
+ #
+ # Management of secret for symmetric encryption.
+ #
+
+ def _decrypt_storage_secret(self, encrypted_secret_dict):
+ """
+ Decrypt the storage secret.
+
+ Storage secret is encrypted before being stored. This method decrypts
+ and returns the decrypted storage secret.
+
+ :param encrypted_secret_dict: The encrypted storage secret.
+ :type encrypted_secret_dict: dict
+
+ :return: The decrypted storage secret.
+ :rtype: str
+
+ :raise SecretsException: Raised in case the decryption of the storage
+ secret fails for some reason.
+ """
+ # calculate the encryption key
+ if encrypted_secret_dict[self.KDF_KEY] != self.KDF_SCRYPT:
+ raise SecretsException("Unknown KDF in stored secret.")
+ key = scrypt.hash(
+ self._passphrase_as_string(),
+ # the salt is stored base64 encoded
+ binascii.a2b_base64(
+ encrypted_secret_dict[self.KDF_SALT_KEY]),
+ buflen=32, # we need a key with 256 bits (32 bytes).
+ )
+ if encrypted_secret_dict[self.KDF_LENGTH_KEY] != len(key):
+ raise SecretsException("Wrong length of decryption key.")
+ if encrypted_secret_dict[self.CIPHER_KEY] != self.CIPHER_AES256:
+ raise SecretsException("Unknown cipher in stored secret.")
+ # recover the initial value and ciphertext
+ iv, ciphertext = encrypted_secret_dict[self.SECRET_KEY].split(
+ self.IV_SEPARATOR, 1)
+ ciphertext = binascii.a2b_base64(ciphertext)
+ decrypted_secret = self._crypto.decrypt_sym(ciphertext, key, iv=iv)
+ if encrypted_secret_dict[self.LENGTH_KEY] != len(decrypted_secret):
+ raise SecretsException("Wrong length of decrypted secret.")
+ return decrypted_secret
+
+ def _encrypt_storage_secret(self, decrypted_secret):
+ """
+ Encrypt the storage secret.
+
+ An encrypted secret has the following structure:
+
+ {
+ '<secret_id>': {
+ 'kdf': 'scrypt',
+ 'kdf_salt': '<b64 repr of salt>'
+ 'kdf_length': <key length>
+ 'cipher': 'aes256',
+ 'length': <secret length>,
+ 'secret': '<encrypted b64 repr of storage_secret>',
+ }
+ }
+
+ :param decrypted_secret: The decrypted storage secret.
+ :type decrypted_secret: str
+
+ :return: The encrypted storage secret.
+ :rtype: dict
+ """
+ # generate random salt
+ salt = os.urandom(self.SALT_LENGTH)
+ # get a 256-bit key
+ key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32)
+ iv, ciphertext = self._crypto.encrypt_sym(decrypted_secret, key)
+ encrypted_secret_dict = {
+ # leap.soledad.crypto submodule uses AES256 for symmetric
+ # encryption.
+ self.KDF_KEY: self.KDF_SCRYPT,
+ self.KDF_SALT_KEY: binascii.b2a_base64(salt),
+ self.KDF_LENGTH_KEY: len(key),
+ self.CIPHER_KEY: self.CIPHER_AES256,
+ self.LENGTH_KEY: len(decrypted_secret),
+ self.SECRET_KEY: '%s%s%s' % (
+ str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)),
+ }
+ return encrypted_secret_dict
+
+ @property
+ def storage_secret(self):
+ """
+ Return the storage secret.
+
+ :return: The decrypted storage secret.
+ :rtype: str
+ """
+ return self._secrets.get(self._secret_id)
+
+ def set_secret_id(self, secret_id):
+ """
+ Define the id of the storage secret to be used.
+
+ This method will also replace the secret in the crypto object.
+
+ :param secret_id: The id of the storage secret to be used.
+ :type secret_id: str
+ """
+ self._secret_id = secret_id
+
+ def _gen_secret(self):
+ """
+ Generate a secret for symmetric encryption and store in a local
+ encrypted file.
+
+ This method emits the following events.signals:
+
+ * SOLEDAD_CREATING_KEYS
+ * SOLEDAD_DONE_CREATING_KEYS
+
+ :return: The id of the generated secret.
+ :rtype: str
+ """
+ events.emit(events.SOLEDAD_CREATING_KEYS, self._uuid)
+ # generate random secret
+ secret = os.urandom(self.GEN_SECRET_LENGTH)
+ secret_id = sha256(secret).hexdigest()
+ self._secrets[secret_id] = secret
+ self._store_secrets()
+ events.emit(events.SOLEDAD_DONE_CREATING_KEYS, self._uuid)
+ return secret_id
+
+ def _store_secrets(self):
+ """
+ Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}.
+ """
+ with open(self._secrets_path, 'w') as f:
+ f.write(
+ json.dumps(
+ self._export_recovery_document()))
+
+ def change_passphrase(self, new_passphrase):
+ """
+ Change the passphrase that encrypts the storage secret.
+
+ :param new_passphrase: The new passphrase.
+ :type new_passphrase: unicode
+
+ :raise NoStorageSecret: Raised if there's no storage secret available.
+ """
+ # TODO: maybe we want to add more checks to guarantee passphrase is
+ # reasonable?
+ soledad_assert_type(new_passphrase, unicode)
+ if len(new_passphrase) < self.MINIMUM_PASSPHRASE_LENGTH:
+ raise PassphraseTooShort(
+ 'Passphrase must be at least %d characters long!' %
+ self.MINIMUM_PASSPHRASE_LENGTH)
+ # ensure there's a secret for which the passphrase will be changed.
+ if not self._has_secret():
+ raise NoStorageSecret()
+ self._passphrase = new_passphrase
+ self._store_secrets()
+ self._put_secrets_in_shared_db()
+
+ #
+ # Setters and getters
+ #
+
+ @property
+ def secret_id(self):
+ return self._secret_id
+
+ def _get_secrets_path(self):
+ return self._secrets_path
+
+ def _set_secrets_path(self, secrets_path):
+ self._secrets_path = secrets_path
+
+ secrets_path = property(
+ _get_secrets_path,
+ _set_secrets_path,
+ doc='The path for the file containing the encrypted symmetric secret.')
+
+ @property
+ def passphrase(self):
+ """
+ Return the passphrase for locking and unlocking encryption secrets for
+ local and remote storage.
+ """
+ return self._passphrase
+
+ def _passphrase_as_string(self):
+ return self._passphrase.encode('utf-8')
+
+ #
+ # remote storage secret
+ #
+
+ @property
+ def remote_storage_secret(self):
+ """
+ Return the secret for remote storage.
+ """
+ key_start = 0
+ key_end = self.REMOTE_STORAGE_SECRET_LENGTH
+ return self.storage_secret[key_start:key_end]
+
+ #
+ # local storage key
+ #
+
+ def _get_local_storage_secret(self):
+ """
+ Return the local storage secret.
+
+ :return: The local storage secret.
+ :rtype: str
+ """
+ secret_len = self.REMOTE_STORAGE_SECRET_LENGTH
+ lsecret_len = self.LOCAL_STORAGE_SECRET_LENGTH
+ pwd_start = secret_len + self.SALT_LENGTH
+ pwd_end = secret_len + lsecret_len
+ return self.storage_secret[pwd_start:pwd_end]
+
+ def _get_local_storage_salt(self):
+ """
+ Return the local storage salt.
+
+ :return: The local storage salt.
+ :rtype: str
+ """
+ salt_start = self.REMOTE_STORAGE_SECRET_LENGTH
+ salt_end = salt_start + self.SALT_LENGTH
+ return self.storage_secret[salt_start:salt_end]
+
+ def get_local_storage_key(self):
+ """
+ Return the local storage key derived from the local storage secret.
+
+ :return: The key for protecting the local database.
+ :rtype: str
+ """
+ return scrypt.hash(
+ password=self._get_local_storage_secret(),
+ salt=self._get_local_storage_salt(),
+ buflen=32, # we need a key with 256 bits (32 bytes)
+ )
+
+ #
+ # sync db key
+ #
+
+ def _get_sync_db_salt(self):
+ """
+ Return the salt for sync db.
+ """
+ salt_start = self.LOCAL_STORAGE_SECRET_LENGTH \
+ + self.REMOTE_STORAGE_SECRET_LENGTH
+ salt_end = salt_start + self.SALT_LENGTH
+ return self.storage_secret[salt_start:salt_end]
+
+ def get_sync_db_key(self):
+ """
+ Return the key for protecting the sync database.
+
+ :return: The key for protecting the sync database.
+ :rtype: str
+ """
+ return scrypt.hash(
+ password=self._get_local_storage_secret(),
+ salt=self._get_sync_db_salt(),
+ buflen=32, # we need a key with 256 bits (32 bytes)
+ )
diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py
index 52e51c6f..f1a2642e 100644
--- a/client/src/leap/soledad/client/shared_db.py
+++ b/client/src/leap/soledad/client/shared_db.py
@@ -14,19 +14,11 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
"""
A shared database for storing/retrieving encrypted key material.
"""
-
-import simplejson as json
-
-
from u1db.remote import http_database
-
-from leap.soledad.common import SHARED_DB_LOCK_DOC_ID_PREFIX
from leap.soledad.client.auth import TokenBasedAuth
@@ -34,6 +26,9 @@ from leap.soledad.client.auth import TokenBasedAuth
# Soledad shared database
# ----------------------------------------------------------------------------
+# TODO could have a hierarchy of soledad exceptions.
+
+
class NoTokenForAuth(Exception):
"""
No token was found for token-based authentication.
@@ -46,6 +41,12 @@ class Unauthorized(Exception):
"""
+class ImproperlyConfiguredError(Exception):
+ """
+ Wrong parameters in the database configuration.
+ """
+
+
class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth):
"""
This is a shared recovery database that enables users to store their
@@ -54,6 +55,10 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth):
# TODO: prevent client from messing with the shared DB.
# TODO: define and document API.
+ # If syncable is False, the database will not attempt to sync against
+ # a remote replica. Default is True.
+ syncable = True
+
#
# Token auth methods.
#
@@ -90,9 +95,7 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth):
#
@staticmethod
- def open_database(url, uuid, create, creds=None):
- # TODO: users should not be able to create the shared database, so we
- # have to remove this from here in the future.
+ def open_database(url, uuid, creds=None, syncable=True):
"""
Open a Soledad shared database.
@@ -100,17 +103,23 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth):
:type url: str
:param uuid: The user's unique id.
:type uuid: str
- :param create: Should the database be created if it does not already
- exist?
- :type create: bool
- :param token: An authentication token for accessing the shared db.
- :type token: str
+ :param creds: A tuple containing the authentication method and
+ credentials.
+ :type creds: tuple
+ :param syncable:
+ If syncable is False, the database will not attempt to sync against
+ a remote replica.
+ :type syncable: bool
:return: The shared database in the given url.
:rtype: SoledadSharedDatabase
"""
+ # XXX fix below, doesn't work with tests.
+ #if syncable and not url.startswith('https://'):
+ # raise ImproperlyConfiguredError(
+ # "Remote soledad server must be an https URI")
db = SoledadSharedDatabase(url, uuid, creds=creds)
- db.open(create)
+ db.syncable = syncable
return db
@staticmethod
@@ -153,9 +162,12 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth):
:raise HTTPError: Raised if any HTTP error occurs.
"""
- res, headers = self._request_json('PUT', ['lock', self._uuid],
- body={})
- return res['token'], res['timeout']
+ if self.syncable:
+ res, headers = self._request_json(
+ 'PUT', ['lock', self._uuid], body={})
+ return res['token'], res['timeout']
+ else:
+ return None, None
def unlock(self, token):
"""
@@ -166,5 +178,6 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth):
:raise HTTPError:
"""
- res, headers = self._request_json('DELETE', ['lock', self._uuid],
- params={'token': token})
+ if self.syncable:
+ _, _ = self._request_json(
+ 'DELETE', ['lock', self._uuid], params={'token': token})
diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index fded2119..b2025130 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -42,261 +42,120 @@ SQLCipher 1.1 databases, we do not implement them as all SQLCipher databases
handled by Soledad should be created by SQLCipher >= 2.0.
"""
import logging
-import multiprocessing
import os
-import sqlite3
-import string
import threading
-import time
import json
+import u1db
+
+from u1db import errors as u1db_errors
+from u1db.backends import sqlite_backend
from hashlib import sha256
from contextlib import contextmanager
from collections import defaultdict
+from functools import partial
-from pysqlcipher import dbapi2
-from u1db.backends import sqlite_backend
-from u1db import errors as u1db_errors
-from taskthread import TimerTask
+from pysqlcipher import dbapi2 as sqlcipher_dbapi2
+
+from twisted.internet import reactor
+from twisted.internet.threads import deferToThreadPool
+from twisted.python.threadpool import ThreadPool
+from twisted.enterprise import adbapi
-from leap.soledad.client.crypto import SyncEncrypterPool, SyncDecrypterPool
-from leap.soledad.client.target import SoledadSyncTarget
-from leap.soledad.client.target import PendingReceivedDocsSyncError
+from leap.soledad.client import encdecpool
+from leap.soledad.client.http_target import SoledadHTTPSyncTarget
from leap.soledad.client.sync import SoledadSynchronizer
+
+from leap.soledad.client import pragmas
+from leap.soledad.common import soledad_assert
from leap.soledad.common.document import SoledadDocument
logger = logging.getLogger(__name__)
-# Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2
-sqlite_backend.dbapi2 = dbapi2
-
-# It seems that, as long as we are not using old sqlite versions, serialized
-# mode is enabled by default at compile time. So accessing db connections from
-# different threads should be safe, as long as no attempt is made to use them
-# from multiple threads with no locking.
-# See https://sqlite.org/threadsafe.html
-# and http://bugs.python.org/issue16509
-
-SQLITE_CHECK_SAME_THREAD = False
-
-# We set isolation_level to None to setup autocommit mode.
-# See: http://docs.python.org/2/library/sqlite3.html#controlling-transactions
-# This avoids problems with sequential operations using the same soledad object
-# trying to open new transactions
-# (The error was:
-# OperationalError:cannot start a transaction within a transaction.)
-SQLITE_ISOLATION_LEVEL = None
-
-
-def open(path, password, create=True, document_factory=None, crypto=None,
- raw_key=False, cipher='aes-256-cbc', kdf_iter=4000,
- cipher_page_size=1024, defer_encryption=False):
- """Open a database at the given location.
-
- Will raise u1db.errors.DatabaseDoesNotExist if create=False and the
- database does not already exist.
-
- :param path: The filesystem path for the database to open.
- :type path: str
- :param create: True/False, should the database be created if it doesn't
- already exist?
- :param create: bool
- :param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- :type document_factory: callable
- :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt
- document contents when syncing.
- :type crypto: soledad.crypto.SoledadCrypto
- :param raw_key: Whether C{password} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
- :type raw_key: bool
- :param cipher: The cipher and mode to use.
- :type cipher: str
- :param kdf_iter: The number of iterations to use.
- :type kdf_iter: int
- :param cipher_page_size: The page size.
- :type cipher_page_size: int
- :param defer_encryption: Whether to defer encryption/decryption of
- documents, or do it inline while syncing.
- :type defer_encryption: bool
-
- :return: An instance of Database.
- :rtype SQLCipherDatabase
- """
- return SQLCipherDatabase.open_database(
- path, password, create=create, document_factory=document_factory,
- crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter,
- cipher_page_size=cipher_page_size, defer_encryption=defer_encryption)
+# Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2
+sqlite_backend.dbapi2 = sqlcipher_dbapi2
-#
-# Exceptions
-#
-class DatabaseIsNotEncrypted(Exception):
- """
- Exception raised when trying to open non-encrypted databases.
+def initialize_sqlcipher_db(opts, on_init=None, check_same_thread=True):
"""
- pass
-
+ Initialize a SQLCipher database.
-class NotAnHexString(Exception):
+ :param opts:
+ :type opts: SQLCipherOptions
+ :param on_init: a tuple of queries to be executed on initialization
+ :type on_init: tuple
+ :return: pysqlcipher.dbapi2.Connection
"""
- Raised when trying to (raw) key the database with a non-hex string.
- """
- pass
+ # Note: There seemed to be a bug in sqlite 3.5.9 (with python2.6)
+ # where without re-opening the database on Windows, it
+ # doesn't see the transaction that was just committed
+ # Removing from here now, look at the pysqlite implementation if the
+ # bug shows up in windows.
+ if not os.path.isfile(opts.path) and not opts.create:
+ raise u1db_errors.DatabaseDoesNotExist()
-#
-# The SQLCipher database
-#
+ conn = sqlcipher_dbapi2.connect(
+ opts.path, check_same_thread=check_same_thread)
+ pragmas.set_init_pragmas(conn, opts, extra_queries=on_init)
+ return conn
-class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
- """
- A U1DB implementation that uses SQLCipher as its persistence layer.
- """
- defer_encryption = False
- _index_storage_value = 'expand referenced encrypted'
- k_lock = threading.Lock()
- create_doc_lock = threading.Lock()
- update_indexes_lock = threading.Lock()
- _sync_watcher = None
- _sync_enc_pool = None
+def initialize_sqlcipher_adbapi_db(opts, extra_queries=None):
+ from leap.soledad.client import sqlcipher_adbapi
+ return sqlcipher_adbapi.getConnectionPool(
+ opts, extra_queries=extra_queries)
- """
- The name of the local symmetrically encrypted documents to
- sync database file.
- """
- LOCAL_SYMMETRIC_SYNC_FILE_NAME = 'sync.u1db'
+class SQLCipherOptions(object):
"""
- A dictionary that hold locks which avoid multiple sync attempts from the
- same database replica.
- """
- encrypting_lock = threading.Lock()
-
- """
- Period or recurrence of the periodic encrypting task, in seconds.
- """
- ENCRYPT_TASK_PERIOD = 1
-
- syncing_lock = defaultdict(threading.Lock)
- """
- A dictionary that hold locks which avoid multiple sync attempts from the
- same database replica.
+ A container with options for the initialization of an SQLCipher database.
"""
- def __init__(self, sqlcipher_file, password, document_factory=None,
- crypto=None, raw_key=False, cipher='aes-256-cbc',
- kdf_iter=4000, cipher_page_size=1024):
+ @classmethod
+ def copy(cls, source, path=None, key=None, create=None,
+ is_raw_key=None, cipher=None, kdf_iter=None,
+ cipher_page_size=None, defer_encryption=None, sync_db_key=None):
"""
- Connect to an existing SQLCipher database, creating a new sqlcipher
- database file if needed.
-
- :param sqlcipher_file: The path for the SQLCipher file.
- :type sqlcipher_file: str
- :param password: The password that protects the SQLCipher db.
- :type password: str
- :param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- :type document_factory: callable
- :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt
- document contents when syncing.
- :type crypto: soledad.crypto.SoledadCrypto
- :param raw_key: Whether password is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the
- encyrption key.
- :type raw_key: bool
- :param cipher: The cipher and mode to use.
- :type cipher: str
- :param kdf_iter: The number of iterations to use.
- :type kdf_iter: int
- :param cipher_page_size: The page size.
- :type cipher_page_size: int
+ Return a copy of C{source} with parameters different than None
+ replaced by new values.
"""
- # ensure the db is encrypted if the file already exists
- if os.path.exists(sqlcipher_file):
- self.assert_db_is_encrypted(
- sqlcipher_file, password, raw_key, cipher, kdf_iter,
- cipher_page_size)
+ local_vars = locals()
+ args = []
+ kwargs = {}
- # connect to the sqlcipher database
- with self.k_lock:
- self._db_handle = dbapi2.connect(
- sqlcipher_file,
- isolation_level=SQLITE_ISOLATION_LEVEL,
- check_same_thread=SQLITE_CHECK_SAME_THREAD)
- # set SQLCipher cryptographic parameters
- self._set_crypto_pragmas(
- self._db_handle, password, raw_key, cipher, kdf_iter,
- cipher_page_size)
- if os.environ.get('LEAP_SQLITE_NOSYNC'):
- self._pragma_synchronous_off(self._db_handle)
+ for name in ["path", "key"]:
+ val = local_vars[name]
+ if val is not None:
+ args.append(val)
else:
- self._pragma_synchronous_normal(self._db_handle)
- if os.environ.get('LEAP_SQLITE_MEMSTORE'):
- self._pragma_mem_temp_store(self._db_handle)
- self._pragma_write_ahead_logging(self._db_handle)
- self._real_replica_uid = None
- self._ensure_schema()
- self._crypto = crypto
-
- self._sync_db = None
- self._sync_db_write_lock = None
- self._sync_enc_pool = None
+ args.append(getattr(source, name))
- if self.defer_encryption:
- if sqlcipher_file != ":memory:":
- self._sync_db_path = "%s-sync" % sqlcipher_file
+ for name in ["create", "is_raw_key", "cipher", "kdf_iter",
+ "cipher_page_size", "defer_encryption", "sync_db_key"]:
+ val = local_vars[name]
+ if val is not None:
+ kwargs[name] = val
else:
- self._sync_db_path = ":memory:"
-
- # initialize sync db
- self._init_sync_db()
-
- # initialize syncing queue encryption pool
- self._sync_enc_pool = SyncEncrypterPool(
- self._crypto, self._sync_db, self._sync_db_write_lock)
- self._sync_watcher = TimerTask(self._encrypt_syncing_docs,
- self.ENCRYPT_TASK_PERIOD)
- self._sync_watcher.start()
-
- def factory(doc_id=None, rev=None, json='{}', has_conflicts=False,
- syncable=True):
- return SoledadDocument(doc_id=doc_id, rev=rev, json=json,
- has_conflicts=has_conflicts,
- syncable=syncable)
- self.set_document_factory(factory)
- # we store syncers in a dictionary indexed by the target URL. We also
- # store a hash of the auth info in case auth info expires and we need
- # to rebuild the syncer for that target. The final self._syncers
- # format is the following:
- #
- # self._syncers = {'<url>': ('<auth_hash>', syncer), ...}
- self._syncers = {}
-
- @classmethod
- def _open_database(cls, sqlcipher_file, password, document_factory=None,
- crypto=None, raw_key=False, cipher='aes-256-cbc',
- kdf_iter=4000, cipher_page_size=1024,
- defer_encryption=False):
- """
- Open a SQLCipher database.
-
- :param sqlcipher_file: The path for the SQLCipher file.
- :type sqlcipher_file: str
- :param password: The password that protects the SQLCipher db.
- :type password: str
- :param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- :type document_factory: callable
- :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt
- document contents when syncing.
- :type crypto: soledad.crypto.SoledadCrypto
- :param raw_key: Whether C{password} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
+ kwargs[name] = getattr(source, name)
+
+ return SQLCipherOptions(*args, **kwargs)
+
+ def __init__(self, path, key, create=True, is_raw_key=False,
+ cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024,
+ defer_encryption=False, sync_db_key=None):
+ """
+ :param path: The filesystem path for the database to open.
+ :type path: str
+ :param create:
+ True/False, should the database be created if it doesn't
+ already exist?
+ :param create: bool
+ :param is_raw_key:
+ Whether ``password`` is a raw 64-char hex string or a passphrase
+ that should be hashed to obtain the encyrption key.
:type raw_key: bool
:param cipher: The cipher and mode to use.
:type cipher: str
@@ -304,233 +163,93 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
:type kdf_iter: int
:param cipher_page_size: The page size.
:type cipher_page_size: int
- :param defer_encryption: Whether to defer encryption/decryption of
- documents, or do it inline while syncing.
+ :param defer_encryption:
+ Whether to defer encryption/decryption of documents, or do it
+ inline while syncing.
:type defer_encryption: bool
-
- :return: The database object.
- :rtype: SQLCipherDatabase
"""
- cls.defer_encryption = defer_encryption
- if not os.path.isfile(sqlcipher_file):
- raise u1db_errors.DatabaseDoesNotExist()
-
- tries = 2
- # Note: There seems to be a bug in sqlite 3.5.9 (with python2.6)
- # where without re-opening the database on Windows, it
- # doesn't see the transaction that was just committed
- while True:
-
- with cls.k_lock:
- db_handle = dbapi2.connect(
- sqlcipher_file,
- check_same_thread=SQLITE_CHECK_SAME_THREAD)
-
- try:
- # set cryptographic params
- cls._set_crypto_pragmas(
- db_handle, password, raw_key, cipher, kdf_iter,
- cipher_page_size)
- c = db_handle.cursor()
- # XXX if we use it here, it should be public
- v, err = cls._which_index_storage(c)
- except Exception as exc:
- logger.warning("ERROR OPENING DATABASE!")
- logger.debug("error was: %r" % exc)
- v, err = None, exc
- finally:
- db_handle.close()
- if v is not None:
- break
- # possibly another process is initializing it, wait for it to be
- # done
- if tries == 0:
- raise err # go for the richest error?
- tries -= 1
- time.sleep(cls.WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL)
- return SQLCipherDatabase._sqlite_registry[v](
- sqlcipher_file, password, document_factory=document_factory,
- crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter,
- cipher_page_size=cipher_page_size)
+ self.path = path
+ self.key = key
+ self.is_raw_key = is_raw_key
+ self.create = create
+ self.cipher = cipher
+ self.kdf_iter = kdf_iter
+ self.cipher_page_size = cipher_page_size
+ self.defer_encryption = defer_encryption
+ self.sync_db_key = sync_db_key
- @classmethod
- def open_database(cls, sqlcipher_file, password, create, backend_cls=None,
- document_factory=None, crypto=None, raw_key=False,
- cipher='aes-256-cbc', kdf_iter=4000,
- cipher_page_size=1024, defer_encryption=False):
+ def __str__(self):
"""
- Open a SQLCipher database.
-
- :param sqlcipher_file: The path for the SQLCipher file.
- :type sqlcipher_file: str
-
- :param password: The password that protects the SQLCipher db.
- :type password: str
-
- :param create: Should the datbase be created if it does not already
- exist?
- :type create: bool
-
- :param backend_cls: A class to use as backend.
- :type backend_cls: type
+ Return string representation of options, for easy debugging.
- :param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- :type document_factory: callable
-
- :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt
- document contents when syncing.
- :type crypto: soledad.crypto.SoledadCrypto
-
- :param raw_key: Whether C{password} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the
- encyrption key.
- :type raw_key: bool
+ :return: String representation of options.
+ :rtype: str
+ """
+ attr_names = filter(lambda a: not a.startswith('_'), dir(self))
+ attr_str = []
+ for a in attr_names:
+ attr_str.append(a + "=" + str(getattr(self, a)))
+ name = self.__class__.__name__
+ return "%s(%s)" % (name, ', '.join(attr_str))
- :param cipher: The cipher and mode to use.
- :type cipher: str
- :param kdf_iter: The number of iterations to use.
- :type kdf_iter: int
+#
+# The SQLCipher database
+#
- :param cipher_page_size: The page size.
- :type cipher_page_size: int
+class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
+ """
+ A U1DB implementation that uses SQLCipher as its persistence layer.
+ """
+ defer_encryption = False
- :param defer_encryption: Whether to defer encryption/decryption of
- documents, or do it inline while syncing.
- :type defer_encryption: bool
+ # The attribute _index_storage_value will be used as the lookup key for the
+ # implementation of the SQLCipher storage backend.
+ _index_storage_value = 'expand referenced encrypted'
- :return: The database object.
- :rtype: SQLCipherDatabase
- """
- cls.defer_encryption = defer_encryption
- try:
- return cls._open_database(
- sqlcipher_file, password, document_factory=document_factory,
- crypto=crypto, raw_key=raw_key, cipher=cipher,
- kdf_iter=kdf_iter, cipher_page_size=cipher_page_size,
- defer_encryption=defer_encryption)
- except u1db_errors.DatabaseDoesNotExist:
- if not create:
- raise
- # TODO: remove backend class from here.
- if backend_cls is None:
- # default is SQLCipherPartialExpandDatabase
- backend_cls = SQLCipherDatabase
- return backend_cls(
- sqlcipher_file, password, document_factory=document_factory,
- crypto=crypto, raw_key=raw_key, cipher=cipher,
- kdf_iter=kdf_iter, cipher_page_size=cipher_page_size)
-
- def sync(self, url, creds=None, autocreate=True, defer_decryption=True):
+ def __init__(self, opts):
"""
- Synchronize documents with remote replica exposed at url.
+ Connect to an existing SQLCipher database, creating a new sqlcipher
+ database file if needed.
- There can be at most one instance syncing the same database replica at
- the same time, so this method will block until the syncing lock can be
- acquired.
+ *** IMPORTANT ***
- :param url: The url of the target replica to sync with.
- :type url: str
- :param creds: optional dictionary giving credentials.
- to authorize the operation with the server.
- :type creds: dict
- :param autocreate: Ask the target to create the db if non-existent.
- :type autocreate: bool
- :param defer_decryption: Whether to defer the decryption process using
- the intermediate database. If False,
- decryption will be done inline.
- :type defer_decryption: bool
+ Don't forget to close the database after use by calling the close()
+ method otherwise some resources might not be freed and you may
+ experience several kinds of leakages.
- :return: The local generation before the synchronisation was performed.
- :rtype: int
- """
- res = None
- # the following context manager blocks until the syncing lock can be
- # acquired.
- with self.syncer(url, creds=creds) as syncer:
+ *** IMPORTANT ***
- # XXX could mark the critical section here...
- try:
- if defer_decryption and not self.defer_encryption:
- logger.warning("Can't defer decryption without first having "
- "created a sync db. Falling back to normal "
- "syncing mode.")
- defer_decryption = False
- res = syncer.sync(autocreate=autocreate,
- defer_decryption=defer_decryption)
-
- except PendingReceivedDocsSyncError:
- logger.warning("Local sync db is not clear, skipping sync...")
- return
-
- return res
-
- def stop_sync(self):
+ :param opts: options for initialization of the SQLCipher database.
+ :type opts: SQLCipherOptions
"""
- Interrupt all ongoing syncs.
- """
- for url in self._syncers:
- _, syncer = self._syncers[url]
- syncer.stop()
+ # ensure the db is encrypted if the file already exists
+ if os.path.isfile(opts.path):
+ _assert_db_is_encrypted(opts)
- @contextmanager
- def syncer(self, url, creds=None):
- """
- Accesor for synchronizer.
+ # connect to the sqlcipher database
+ self._db_handle = initialize_sqlcipher_db(opts)
- As we reuse the same synchronizer for every sync, there can be only
- one instance synchronizing the same database replica at the same time.
- Because of that, this method blocks until the syncing lock can be
- acquired.
- """
- with SQLCipherDatabase.syncing_lock[self._get_replica_uid()]:
- syncer = self._get_syncer(url, creds=creds)
- yield syncer
+ # TODO ---------------------------------------------------
+ # Everything else in this initialization has to be factored
+ # out, so it can be used from SoledadSQLCipherWrapper.__init__
+ # too.
+ # ---------------------------------------------------------
- @property
- def syncing(self):
- lock = SQLCipherDatabase.syncing_lock[self._get_replica_uid()]
- acquired_lock = lock.acquire(False)
- if acquired_lock is False:
- return True
- lock.release()
- return False
+ self._ensure_schema()
+ self.set_document_factory(soledad_doc_factory)
+ self._prime_replica_uid()
- def _get_syncer(self, url, creds=None):
+ def _prime_replica_uid(self):
"""
- Get a synchronizer for C{url} using C{creds}.
-
- :param url: The url of the target replica to sync with.
- :type url: str
- :param creds: optional dictionary giving credentials.
- to authorize the operation with the server.
- :type creds: dict
-
- :return: A synchronizer.
- :rtype: Synchronizer
+ In the u1db implementation, _replica_uid is a property
+ that returns the value in _real_replica_uid, and does
+ a db query if no value found.
+ Here we prime the replica uid during initialization so
+ that we don't have to wait for the query afterwards.
"""
- # we want to store at most one syncer for each url, so we also store a
- # hash of the connection credentials and replace the stored syncer for
- # a certain url if credentials have changed.
- h = sha256(json.dumps([url, creds])).hexdigest()
- cur_h, syncer = self._syncers.get(url, (None, None))
- if syncer is None or h != cur_h:
- wlock = self._sync_db_write_lock
- syncer = SoledadSynchronizer(
- self,
- SoledadSyncTarget(url,
- self._replica_uid,
- creds=creds,
- crypto=self._crypto,
- sync_db=self._sync_db,
- sync_db_write_lock=wlock))
- self._syncers[url] = (h, syncer)
- # in order to reuse the same synchronizer multiple times we have to
- # reset its state (i.e. the number of documents received from target
- # and inserted in the local replica).
- syncer.num_inserted = 0
- return syncer
+ self._real_replica_uid = None
+ self._get_replica_uid()
def _extra_schema_init(self, c):
"""
@@ -547,63 +266,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
'ALTER TABLE document '
'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE')
- def _init_sync_db(self):
- """
- Initialize the Symmetrically-Encrypted document to be synced database,
- and the queue to communicate with subprocess workers.
- """
- self._sync_db = sqlite3.connect(self._sync_db_path,
- check_same_thread=False)
-
- self._sync_db_write_lock = threading.Lock()
- self._create_sync_db_tables()
- self.sync_queue = multiprocessing.Queue()
-
- def _create_sync_db_tables(self):
- """
- Create tables for the local sync documents db if needed.
- """
- encr = SyncEncrypterPool
- decr = SyncDecrypterPool
- sql_encr = ("CREATE TABLE IF NOT EXISTS %s (%s)" % (
- encr.TABLE_NAME, encr.FIELD_NAMES))
- sql_decr = ("CREATE TABLE IF NOT EXISTS %s (%s)" % (
- decr.TABLE_NAME, decr.FIELD_NAMES))
-
- with self._sync_db_write_lock:
- with self._sync_db:
- self._sync_db.execute(sql_encr)
- self._sync_db.execute(sql_decr)
-
- #
- # Symmetric encryption of syncing docs
- #
-
- def _encrypt_syncing_docs(self):
- """
- Process the syncing queue and send the documents there
- to be encrypted in the sync db. They will be read by the
- SoledadSyncTarget during the sync_exchange.
-
- Called periodical from the TimerTask self._sync_watcher.
- """
- lock = self.encrypting_lock
- # optional wait flag used to avoid blocking
- if not lock.acquire(False):
- return
- else:
- queue = self.sync_queue
- try:
- while not queue.empty():
- doc = queue.get_nowait()
- self._sync_enc_pool.encrypt_doc(doc)
-
- except Exception as exc:
- logger.error("Error while encrypting docs to sync")
- logger.exception(exc)
- finally:
- lock.release()
-
#
# Document operations
#
@@ -619,12 +281,82 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
:return: The new document revision.
:rtype: str
"""
- doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc(
- self, doc)
+ doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc(self, doc)
if self.defer_encryption:
- self.sync_queue.put_nowait(doc)
+ # TODO move to api?
+ self._sync_enc_pool.enqueue_doc_for_encryption(doc)
return doc_rev
+ #
+ # SQLCipher API methods
+ #
+
+ # Extra query methods: extensions to the base u1db sqlite implmentation.
+
+ def get_count_from_index(self, index_name, *key_values):
+ """
+ Return the count for a given combination of index_name
+ and key values.
+
+ Extension method made from similar methods in u1db version 13.09
+
+ :param index_name: The index to query
+ :type index_name: str
+ :param key_values: values to match. eg, if you have
+ an index with 3 fields then you would have:
+ get_from_index(index_name, val1, val2, val3)
+ :type key_values: tuple
+ :return: count.
+ :rtype: int
+ """
+ c = self._db_handle.cursor()
+ definition = self._get_index_definition(index_name)
+
+ if len(key_values) != len(definition):
+ raise u1db_errors.InvalidValueForIndex()
+ tables = ["document_fields d%d" % i for i in range(len(definition))]
+ novalue_where = ["d.doc_id = d%d.doc_id"
+ " AND d%d.field_name = ?"
+ % (i, i) for i in range(len(definition))]
+ exact_where = [novalue_where[i]
+ + (" AND d%d.value = ?" % (i,))
+ for i in range(len(definition))]
+ args = []
+ where = []
+ for idx, (field, value) in enumerate(zip(definition, key_values)):
+ args.append(field)
+ where.append(exact_where[idx])
+ args.append(value)
+
+ tables = ["document_fields d%d" % i for i in range(len(definition))]
+ statement = (
+ "SELECT COUNT(*) FROM document d, %s WHERE %s " % (
+ ', '.join(tables),
+ ' AND '.join(where),
+ ))
+ try:
+ c.execute(statement, tuple(args))
+ except sqlcipher_dbapi2.OperationalError, e:
+ raise sqlcipher_dbapi2.OperationalError(
+ str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args))
+ res = c.fetchall()
+ return res[0][0]
+
+ def close(self):
+ """
+ Close db connections.
+ """
+ # TODO should be handled by adbapi instead
+ # TODO syncdb should be stopped first
+
+ if logger is not None: # logger might be none if called from __del__
+ logger.debug("SQLCipher backend: closing")
+
+ # close the actual database
+ if getattr(self, '_db_handle', False):
+ self._db_handle.close()
+ self._db_handle = None
+
# indexes
def _put_and_update_indexes(self, old_doc, doc):
@@ -636,13 +368,11 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
:param doc: The new version of the document.
:type doc: u1db.Document
"""
- with self.update_indexes_lock:
- sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes(
- self, old_doc, doc)
- c = self._db_handle.cursor()
- c.execute('UPDATE document SET syncable=? '
- 'WHERE doc_id=?',
- (doc.syncable, doc.doc_id))
+ sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes(
+ self, old_doc, doc)
+ c = self._db_handle.cursor()
+ c.execute('UPDATE document SET syncable=? WHERE doc_id=?',
+ (doc.syncable, doc.doc_id))
def _get_doc(self, doc_id, check_for_conflicts=False):
"""
@@ -662,438 +392,426 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
self, doc_id, check_for_conflicts)
if doc:
c = self._db_handle.cursor()
- c.execute('SELECT syncable FROM document '
- 'WHERE doc_id=?',
+ c.execute('SELECT syncable FROM document WHERE doc_id=?',
(doc.doc_id,))
result = c.fetchone()
doc.syncable = bool(result[0])
return doc
- #
- # SQLCipher API methods
- #
-
- @classmethod
- def assert_db_is_encrypted(cls, sqlcipher_file, key, raw_key, cipher,
- kdf_iter, cipher_page_size):
+ def __del__(self):
"""
- Assert that C{sqlcipher_file} contains an encrypted database.
-
- When opening an existing database, PRAGMA key will not immediately
- throw an error if the key provided is incorrect. To test that the
- database can be successfully opened with the provided key, it is
- necessary to perform some operation on the database (i.e. read from
- it) and confirm it is success.
-
- The easiest way to do this is select off the sqlite_master table,
- which will attempt to read the first page of the database and will
- parse the schema.
-
- :param sqlcipher_file: The path for the SQLCipher file.
- :type sqlcipher_file: str
- :param key: The key that protects the SQLCipher db.
- :type key: str
- :param raw_key: Whether C{key} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
- :type raw_key: bool
- :param cipher: The cipher and mode to use.
- :type cipher: str
- :param kdf_iter: The number of iterations to use.
- :type kdf_iter: int
- :param cipher_page_size: The page size.
- :type cipher_page_size: int
- """
- try:
- # try to open an encrypted database with the regular u1db
- # backend should raise a DatabaseError exception.
- sqlite_backend.SQLitePartialExpandDatabase(sqlcipher_file)
- raise DatabaseIsNotEncrypted()
- except dbapi2.DatabaseError:
- # assert that we can access it using SQLCipher with the given
- # key
- with cls.k_lock:
- db_handle = dbapi2.connect(
- sqlcipher_file,
- isolation_level=SQLITE_ISOLATION_LEVEL,
- check_same_thread=SQLITE_CHECK_SAME_THREAD)
- cls._set_crypto_pragmas(
- db_handle, key, raw_key, cipher,
- kdf_iter, cipher_page_size)
- db_handle.cursor().execute(
- 'SELECT count(*) FROM sqlite_master')
+ Free resources when deleting or garbage collecting the database.
- @classmethod
- def _set_crypto_pragmas(cls, db_handle, key, raw_key, cipher, kdf_iter,
- cipher_page_size):
- """
- Set cryptographic params (key, cipher, KDF number of iterations and
- cipher page size).
+ This is only here to minimze problems if someone ever forgets to call
+ the close() method after using the database; you should not rely on
+ garbage collecting to free up the database resources.
"""
- cls._pragma_key(db_handle, key, raw_key)
- cls._pragma_cipher(db_handle, cipher)
- cls._pragma_kdf_iter(db_handle, kdf_iter)
- cls._pragma_cipher_page_size(db_handle, cipher_page_size)
+ self.close()
- @classmethod
- def _pragma_key(cls, db_handle, key, raw_key):
- """
- Set the C{key} for use with the database.
- The process of creating a new, encrypted database is called 'keying'
- the database. SQLCipher uses just-in-time key derivation at the point
- it is first needed for an operation. This means that the key (and any
- options) must be set before the first operation on the database. As
- soon as the database is touched (e.g. SELECT, CREATE TABLE, UPDATE,
- etc.) and pages need to be read or written, the key is prepared for
- use.
+class SQLCipherU1DBSync(SQLCipherDatabase):
+ """
+ Soledad syncer implementation.
+ """
- Implementation Notes:
+ _sync_enc_pool = None
- * PRAGMA key should generally be called as the first operation on a
- database.
+ """
+ The name of the local symmetrically encrypted documents to
+ sync database file.
+ """
+ LOCAL_SYMMETRIC_SYNC_FILE_NAME = 'sync.u1db'
- :param key: The key for use with the database.
- :type key: str
- :param raw_key: Whether C{key} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
- :type raw_key: bool
- """
- if raw_key:
- cls._pragma_key_raw(db_handle, key)
- else:
- cls._pragma_key_passphrase(db_handle, key)
+ """
+ Period or recurrence of the Looping Call that will do the encryption to the
+ syncdb (in seconds).
+ """
+ ENCRYPT_LOOP_PERIOD = 1
- @classmethod
- def _pragma_key_passphrase(cls, db_handle, passphrase):
- """
- Set a passphrase for encryption key derivation.
-
- The key itself can be a passphrase, which is converted to a key using
- PBKDF2 key derivation. The result is used as the encryption key for
- the database. By using this method, there is no way to alter the KDF;
- if you want to do so you should use a raw key instead and derive the
- key using your own KDF.
-
- :param db_handle: A handle to the SQLCipher database.
- :type db_handle: pysqlcipher.Connection
- :param passphrase: The passphrase used to derive the encryption key.
- :type passphrase: str
- """
- db_handle.cursor().execute("PRAGMA key = '%s'" % passphrase)
+ """
+ A dictionary that hold locks which avoid multiple sync attempts from the
+ same database replica.
+ """
+ syncing_lock = defaultdict(threading.Lock)
- @classmethod
- def _pragma_key_raw(cls, db_handle, key):
- """
- Set a raw hexadecimal encryption key.
+ def __init__(self, opts, soledad_crypto, replica_uid, cert_file,
+ defer_encryption=False):
- It is possible to specify an exact byte sequence using a blob literal.
- With this method, it is the calling application's responsibility to
- ensure that the data provided is a 64 character hex string, which will
- be converted directly to 32 bytes (256 bits) of key data.
+ self._opts = opts
+ self._path = opts.path
+ self._crypto = soledad_crypto
+ self.__replica_uid = replica_uid
+ self._cert_file = cert_file
- :param db_handle: A handle to the SQLCipher database.
- :type db_handle: pysqlcipher.Connection
- :param key: A 64 character hex string.
- :type key: str
- """
- if not all(c in string.hexdigits for c in key):
- raise NotAnHexString(key)
- db_handle.cursor().execute('PRAGMA key = "x\'%s"' % key)
+ self._sync_db_key = opts.sync_db_key
+ self._sync_db = None
+ self._sync_enc_pool = None
- @classmethod
- def _pragma_cipher(cls, db_handle, cipher='aes-256-cbc'):
- """
- Set the cipher and mode to use for symmetric encryption.
+ # we store syncers in a dictionary indexed by the target URL. We also
+ # store a hash of the auth info in case auth info expires and we need
+ # to rebuild the syncer for that target. The final self._syncers
+ # format is the following:
+ #
+ # self._syncers = {'<url>': ('<auth_hash>', syncer), ...}
- SQLCipher uses aes-256-cbc as the default cipher and mode of
- operation. It is possible to change this, though not generally
- recommended, using PRAGMA cipher.
+ self._syncers = {}
- SQLCipher makes direct use of libssl, so all cipher options available
- to libssl are also available for use with SQLCipher. See `man enc` for
- OpenSSL's supported ciphers.
+ self.running = False
+ self._sync_threadpool = None
+ self._initialize_sync_threadpool()
- Implementation Notes:
+ self._reactor = reactor
+ self._reactor.callWhenRunning(self._start)
- * PRAGMA cipher must be called after PRAGMA key and before the first
- actual database operation or it will have no effect.
+ self._db_handle = None
+ self._initialize_main_db()
- * If a non-default value is used PRAGMA cipher to create a database,
- it must also be called every time that database is opened.
+ # the sync_db is used both for deferred encryption and decryption, so
+ # we want to initialize it anyway to allow for all combinations of
+ # deferred encryption and decryption configurations.
+ self._initialize_sync_db(opts)
- * SQLCipher does not implement its own encryption. Instead it uses the
- widely available and peer-reviewed OpenSSL libcrypto for all
- cryptographic functions.
+ if defer_encryption:
+ # initialize syncing queue encryption pool
+ self._sync_enc_pool = encdecpool.SyncEncrypterPool(
+ self._crypto, self._sync_db)
- :param db_handle: A handle to the SQLCipher database.
- :type db_handle: pysqlcipher.Connection
- :param cipher: The cipher and mode to use.
- :type cipher: str
- """
- db_handle.cursor().execute("PRAGMA cipher = '%s'" % cipher)
+ self.shutdownID = None
- @classmethod
- def _pragma_kdf_iter(cls, db_handle, kdf_iter=4000):
- """
- Set the number of iterations for the key derivation function.
+ @property
+ def _replica_uid(self):
+ return str(self.__replica_uid)
- SQLCipher uses PBKDF2 key derivation to strengthen the key and make it
- resistent to brute force and dictionary attacks. The default
- configuration uses 4000 PBKDF2 iterations (effectively 16,000 SHA1
- operations). PRAGMA kdf_iter can be used to increase or decrease the
- number of iterations used.
+ def _start(self):
+ if not self.running:
+ self._sync_threadpool.start()
+ self.shutdownID = self._reactor.addSystemEventTrigger(
+ 'during', 'shutdown', self.finalClose)
+ self.running = True
- Implementation Notes:
+ def _defer_to_sync_threadpool(self, meth, *args, **kwargs):
+ return deferToThreadPool(
+ self._reactor, self._sync_threadpool, meth, *args, **kwargs)
- * PRAGMA kdf_iter must be called after PRAGMA key and before the first
- actual database operation or it will have no effect.
+ def _initialize_main_db(self):
- * If a non-default value is used PRAGMA kdf_iter to create a database,
- it must also be called every time that database is opened.
+ def _init_db():
+ self._db_handle = initialize_sqlcipher_db(
+ self._opts, check_same_thread=False)
+ self._real_replica_uid = None
+ self._ensure_schema()
+ self.set_document_factory(soledad_doc_factory)
- * It is not recommended to reduce the number of iterations if a
- passphrase is in use.
+ return self._defer_to_sync_threadpool(_init_db)
- :param db_handle: A handle to the SQLCipher database.
- :type db_handle: pysqlcipher.Connection
- :param kdf_iter: The number of iterations to use.
- :type kdf_iter: int
+ def _initialize_sync_threadpool(self):
"""
- db_handle.cursor().execute("PRAGMA kdf_iter = '%d'" % kdf_iter)
+ Initialize a ThreadPool with exactly one thread, that will be used to
+ run all the network blocking calls for syncing on a separate thread.
- @classmethod
- def _pragma_cipher_page_size(cls, db_handle, cipher_page_size=1024):
+ TODO this needs to be ported away from urllib and into twisted async
+ calls, and then we can ditch this syncing thread and reintegrate into
+ the main reactor.
"""
- Set the page size of the encrypted database.
-
- SQLCipher 2 introduced the new PRAGMA cipher_page_size that can be
- used to adjust the page size for the encrypted database. The default
- page size is 1024 bytes, but it can be desirable for some applications
- to use a larger page size for increased performance. For instance,
- some recent testing shows that increasing the page size can noticeably
- improve performance (5-30%) for certain queries that manipulate a
- large number of pages (e.g. selects without an index, large inserts in
- a transaction, big deletes).
+ # XXX if the number of threads in this thread pool is ever changed, we
+ # should make sure that no operations on the database shuold occur
+ # before the database has been initialized.
+ self._sync_threadpool = ThreadPool(0, 1)
- To adjust the page size, call the pragma immediately after setting the
- key for the first time and each subsequent time that you open the
- database.
-
- Implementation Notes:
+ def _initialize_sync_db(self, opts):
+ """
+ Initialize the Symmetrically-Encrypted document to be synced database,
+ and the queue to communicate with subprocess workers.
- * PRAGMA cipher_page_size must be called after PRAGMA key and before
- the first actual database operation or it will have no effect.
+ :param opts:
+ :type opts: SQLCipherOptions
+ """
+ soledad_assert(opts.sync_db_key is not None)
+ sync_db_path = None
+ if opts.path != ":memory:":
+ sync_db_path = "%s-sync" % opts.path
+ else:
+ sync_db_path = ":memory:"
- * If a non-default value is used PRAGMA cipher_page_size to create a
- database, it must also be called every time that database is opened.
+ # we copy incoming options because the opts object might be used
+ # somewhere else
+ sync_opts = SQLCipherOptions.copy(
+ opts, path=sync_db_path, create=True)
+ self._sync_db = getConnectionPool(
+ sync_opts, extra_queries=self._sync_db_extra_init)
- :param db_handle: A handle to the SQLCipher database.
- :type db_handle: pysqlcipher.Connection
- :param cipher_page_size: The page size.
- :type cipher_page_size: int
+ @property
+ def _sync_db_extra_init(self):
"""
- db_handle.cursor().execute(
- "PRAGMA cipher_page_size = '%d'" % cipher_page_size)
+ Queries for creating tables for the local sync documents db if needed.
+ They are passed as extra initialization to initialize_sqlciphjer_db
- @classmethod
- def _pragma_rekey(cls, db_handle, new_key, raw_key):
- """
- Change the key of an existing encrypted database.
-
- To change the key on an existing encrypted database, it must first be
- unlocked with the current encryption key. Once the database is
- readable and writeable, PRAGMA rekey can be used to re-encrypt every
- page in the database with a new key.
-
- * PRAGMA rekey must be called after PRAGMA key. It can be called at any
- time once the database is readable.
-
- * PRAGMA rekey can not be used to encrypted a standard SQLite
- database! It is only useful for changing the key on an existing
- database.
-
- * Previous versions of SQLCipher provided a PRAGMA rekey_cipher and
- code>PRAGMA rekey_kdf_iter. These are deprecated and should not be
- used. Instead, use sqlcipher_export().
-
- :param db_handle: A handle to the SQLCipher database.
- :type db_handle: pysqlcipher.Connection
- :param new_key: The new key.
- :type new_key: str
- :param raw_key: Whether C{password} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
- :type raw_key: bool
+ :rtype: tuple of strings
"""
- # XXX change key param!
- if raw_key:
- cls._pragma_rekey_raw(db_handle, key)
- else:
- cls._pragma_rekey_passphrase(db_handle, key)
+ maybe_create = "CREATE TABLE IF NOT EXISTS %s (%s)"
+ encr = encdecpool.SyncEncrypterPool
+ decr = encdecpool.SyncDecrypterPool
+ sql_encr_table_query = (maybe_create % (
+ encr.TABLE_NAME, encr.FIELD_NAMES))
+ sql_decr_table_query = (maybe_create % (
+ decr.TABLE_NAME, decr.FIELD_NAMES))
+ return (sql_encr_table_query, sql_decr_table_query)
- @classmethod
- def _pragma_rekey_passphrase(cls, db_handle, passphrase):
+ def sync(self, url, creds=None, defer_decryption=True):
"""
- Change the passphrase for encryption key derivation.
+ Synchronize documents with remote replica exposed at url.
- The key itself can be a passphrase, which is converted to a key using
- PBKDF2 key derivation. The result is used as the encryption key for
- the database.
+ This method defers a sync to a 1-threaded threadpool. The main
+ database initialziation was deferred to that thread during this
+ object's initialization. As there's currently only one thread in that
+ threadpool, the db init was queued before this method was called, so
+ we don't need to actually wait for the db to be ready. If this ever
+ changes, we should add a thread-safe condition to ensure the db is
+ ready before using it.
+
+ :param url: The url of the target replica to sync with.
+ :type url: str
+ :param creds: optional dictionary giving credentials to authorize the
+ operation with the server.
+ :type creds: dict
+ :param defer_decryption:
+ Whether to defer the decryption process using the intermediate
+ database. If False, decryption will be done inline.
+ :type defer_decryption: bool
- :param db_handle: A handle to the SQLCipher database.
- :type db_handle: pysqlcipher.Connection
- :param passphrase: The passphrase used to derive the encryption key.
- :type passphrase: str
+ :return:
+ A Deferred, that will fire with the local generation (type `int`)
+ before the synchronisation was performed.
+ :rtype: Deferred
"""
- db_handle.cursor().execute("PRAGMA rekey = '%s'" % passphrase)
+ # the following context manager blocks until the syncing lock can be
+ # acquired.
+ with self._syncer(url, creds=creds) as syncer:
+ # XXX could mark the critical section here...
+ return syncer.sync(defer_decryption=defer_decryption)
- @classmethod
- def _pragma_rekey_raw(cls, db_handle, key):
+ @contextmanager
+ def _syncer(self, url, creds=None):
"""
- Change the raw hexadecimal encryption key.
+ Accesor for synchronizer.
- It is possible to specify an exact byte sequence using a blob literal.
- With this method, it is the calling application's responsibility to
- ensure that the data provided is a 64 character hex string, which will
- be converted directly to 32 bytes (256 bits) of key data.
+ As we reuse the same synchronizer for every sync, there can be only
+ one instance synchronizing the same database replica at the same time.
+ Because of that, this method blocks until the syncing lock can be
+ acquired.
- :param db_handle: A handle to the SQLCipher database.
- :type db_handle: pysqlcipher.Connection
- :param key: A 64 character hex string.
- :type key: str
+ :param creds: optional dictionary giving credentials to authorize the
+ operation with the server.
+ :type creds: dict
"""
- if not all(c in string.hexdigits for c in key):
- raise NotAnHexString(key)
- # XXX change passphrase param!
- db_handle.cursor().execute('PRAGMA rekey = "x\'%s"' % passphrase)
+ with self.syncing_lock[self._path]:
+ syncer = self._get_syncer(url, creds=creds)
+ yield syncer
- @classmethod
- def _pragma_synchronous_off(cls, db_handle):
- """
- Change the setting of the "synchronous" flag to OFF.
- """
- logger.debug("SQLCIPHER: SETTING SYNCHRONOUS OFF")
- db_handle.cursor().execute('PRAGMA synchronous=OFF')
+ @property
+ def syncing(self):
+ lock = self.syncing_lock[self._path]
+ acquired_lock = lock.acquire(False)
+ if acquired_lock is False:
+ return True
+ lock.release()
+ return False
- @classmethod
- def _pragma_synchronous_normal(cls, db_handle):
+ def _get_syncer(self, url, creds=None):
"""
- Change the setting of the "synchronous" flag to NORMAL.
+ Get a synchronizer for ``url`` using ``creds``.
+
+ :param url: The url of the target replica to sync with.
+ :type url: str
+ :param creds: optional dictionary giving credentials.
+ to authorize the operation with the server.
+ :type creds: dict
+
+ :return: A synchronizer.
+ :rtype: Synchronizer
"""
- logger.debug("SQLCIPHER: SETTING SYNCHRONOUS NORMAL")
- db_handle.cursor().execute('PRAGMA synchronous=NORMAL')
+ # we want to store at most one syncer for each url, so we also store a
+ # hash of the connection credentials and replace the stored syncer for
+ # a certain url if credentials have changed.
+ h = sha256(json.dumps([url, creds])).hexdigest()
+ cur_h, syncer = self._syncers.get(url, (None, None))
+ if syncer is None or h != cur_h:
+ syncer = SoledadSynchronizer(
+ self,
+ SoledadHTTPSyncTarget(
+ url,
+ # XXX is the replica_uid ready?
+ self._replica_uid,
+ creds=creds,
+ crypto=self._crypto,
+ cert_file=self._cert_file,
+ sync_db=self._sync_db,
+ sync_enc_pool=self._sync_enc_pool))
+ self._syncers[url] = (h, syncer)
+ # in order to reuse the same synchronizer multiple times we have to
+ # reset its state (i.e. the number of documents received from target
+ # and inserted in the local replica).
+ syncer.num_inserted = 0
+ return syncer
- @classmethod
- def _pragma_mem_temp_store(cls, db_handle):
+ #
+ # Symmetric encryption of syncing docs
+ #
+
+ def get_generation(self):
+ # FIXME
+ # XXX this SHOULD BE a callback
+ return self._get_generation()
+
+ def finalClose(self):
"""
- Use a in-memory store for temporary tables.
+ This should only be called by the shutdown trigger.
"""
- logger.debug("SQLCIPHER: SETTING TEMP_STORE MEMORY")
- db_handle.cursor().execute('PRAGMA temp_store=MEMORY')
+ self.shutdownID = None
+ self._sync_threadpool.stop()
+ self.running = False
- @classmethod
- def _pragma_write_ahead_logging(cls, db_handle):
+ def close(self):
"""
- Enable write-ahead logging, and set the autocheckpoint to 50 pages.
-
- Setting the autocheckpoint to a small value, we make the reads not
- suffer too much performance degradation.
-
- From the sqlite docs:
-
- "There is a tradeoff between average read performance and average write
- performance. To maximize the read performance, one wants to keep the
- WAL as small as possible and hence run checkpoints frequently, perhaps
- as often as every COMMIT. To maximize write performance, one wants to
- amortize the cost of each checkpoint over as many writes as possible,
- meaning that one wants to run checkpoints infrequently and let the WAL
- grow as large as possible before each checkpoint. The decision of how
- often to run checkpoints may therefore vary from one application to
- another depending on the relative read and write performance
- requirements of the application. The default strategy is to run a
- checkpoint once the WAL reaches 1000 pages"
+ Close the syncer and syncdb orderly
"""
- logger.debug("SQLCIPHER: SETTING WRITE-AHEAD LOGGING")
- db_handle.cursor().execute('PRAGMA journal_mode=WAL')
- # The optimum value can still use a little bit of tuning, but we favor
- # small sizes of the WAL file to get fast reads, since we assume that
- # the writes will be quick enough to not block too much.
+ # close all open syncers
+ for url in self._syncers.keys():
+ del self._syncers[url]
- # TODO
- # As a further improvement, we might want to set autocheckpoint to 0
- # here and do the checkpoints manually in a separate thread, to avoid
- # any blocks in the main thread (we should run a loopingcall from here)
- db_handle.cursor().execute('PRAGMA wal_autocheckpoint=50')
+ # stop the encryption pool
+ if self._sync_enc_pool is not None:
+ self._sync_enc_pool.close()
+ self._sync_enc_pool = None
- # Extra query methods: extensions to the base sqlite implmentation.
+ # close the sync database
+ if self._sync_db is not None:
+ self._sync_db.close()
+ self._sync_db = None
- def get_count_from_index(self, index_name, *key_values):
- """
- Returns the count for a given combination of index_name
- and key values.
- Extension method made from similar methods in u1db version 13.09
+class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase):
+ """
+ A very simple wrapper for u1db around sqlcipher backend.
- :param index_name: The index to query
- :type index_name: str
- :param key_values: values to match. eg, if you have
- an index with 3 fields then you would have:
- get_from_index(index_name, val1, val2, val3)
- :type key_values: tuple
- :return: count.
- :rtype: int
- """
- c = self._db_handle.cursor()
- definition = self._get_index_definition(index_name)
+ Instead of initializing the database on the fly, it just uses an existing
+ connection that is passed to it in the initializer.
- if len(key_values) != len(definition):
- raise u1db_errors.InvalidValueForIndex()
- tables = ["document_fields d%d" % i for i in range(len(definition))]
- novalue_where = ["d.doc_id = d%d.doc_id"
- " AND d%d.field_name = ?"
- % (i, i) for i in range(len(definition))]
- exact_where = [novalue_where[i]
- + (" AND d%d.value = ?" % (i,))
- for i in range(len(definition))]
- args = []
- where = []
- for idx, (field, value) in enumerate(zip(definition, key_values)):
- args.append(field)
- where.append(exact_where[idx])
- args.append(value)
+ It can be used in tests and debug runs to initialize the adbapi with plain
+ sqlite connections, decoupled from the sqlcipher layer.
+ """
- tables = ["document_fields d%d" % i for i in range(len(definition))]
- statement = (
- "SELECT COUNT(*) FROM document d, %s WHERE %s " % (
- ', '.join(tables),
- ' AND '.join(where),
- ))
- try:
- c.execute(statement, tuple(args))
- except dbapi2.OperationalError, e:
- raise dbapi2.OperationalError(
- str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args))
- res = c.fetchall()
- return res[0][0]
+ def __init__(self, conn):
+ self._db_handle = conn
+ self._real_replica_uid = None
+ self._ensure_schema()
+ self._factory = u1db.Document
- def close(self):
- """
- Close db_handle and close syncer.
- """
- logger.debug("Sqlcipher backend: closing")
- if self._sync_watcher is not None:
- self._sync_watcher.stop()
- self._sync_watcher.shutdown()
- for url in self._syncers:
- _, syncer = self._syncers[url]
- syncer.close()
- if self._sync_enc_pool is not None:
- self._sync_enc_pool.close()
- if self._db_handle is not None:
- self._db_handle.close()
- @property
- def replica_uid(self):
- return self._get_replica_uid()
+class SoledadSQLCipherWrapper(SQLCipherDatabase):
+ """
+ A wrapper for u1db that uses the Soledad-extended sqlcipher backend.
+
+ Instead of initializing the database on the fly, it just uses an existing
+ connection that is passed to it in the initializer.
+
+ It can be used from adbapi to initialize a soledad database after
+ getting a regular connection to a sqlcipher database.
+ """
+ def __init__(self, conn):
+ self._db_handle = conn
+ self._real_replica_uid = None
+ self._ensure_schema()
+ self.set_document_factory(soledad_doc_factory)
+ self._prime_replica_uid()
+
+
+def _assert_db_is_encrypted(opts):
+ """
+ Assert that the sqlcipher file contains an encrypted database.
+
+ When opening an existing database, PRAGMA key will not immediately
+ throw an error if the key provided is incorrect. To test that the
+ database can be successfully opened with the provided key, it is
+ necessary to perform some operation on the database (i.e. read from
+ it) and confirm it is success.
+
+ The easiest way to do this is select off the sqlite_master table,
+ which will attempt to read the first page of the database and will
+ parse the schema.
+
+ :param opts:
+ """
+ # We try to open an encrypted database with the regular u1db
+ # backend should raise a DatabaseError exception.
+ # If the regular backend succeeds, then we need to stop because
+ # the database was not properly initialized.
+ try:
+ sqlite_backend.SQLitePartialExpandDatabase(opts.path)
+ except sqlcipher_dbapi2.DatabaseError:
+ # assert that we can access it using SQLCipher with the given
+ # key
+ dummy_query = ('SELECT count(*) FROM sqlite_master',)
+ initialize_sqlcipher_db(opts, on_init=dummy_query)
+ else:
+ raise DatabaseIsNotEncrypted()
+
+#
+# Exceptions
+#
+class DatabaseIsNotEncrypted(Exception):
+ """
+ Exception raised when trying to open non-encrypted databases.
+ """
+ pass
+
+
+def soledad_doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False,
+ syncable=True):
+ """
+ Return a default Soledad Document.
+ Used in the initialization for SQLCipherDatabase
+ """
+ return SoledadDocument(doc_id=doc_id, rev=rev, json=json,
+ has_conflicts=has_conflicts, syncable=syncable)
+
sqlite_backend.SQLiteDatabase.register_implementation(SQLCipherDatabase)
+
+
+#
+# twisted.enterprise.adbapi SQLCipher implementation
+#
+
+SQLCIPHER_CONNECTION_TIMEOUT = 10
+
+
+def getConnectionPool(opts, extra_queries=None):
+ openfun = partial(
+ pragmas.set_init_pragmas,
+ opts=opts,
+ extra_queries=extra_queries)
+ return SQLCipherConnectionPool(
+ database=opts.path,
+ check_same_thread=False,
+ cp_openfun=openfun,
+ timeout=SQLCIPHER_CONNECTION_TIMEOUT)
+
+
+class SQLCipherConnection(adbapi.Connection):
+ pass
+
+
+class SQLCipherTransaction(adbapi.Transaction):
+ pass
+
+
+class SQLCipherConnectionPool(adbapi.ConnectionPool):
+
+ connectionFactory = SQLCipherConnection
+ transactionFactory = SQLCipherTransaction
+
+ def __init__(self, *args, **kwargs):
+ adbapi.ConnectionPool.__init__(
+ self, "pysqlcipher.dbapi2", *args, **kwargs)
diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py
index 5d545a77..53172f31 100644
--- a/client/src/leap/soledad/client/sync.py
+++ b/client/src/leap/soledad/client/sync.py
@@ -14,26 +14,12 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
"""
Soledad synchronization utilities.
-
-
-Extend u1db Synchronizer with the ability to:
-
- * Defer the update of the known replica uid until all the decryption of
- the incoming messages has been processed.
-
- * Be interrupted and recovered.
"""
-
-
-import json
-
import logging
-import traceback
-from threading import Lock
+
+from twisted.internet import defer
from u1db import errors
from u1db.sync import Synchronizer
@@ -54,15 +40,8 @@ class SoledadSynchronizer(Synchronizer):
Also modified to allow for interrupting the synchronization process.
"""
- syncing_lock = Lock()
-
- def stop(self):
- """
- Stop the current sync in progress.
- """
- self.sync_target.stop()
-
- def sync(self, autocreate=False, defer_decryption=True):
+ @defer.inlineCallbacks
+ def sync(self, defer_decryption=True):
"""
Synchronize documents between source and target.
@@ -74,45 +53,22 @@ class SoledadSynchronizer(Synchronizer):
This is done to allow the ongoing parallel decryption of the incoming
docs to proceed without `InvalidGeneration` conflicts.
- :param autocreate: Whether the target replica should be created or not.
- :type autocreate: bool
:param defer_decryption: Whether to defer the decryption process using
the intermediate database. If False,
decryption will be done inline.
:type defer_decryption: bool
- """
- self.syncing_lock.acquire()
- try:
- return self._sync(autocreate=autocreate,
- defer_decryption=defer_decryption)
- except Exception:
- # re-raising the exceptions to let syqlcipher.sync catch them
- # (and re-create the syncer instance if needed)
- raise
- finally:
- self.release_syncing_lock()
-
- def _sync(self, autocreate=False, defer_decryption=True):
- """
- Helper function, called from the main `sync` method.
- See `sync` docstring.
+
+ :return: A deferred which will fire after the sync has finished.
+ :rtype: twisted.internet.defer.Deferred
"""
sync_target = self.sync_target
# get target identifier, its current generation,
# and its last-seen database generation for this source
ensure_callback = None
- try:
- (self.target_replica_uid, target_gen, target_trans_id,
- target_my_gen, target_my_trans_id) = \
- sync_target.get_sync_info(self.source._replica_uid)
- except errors.DatabaseDoesNotExist:
- if not autocreate:
- raise
- # will try to ask sync_exchange() to create the db
- self.target_replica_uid = None
- target_gen, target_trans_id = (0, '')
- target_my_gen, target_my_trans_id = (0, '')
+ (self.target_replica_uid, target_gen, target_trans_id,
+ target_my_gen, target_my_trans_id) = yield \
+ sync_target.get_sync_info(self.source._replica_uid)
logger.debug(
"Soledad target sync info:\n"
@@ -120,9 +76,10 @@ class SoledadSynchronizer(Synchronizer):
" target generation: %d\n"
" target trans id: %s\n"
" target my gen: %d\n"
- " target my trans_id: %s"
+ " target my trans_id: %s\n"
+ " source replica_uid: %s\n"
% (self.target_replica_uid, target_gen, target_trans_id,
- target_my_gen, target_my_trans_id))
+ target_my_gen, target_my_trans_id, self.source._replica_uid))
# make sure we'll have access to target replica uid once it exists
if self.target_replica_uid is None:
@@ -140,7 +97,7 @@ class SoledadSynchronizer(Synchronizer):
# what's changed since that generation and this current gen
my_gen, _, changes = self.source.whats_changed(target_my_gen)
- logger.debug("Soledad sync: there are %d documents to send." \
+ logger.debug("Soledad sync: there are %d documents to send."
% len(changes))
# get source last-seen database generation for the target
@@ -152,15 +109,15 @@ class SoledadSynchronizer(Synchronizer):
self.target_replica_uid)
logger.debug(
"Soledad source sync info:\n"
- " source target gen: %d\n"
- " source target trans_id: %s"
+ " last target gen known to source: %d\n"
+ " last target trans_id known to source: %s"
% (target_last_known_gen, target_last_known_trans_id))
# validate transaction ids
if not changes and target_last_known_gen == target_gen:
if target_trans_id != target_last_known_trans_id:
raise errors.InvalidTransactionId
- return my_gen
+ defer.returnValue(my_gen)
# prepare to send all the changed docs
changed_doc_ids = [doc_id for doc_id, _, _ in changes]
@@ -175,40 +132,26 @@ class SoledadSynchronizer(Synchronizer):
# exchange documents and try to insert the returned ones with
# the target, return target synced-up-to gen.
- #
- # The sync_exchange method may be interrupted, in which case it will
- # return a tuple of Nones.
- try:
- new_gen, new_trans_id = sync_target.sync_exchange(
- docs_by_generation, self.source._replica_uid,
- target_last_known_gen, target_last_known_trans_id,
- self._insert_doc_from_target, ensure_callback=ensure_callback,
- defer_decryption=defer_decryption)
- logger.debug(
- "Soledad source sync info after sync exchange:\n"
- " source target gen: %d\n"
- " source target trans_id: %s"
- % (new_gen, new_trans_id))
- info = {
- "target_replica_uid": self.target_replica_uid,
- "new_gen": new_gen,
- "new_trans_id": new_trans_id,
- "my_gen": my_gen
- }
- self._syncing_info = info
- if defer_decryption and not sync_target.has_syncdb():
- logger.debug("Sync target has no valid sync db, "
- "aborting defer_decryption")
- defer_decryption = False
- self.complete_sync()
- except Exception as e:
- logger.error("Soledad sync error: %s" % str(e))
- logger.error(traceback.format_exc())
- sync_target.stop()
- finally:
- sync_target.close()
-
- return my_gen
+ new_gen, new_trans_id = yield sync_target.sync_exchange(
+ docs_by_generation, self.source._replica_uid,
+ target_last_known_gen, target_last_known_trans_id,
+ self._insert_doc_from_target, ensure_callback=ensure_callback,
+ defer_decryption=defer_decryption)
+ logger.debug(
+ "Soledad source sync info after sync exchange:\n"
+ " source known target gen: %d\n"
+ " source known target trans_id: %s"
+ % (new_gen, new_trans_id))
+ info = {
+ "target_replica_uid": self.target_replica_uid,
+ "new_gen": new_gen,
+ "new_trans_id": new_trans_id,
+ "my_gen": my_gen
+ }
+ self._syncing_info = info
+ yield self.complete_sync()
+
+ defer.returnValue(my_gen)
def complete_sync(self):
"""
@@ -216,6 +159,9 @@ class SoledadSynchronizer(Synchronizer):
(a) record last known generation and transaction uid for the remote
replica, and
(b) make target aware of our current reached generation.
+
+ :return: A deferred which will fire when the sync has been completed.
+ :rtype: twisted.internet.defer.Deferred
"""
logger.debug("Completing deferred last step in SYNC...")
@@ -226,37 +172,23 @@ class SoledadSynchronizer(Synchronizer):
info["target_replica_uid"], info["new_gen"], info["new_trans_id"])
# if gapless record current reached generation with target
- self._record_sync_info_with_the_target(info["my_gen"])
+ return self._record_sync_info_with_the_target(info["my_gen"])
- @property
- def syncing(self):
+ def _record_sync_info_with_the_target(self, start_generation):
"""
- Return True if a sync is ongoing, False otherwise.
- :rtype: bool
- """
- # XXX FIXME we need some mechanism for timeout: should cleanup and
- # release if something in the syncdb-decrypt goes wrong. we could keep
- # track of the release date and cleanup unrealistic sync entries after
- # some time.
- locked = self.syncing_lock.locked()
- return locked
-
- def release_syncing_lock(self):
- """
- Release syncing lock if it's locked.
- """
- if self.syncing_lock.locked():
- self.syncing_lock.release()
+ Store local replica metadata in server.
- def close(self):
- """
- Close sync target pool of workers.
- """
- self.release_syncing_lock()
- self.sync_target.close()
+ :param start_generation: The local generation when the sync was
+ started.
+ :type start_generation: int
- def __del__(self):
- """
- Cleanup: release lock.
+ :return: A deferred which will fire when the operation has been
+ completed.
+ :rtype: twisted.internet.defer.Deferred
"""
- self.release_syncing_lock()
+ cur_gen, trans_id = self.source._get_generation_info()
+ if (cur_gen == start_generation + self.num_inserted
+ and self.num_inserted > 0):
+ return self.sync_target.record_sync_info(
+ self.source._replica_uid, cur_gen, trans_id)
+ return defer.succeed(None)
diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py
deleted file mode 100644
index 1eb84e64..00000000
--- a/client/src/leap/soledad/client/target.py
+++ /dev/null
@@ -1,1469 +0,0 @@
-# -*- coding: utf-8 -*-
-# target.py
-# Copyright (C) 2013, 2014 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-A U1DB backend for encrypting data before sending to server and decrypting
-after receiving.
-"""
-
-
-import cStringIO
-import gzip
-import logging
-import re
-import urllib
-import threading
-
-from collections import defaultdict
-from time import sleep
-from uuid import uuid4
-
-import simplejson as json
-from taskthread import TimerTask
-from u1db import errors
-from u1db.remote import utils, http_errors
-from u1db.remote.http_target import HTTPSyncTarget
-from u1db.remote.http_client import _encode_query_parameter, HTTPClientBase
-from zope.proxy import ProxyBase
-from zope.proxy import sameProxiedObjects, setProxiedObject
-
-from leap.soledad.common.document import SoledadDocument
-from leap.soledad.client.auth import TokenBasedAuth
-from leap.soledad.client.crypto import is_symmetrically_encrypted
-from leap.soledad.client.crypto import encrypt_doc, decrypt_doc
-from leap.soledad.client.crypto import SyncEncrypterPool, SyncDecrypterPool
-from leap.soledad.client.events import SOLEDAD_SYNC_SEND_STATUS
-from leap.soledad.client.events import SOLEDAD_SYNC_RECEIVE_STATUS
-from leap.soledad.client.events import signal
-
-
-logger = logging.getLogger(__name__)
-
-
-def _gunzip(data):
- """
- Uncompress data that is gzipped.
-
- :param data: gzipped data
- :type data: basestring
- """
- buffer = cStringIO.StringIO()
- buffer.write(data)
- buffer.seek(0)
- try:
- data = gzip.GzipFile(mode='r', fileobj=buffer).read()
- except Exception:
- logger.warning("Error while decrypting gzipped data")
- buffer.close()
- return data
-
-
-class PendingReceivedDocsSyncError(Exception):
- pass
-
-
-class DocumentSyncerThread(threading.Thread):
- """
- A thread that knowns how to either send or receive a document during the
- sync process.
- """
-
- def __init__(self, doc_syncer, release_method, failed_method,
- idx, total, last_request_lock=None, last_callback_lock=None):
- """
- Initialize a new syncer thread.
-
- :param doc_syncer: A document syncer.
- :type doc_syncer: HTTPDocumentSyncer
- :param release_method: A method to be called when finished running.
- :type release_method: callable(DocumentSyncerThread)
- :param failed_method: A method to be called when we failed.
- :type failed_method: callable(DocumentSyncerThread)
- :param idx: The index count of the current operation.
- :type idx: int
- :param total: The total number of operations.
- :type total: int
- :param last_request_lock: A lock to wait for before actually performing
- the request.
- :type last_request_lock: threading.Lock
- :param last_callback_lock: A lock to wait for before actually running
- the success callback.
- :type last_callback_lock: threading.Lock
- """
- threading.Thread.__init__(self)
- self._doc_syncer = doc_syncer
- self._release_method = release_method
- self._failed_method = failed_method
- self._idx = idx
- self._total = total
- self._last_request_lock = last_request_lock
- self._last_callback_lock = last_callback_lock
- self._response = None
- self._exception = None
- self._result = None
- self._success = False
- # a lock so we can signal when we're finished
- self._request_lock = threading.Lock()
- self._request_lock.acquire()
- self._callback_lock = threading.Lock()
- self._callback_lock.acquire()
- # make thread interruptable
- self._stopped = None
- self._stop_lock = threading.Lock()
-
- def run(self):
- """
- Run the HTTP request and store results.
-
- This method will block and wait for an eventual previous operation to
- finish before actually performing the request. It also traps any
- exception and register any failure with the request.
- """
- with self._stop_lock:
- if self._stopped is None:
- self._stopped = False
- else:
- return
-
- # eventually wait for the previous thread to finish
- if self._last_request_lock is not None:
- self._last_request_lock.acquire()
-
- # bail out in case we've been interrupted
- if self.stopped is True:
- return
-
- try:
- self._response = self._doc_syncer.do_request()
- self._request_lock.release()
-
- # run success callback
- if self._doc_syncer.success_callback is not None:
-
- # eventually wait for callback lock release
- if self._last_callback_lock is not None:
- self._last_callback_lock.acquire()
-
- # bail out in case we've been interrupted
- if self._stopped is True:
- return
-
- self._result = self._doc_syncer.success_callback(
- self._idx, self._total, self._response)
- self._success = True
- doc_syncer = self._doc_syncer
- self._release_method(self, doc_syncer)
- self._doc_syncer = None
- # let next thread executed its callback
- self._callback_lock.release()
-
- # trap any exception and signal failure
- except Exception as e:
- self._exception = e
- self._success = False
- # run failure callback
- if self._doc_syncer.failure_callback is not None:
-
- # eventually wait for callback lock release
- if self._last_callback_lock is not None:
- self._last_callback_lock.acquire()
-
- # bail out in case we've been interrupted
- if self.stopped is True:
- return
-
- self._doc_syncer.failure_callback(
- self._idx, self._total, self._exception)
-
- self._failed_method(self)
- # we do not release the callback lock here because we
- # failed and so we don't want other threads to succeed.
-
- @property
- def doc_syncer(self):
- return self._doc_syncer
-
- @property
- def response(self):
- return self._response
-
- @property
- def exception(self):
- return self._exception
-
- @property
- def callback_lock(self):
- return self._callback_lock
-
- @property
- def request_lock(self):
- return self._request_lock
-
- @property
- def success(self):
- return self._success
-
- def stop(self):
- with self._stop_lock:
- self._stopped = True
-
- @property
- def stopped(self):
- with self._stop_lock:
- return self._stopped
-
- @property
- def result(self):
- return self._result
-
-
-class DocumentSyncerPool(object):
- """
- A pool of reusable document syncers.
- """
-
- POOL_SIZE = 10
- """
- The maximum amount of syncer threads running at the same time.
- """
-
- def __init__(self, raw_url, raw_creds, query_string, headers,
- ensure_callback, stop_method):
- """
- Initialize the document syncer pool.
-
- :param raw_url: The complete raw URL for the HTTP request.
- :type raw_url: str
- :param raw_creds: The credentials for the HTTP request.
- :type raw_creds: dict
- :param query_string: The query string for the HTTP request.
- :type query_string: str
- :param headers: The headers for the HTTP request.
- :type headers: dict
- :param ensure_callback: A callback to ensure we have the correct
- target_replica_uid, if it was just created.
- :type ensure_callback: callable
-
- """
- # save syncer params
- self._raw_url = raw_url
- self._raw_creds = raw_creds
- self._query_string = query_string
- self._headers = headers
- self._ensure_callback = ensure_callback
- self._stop_method = stop_method
- # pool attributes
- self._failures = False
- self._semaphore_pool = threading.BoundedSemaphore(
- DocumentSyncerPool.POOL_SIZE)
- self._pool_access_lock = threading.Lock()
- self._doc_syncers = []
- self._threads = []
-
- def new_syncer_thread(self, idx, total, last_request_lock=None,
- last_callback_lock=None):
- """
- Yield a new document syncer thread.
-
- :param idx: The index count of the current operation.
- :type idx: int
- :param total: The total number of operations.
- :type total: int
- :param last_request_lock: A lock to wait for before actually performing
- the request.
- :type last_request_lock: threading.Lock
- :param last_callback_lock: A lock to wait for before actually running
- the success callback.
- :type last_callback_lock: threading.Lock
- """
- t = None
- # wait for available threads
- self._semaphore_pool.acquire()
- with self._pool_access_lock:
- if self._failures is True:
- return None
- # get a syncer
- doc_syncer = self._get_syncer()
- # we rely on DocumentSyncerThread.run() to release the lock using
- # self.release_syncer so we can launch a new thread.
- t = DocumentSyncerThread(
- doc_syncer, self.release_syncer, self.cancel_threads,
- idx, total,
- last_request_lock=last_request_lock,
- last_callback_lock=last_callback_lock)
- self._threads.append(t)
- return t
-
- def _failed(self):
- with self._pool_access_lock:
- self._failures = True
-
- @property
- def failures(self):
- return self._failures
-
- def _get_syncer(self):
- """
- Get a document syncer from the pool.
-
- This method will create a new syncer whenever there is no syncer
- available in the pool.
-
- :return: A syncer.
- :rtype: HTTPDocumentSyncer
- """
- syncer = None
- # get an available syncer or create a new one
- try:
- syncer = self._doc_syncers.pop()
- except IndexError:
- syncer = HTTPDocumentSyncer(
- self._raw_url, self._raw_creds, self._query_string,
- self._headers, self._ensure_callback)
- return syncer
-
- def release_syncer(self, syncer_thread, doc_syncer):
- """
- Return a syncer to the pool after use and check for any failures.
-
- :param syncer: The syncer to be returned to the pool.
- :type syncer: HTTPDocumentSyncer
- """
- with self._pool_access_lock:
- self._doc_syncers.append(doc_syncer)
- if syncer_thread.success is True:
- self._threads.remove(syncer_thread)
- self._semaphore_pool.release()
-
- def cancel_threads(self, calling_thread):
- """
- Stop all threads in the pool.
- """
- # stop sync
- self._stop_method()
- stopped = []
- # stop all threads
- logger.warning("Soledad sync: cancelling sync threads...")
- with self._pool_access_lock:
- self._failures = True
- while self._threads:
- t = self._threads.pop(0)
- t.stop()
- self._doc_syncers.append(t.doc_syncer)
- stopped.append(t)
- # release locks and join
- while stopped:
- t = stopped.pop(0)
- t.request_lock.acquire(False) # just in case
- t.request_lock.release()
- t.callback_lock.acquire(False) # just in case
- t.callback_lock.release()
- logger.warning("Soledad sync: cancelled sync threads.")
-
- def cleanup(self):
- """
- Close and remove any syncers from the pool.
- """
- with self._pool_access_lock:
- while self._doc_syncers:
- syncer = self._doc_syncers.pop()
- syncer.close()
- del syncer
-
-
-class HTTPDocumentSyncer(HTTPClientBase, TokenBasedAuth):
-
- def __init__(self, raw_url, creds, query_string, headers, ensure_callback):
- """
- Initialize the client.
-
- :param raw_url: The raw URL of the target HTTP server.
- :type raw_url: str
- :param creds: Authentication credentials.
- :type creds: dict
- :param query_string: The query string for the HTTP request.
- :type query_string: str
- :param headers: The headers for the HTTP request.
- :type headers: dict
- :param ensure_callback: A callback to ensure we have the correct
- target_replica_uid, if it was just created.
- :type ensure_callback: callable
- """
- HTTPClientBase.__init__(self, raw_url, creds=creds)
- # info needed to perform the request
- self._query_string = query_string
- self._headers = headers
- self._ensure_callback = ensure_callback
- # the actual request method
- self._request_method = None
- self._success_callback = None
- self._failure_callback = None
-
- def _reset(self):
- """
- Reset this document syncer so we can reuse it.
- """
- self._request_method = None
- self._success_callback = None
- self._failure_callback = None
- self._request_method = None
-
- def set_request_method(self, method, *args, **kwargs):
- """
- Set the actual method to perform the request.
-
- :param method: Either 'get' or 'put'.
- :type method: str
- :param args: Arguments for the request method.
- :type args: list
- :param kwargs: Keyworded arguments for the request method.
- :type kwargs: dict
- """
- self._reset()
- # resolve request method
- if method is 'get':
- self._request_method = self._get_doc
- elif method is 'put':
- self._request_method = self._put_doc
- else:
- raise Exception
- # store request method args
- self._args = args
- self._kwargs = kwargs
-
- def set_success_callback(self, callback):
- self._success_callback = callback
-
- def set_failure_callback(self, callback):
- self._failure_callback = callback
-
- @property
- def success_callback(self):
- return self._success_callback
-
- @property
- def failure_callback(self):
- return self._failure_callback
-
- def do_request(self):
- """
- Actually perform the request.
-
- :return: The body and headers of the response.
- :rtype: tuple
- """
- self._ensure_connection()
- args = self._args
- kwargs = self._kwargs
- return self._request_method(*args, **kwargs)
-
- def _request(self, method, url_parts, params=None, body=None,
- content_type=None):
- """
- Perform an HTTP request.
-
- :param method: The HTTP request method.
- :type method: str
- :param url_parts: A list representing the request path.
- :type url_parts: list
- :param params: Parameters for the URL query string.
- :type params: dict
- :param body: The body of the request.
- :type body: str
- :param content-type: The content-type of the request.
- :type content-type: str
-
- :return: The body and headers of the response.
- :rtype: tuple
-
- :raise errors.Unavailable: Raised after a number of unsuccesful
- request attempts.
- :raise Exception: Raised for any other exception ocurring during the
- request.
- """
-
- self._ensure_connection()
- unquoted_url = url_query = self._url.path
- if url_parts:
- if not url_query.endswith('/'):
- url_query += '/'
- unquoted_url = url_query
- url_query += '/'.join(urllib.quote(part, safe='')
- for part in url_parts)
- # oauth performs its own quoting
- unquoted_url += '/'.join(url_parts)
- encoded_params = {}
- if params:
- for key, value in params.items():
- key = unicode(key).encode('utf-8')
- encoded_params[key] = _encode_query_parameter(value)
- url_query += ('?' + urllib.urlencode(encoded_params))
- if body is not None and not isinstance(body, basestring):
- body = json.dumps(body)
- content_type = 'application/json'
- headers = {}
- if content_type:
- headers['content-type'] = content_type
-
- # Patched: We would like to receive gzip pretty please
- # ----------------------------------------------------
- headers['accept-encoding'] = "gzip"
- # ----------------------------------------------------
-
- headers.update(
- self._sign_request(method, unquoted_url, encoded_params))
-
- for delay in self._delays:
- try:
- self._conn.request(method, url_query, body, headers)
- return self._response()
- except errors.Unavailable, e:
- sleep(delay)
- raise e
-
- def _response(self):
- """
- Return the response of the (possibly gzipped) HTTP request.
-
- :return: The body and headers of the response.
- :rtype: tuple
- """
- resp = self._conn.getresponse()
- body = resp.read()
- headers = dict(resp.getheaders())
-
- # Patched: We would like to decode gzip
- # ----------------------------------------------------
- encoding = headers.get('content-encoding', '')
- if "gzip" in encoding:
- body = _gunzip(body)
- # ----------------------------------------------------
-
- if resp.status in (200, 201):
- return body, headers
- elif resp.status in http_errors.ERROR_STATUSES:
- try:
- respdic = json.loads(body)
- except ValueError:
- pass
- else:
- self._error(respdic)
- # special case
- if resp.status == 503:
- raise errors.Unavailable(body, headers)
- raise errors.HTTPError(resp.status, body, headers)
-
- def _prepare(self, comma, entries, **dic):
- """
- Prepare an entry to be sent through a syncing POST request.
-
- :param comma: A string to be prepended to the current entry.
- :type comma: str
- :param entries: A list of entries accumulated to be sent on the
- request.
- :type entries: list
- :param dic: The data to be included in this entry.
- :type dic: dict
-
- :return: The size of the prepared entry.
- :rtype: int
- """
- entry = comma + '\r\n' + json.dumps(dic)
- entries.append(entry)
- return len(entry)
-
- def _init_post_request(self, action, content_length):
- """
- Initiate a syncing POST request.
-
- :param url: The syncing URL.
- :type url: str
- :param action: The syncing action, either 'get' or 'receive'.
- :type action: str
- :param headers: The initial headers to be sent on this request.
- :type headers: dict
- :param content_length: The content-length of the request.
- :type content_length: int
- """
- self._conn.putrequest('POST', self._query_string)
- self._conn.putheader(
- 'content-type', 'application/x-soledad-sync-%s' % action)
- for header_name, header_value in self._headers:
- self._conn.putheader(header_name, header_value)
- self._conn.putheader('accept-encoding', 'gzip')
- self._conn.putheader('content-length', str(content_length))
- self._conn.endheaders()
-
- def _get_doc(self, received, sync_id, last_known_generation,
- last_known_trans_id):
- """
- Get a sync document from server by means of a POST request.
-
- :param received: The number of documents already received in the
- current sync session.
- :type received: int
- :param sync_id: The id for the current sync session.
- :type sync_id: str
- :param last_known_generation: Target's last known generation.
- :type last_known_generation: int
- :param last_known_trans_id: Target's last known transaction id.
- :type last_known_trans_id: str
-
- :return: The body and headers of the response.
- :rtype: tuple
- """
- entries = ['[']
- size = 1
- # add remote replica metadata to the request
- size += self._prepare(
- '', entries,
- last_known_generation=last_known_generation,
- last_known_trans_id=last_known_trans_id,
- sync_id=sync_id,
- ensure=self._ensure_callback is not None)
- # inform server of how many documents have already been received
- size += self._prepare(
- ',', entries, received=received)
- entries.append('\r\n]')
- size += len(entries[-1])
- # send headers
- self._init_post_request('get', size)
- # get document
- for entry in entries:
- self._conn.send(entry)
- return self._response()
-
- def _put_doc(self, sync_id, last_known_generation, last_known_trans_id,
- id, rev, content, gen, trans_id, number_of_docs, doc_idx):
- """
- Put a sync document on server by means of a POST request.
-
- :param sync_id: The id for the current sync session.
- :type sync_id: str
- :param last_known_generation: Target's last known generation.
- :type last_known_generation: int
- :param last_known_trans_id: Target's last known transaction id.
- :type last_known_trans_id: str
- :param id: The document id.
- :type id: str
- :param rev: The document revision.
- :type rev: str
- :param content: The serialized document content.
- :type content: str
- :param gen: The generation of the modification of the document.
- :type gen: int
- :param trans_id: The transaction id of the modification of the
- document.
- :type trans_id: str
- :param number_of_docs: The total amount of documents sent on this sync
- session.
- :type number_of_docs: int
- :param doc_idx: The index of the current document being sent.
- :type doc_idx: int
-
- :return: The body and headers of the response.
- :rtype: tuple
- """
- # prepare to send the document
- entries = ['[']
- size = 1
- # add remote replica metadata to the request
- size += self._prepare(
- '', entries,
- last_known_generation=last_known_generation,
- last_known_trans_id=last_known_trans_id,
- sync_id=sync_id,
- ensure=self._ensure_callback is not None)
- # add the document to the request
- size += self._prepare(
- ',', entries,
- id=id, rev=rev, content=content, gen=gen, trans_id=trans_id,
- number_of_docs=number_of_docs, doc_idx=doc_idx)
- entries.append('\r\n]')
- size += len(entries[-1])
- # send headers
- self._init_post_request('put', size)
- # send document
- for entry in entries:
- self._conn.send(entry)
- return self._response()
-
- def _sign_request(self, method, url_query, params):
- """
- Return an authorization header to be included in the HTTP request.
-
- :param method: The HTTP method.
- :type method: str
- :param url_query: The URL query string.
- :type url_query: str
- :param params: A list with encoded query parameters.
- :type param: list
-
- :return: The Authorization header.
- :rtype: list of tuple
- """
- return TokenBasedAuth._sign_request(self, method, url_query, params)
-
- def set_token_credentials(self, uuid, token):
- """
- Store given credentials so we can sign the request later.
-
- :param uuid: The user's uuid.
- :type uuid: str
- :param token: The authentication token.
- :type token: str
- """
- TokenBasedAuth.set_token_credentials(self, uuid, token)
-
-
-class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth):
- """
- A SyncTarget that encrypts data before sending and decrypts data after
- receiving.
-
- Normally encryption will have been written to the sync database upon
- document modification. The sync database is also used to write temporarily
- the parsed documents that the remote send us, before being decrypted and
- written to the main database.
- """
-
- # will later keep a reference to the insert-doc callback
- # passed to sync_exchange
- _insert_doc_cb = defaultdict(lambda: ProxyBase(None))
-
- """
- Period of recurrence of the periodic decrypting task, in seconds.
- """
- DECRYPT_TASK_PERIOD = 0.5
-
- #
- # Modified HTTPSyncTarget methods.
- #
-
- def __init__(self, url, source_replica_uid=None, creds=None, crypto=None,
- sync_db=None, sync_db_write_lock=None):
- """
- Initialize the SoledadSyncTarget.
-
- :param source_replica_uid: The source replica uid which we use when
- deferring decryption.
- :type source_replica_uid: str
- :param url: The url of the target replica to sync with.
- :type url: str
- :param creds: Optional dictionary giving credentials.
- to authorize the operation with the server.
- :type creds: dict
- :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt
- document contents when syncing.
- :type crypto: soledad.crypto.SoledadCrypto
- :param sync_db: Optional. handler for the db with the symmetric
- encryption of the syncing documents. If
- None, encryption will be done in-place,
- instead of retreiving it from the dedicated
- database.
- :type sync_db: Sqlite handler
- :param sync_db_write_lock: a write lock for controlling concurrent
- access to the sync_db
- :type sync_db_write_lock: threading.Lock
- """
- HTTPSyncTarget.__init__(self, url, creds)
- self._raw_url = url
- self._raw_creds = creds
- self._crypto = crypto
- self._stopped = True
- self._stop_lock = threading.Lock()
- self._sync_exchange_lock = threading.Lock()
- self.source_replica_uid = source_replica_uid
- self._defer_decryption = False
-
- # deferred decryption attributes
- self._sync_db = None
- self._sync_db_write_lock = None
- self._decryption_callback = None
- self._sync_decr_pool = None
- self._sync_watcher = None
- if sync_db and sync_db_write_lock is not None:
- self._sync_db = sync_db
- self._sync_db_write_lock = sync_db_write_lock
-
- def _setup_sync_decr_pool(self):
- """
- Set up the SyncDecrypterPool for deferred decryption.
- """
- if self._sync_decr_pool is None:
- # initialize syncing queue decryption pool
- self._sync_decr_pool = SyncDecrypterPool(
- self._crypto, self._sync_db,
- self._sync_db_write_lock,
- insert_doc_cb=self._insert_doc_cb)
- self._sync_decr_pool.set_source_replica_uid(
- self.source_replica_uid)
-
- def _teardown_sync_decr_pool(self):
- """
- Tear down the SyncDecrypterPool.
- """
- if self._sync_decr_pool is not None:
- self._sync_decr_pool.close()
- self._sync_decr_pool = None
-
- def _setup_sync_watcher(self):
- """
- Set up the sync watcher for deferred decryption.
- """
- if self._sync_watcher is None:
- self._sync_watcher = TimerTask(
- self._decrypt_syncing_received_docs,
- delay=self.DECRYPT_TASK_PERIOD)
-
- def _teardown_sync_watcher(self):
- """
- Tear down the sync watcher.
- """
- if self._sync_watcher is not None:
- self._sync_watcher.stop()
- self._sync_watcher.shutdown()
- self._sync_watcher = None
-
- def _get_replica_uid(self, url):
- """
- Return replica uid from the url, or None.
-
- :param url: the replica url
- :type url: str
- """
- replica_uid_match = re.findall("user-([0-9a-fA-F]+)", url)
- return replica_uid_match[0] if len(replica_uid_match) > 0 else None
-
- @staticmethod
- def connect(url, source_replica_uid=None, crypto=None):
- return SoledadSyncTarget(
- url, source_replica_uid=source_replica_uid, crypto=crypto)
-
- def _parse_received_doc_response(self, response):
- """
- Parse the response from the server containing the received document.
-
- :param response: The body and headers of the response.
- :type response: tuple(str, dict)
- """
- data, _ = response
- # decode incoming stream
- parts = data.splitlines()
- if not parts or parts[0] != '[' or parts[-1] != ']':
- raise errors.BrokenSyncStream
- data = parts[1:-1]
- # decode metadata
- line, comma = utils.check_and_strip_comma(data[0])
- metadata = None
- try:
- metadata = json.loads(line)
- new_generation = metadata['new_generation']
- new_transaction_id = metadata['new_transaction_id']
- number_of_changes = metadata['number_of_changes']
- except (json.JSONDecodeError, KeyError):
- raise errors.BrokenSyncStream
- # make sure we have replica_uid from fresh new dbs
- if self._ensure_callback and 'replica_uid' in metadata:
- self._ensure_callback(metadata['replica_uid'])
- # parse incoming document info
- doc_id = None
- rev = None
- content = None
- gen = None
- trans_id = None
- if number_of_changes > 0:
- try:
- entry = json.loads(data[1])
- doc_id = entry['id']
- rev = entry['rev']
- content = entry['content']
- gen = entry['gen']
- trans_id = entry['trans_id']
- except (IndexError, KeyError):
- raise errors.BrokenSyncStream
- return new_generation, new_transaction_id, number_of_changes, \
- doc_id, rev, content, gen, trans_id
-
- def _insert_received_doc(self, idx, total, response):
- """
- Insert a received document into the local replica.
-
- :param idx: The index count of the current operation.
- :type idx: int
- :param total: The total number of operations.
- :type total: int
- :param response: The body and headers of the response.
- :type response: tuple(str, dict)
- """
- new_generation, new_transaction_id, number_of_changes, doc_id, \
- rev, content, gen, trans_id = \
- self._parse_received_doc_response(response)
- if doc_id is not None:
- # decrypt incoming document and insert into local database
- # -------------------------------------------------------------
- # symmetric decryption of document's contents
- # -------------------------------------------------------------
- # If arriving content was symmetrically encrypted, we decrypt it.
- # We do it inline if defer_decryption flag is False or no sync_db
- # was defined, otherwise we defer it writing it to the received
- # docs table.
- doc = SoledadDocument(doc_id, rev, content)
- if is_symmetrically_encrypted(doc):
- if self._queue_for_decrypt:
- self._save_encrypted_received_doc(
- doc, gen, trans_id, idx, total)
- else:
- # defer_decryption is False or no-sync-db fallback
- doc.set_json(decrypt_doc(self._crypto, doc))
- self._return_doc_cb(doc, gen, trans_id)
- else:
- # not symmetrically encrypted doc, insert it directly
- # or save it in the decrypted stage.
- if self._queue_for_decrypt:
- self._save_received_doc(doc, gen, trans_id, idx, total)
- else:
- self._return_doc_cb(doc, gen, trans_id)
- # -------------------------------------------------------------
- # end of symmetric decryption
- # -------------------------------------------------------------
- msg = "%d/%d" % (idx + 1, total)
- signal(SOLEDAD_SYNC_RECEIVE_STATUS, msg)
- logger.debug("Soledad sync receive status: %s" % msg)
- return number_of_changes, new_generation, new_transaction_id
-
- def _get_remote_docs(self, url, last_known_generation, last_known_trans_id,
- headers, return_doc_cb, ensure_callback, sync_id,
- syncer_pool, defer_decryption=False):
- """
- Fetch sync documents from the remote database and insert them in the
- local database.
-
- If an incoming document's encryption scheme is equal to
- EncryptionSchemes.SYMKEY, then this method will decrypt it with
- Soledad's symmetric key.
-
- :param url: The syncing URL.
- :type url: str
- :param last_known_generation: Target's last known generation.
- :type last_known_generation: int
- :param last_known_trans_id: Target's last known transaction id.
- :type last_known_trans_id: str
- :param headers: The headers of the HTTP request.
- :type headers: dict
- :param return_doc_cb: A callback to insert docs from target.
- :type return_doc_cb: callable
- :param ensure_callback: A callback to ensure we have the correct
- target_replica_uid, if it was just created.
- :type ensure_callback: callable
- :param sync_id: The id for the current sync session.
- :type sync_id: str
- :param defer_decryption: Whether to defer the decryption process using
- the intermediate database. If False,
- decryption will be done inline.
- :type defer_decryption: bool
-
- :raise BrokenSyncStream: If `data` is malformed.
-
- :return: A dictionary representing the first line of the response got
- from remote replica.
- :rtype: dict
- """
- # we keep a reference to the callback in case we defer the decryption
- self._return_doc_cb = return_doc_cb
- self._queue_for_decrypt = defer_decryption \
- and self._sync_db is not None
-
- new_generation = last_known_generation
- new_transaction_id = last_known_trans_id
-
- if self._queue_for_decrypt:
- logger.debug(
- "Soledad sync: will queue received docs for decrypting.")
-
- idx = 0
- number_of_changes = 1
-
- first_request = True
- last_callback_lock = None
- threads = []
-
- # get incoming documents
- while idx < number_of_changes:
- # bail out if sync process was interrupted
- if self.stopped is True:
- break
-
- # launch a thread to fetch one document from target
- t = syncer_pool.new_syncer_thread(
- idx, number_of_changes,
- last_callback_lock=last_callback_lock)
-
- # bail out if any thread failed
- if t is None:
- self.stop()
- break
-
- t.doc_syncer.set_request_method(
- 'get', idx, sync_id, last_known_generation,
- last_known_trans_id)
- t.doc_syncer.set_success_callback(self._insert_received_doc)
-
- def _failure_callback(idx, total, exception):
- _failure_msg = "Soledad sync: error while getting document " \
- "%d/%d: %s" \
- % (idx + 1, total, exception)
- logger.warning("%s" % _failure_msg)
- logger.warning("Soledad sync: failing gracefully, will "
- "recover on next sync.")
-
- t.doc_syncer.set_failure_callback(_failure_callback)
- threads.append(t)
- t.start()
- last_callback_lock = t.callback_lock
- idx += 1
-
- # if this is the first request, wait to update the number of
- # changes
- if first_request is True:
- t.join()
- if t.success:
- number_of_changes, _, _ = t.result
- first_request = False
-
- # make sure all threads finished and we have up-to-date info
- last_successful_thread = None
- while threads:
- # check if there are failures
- t = threads.pop(0)
- t.join()
- if t.success:
- last_successful_thread = t
-
- # get information about last successful thread
- if last_successful_thread is not None:
- body, _ = last_successful_thread.response
- parsed_body = json.loads(body)
- # get current target gen and trans id in case no documents were
- # transferred
- if len(parsed_body) == 1:
- metadata = parsed_body[0]
- new_generation = metadata['new_generation']
- new_transaction_id = metadata['new_transaction_id']
- # get current target gen and trans id from last transferred
- # document
- else:
- doc_data = parsed_body[1]
- new_generation = doc_data['gen']
- new_transaction_id = doc_data['trans_id']
-
- return new_generation, new_transaction_id
-
- def sync_exchange(self, docs_by_generations,
- source_replica_uid, last_known_generation,
- last_known_trans_id, return_doc_cb,
- ensure_callback=None, defer_decryption=True,
- sync_id=None):
- """
- Find out which documents the remote database does not know about,
- encrypt and send them.
-
- This does the same as the parent's method but encrypts content before
- syncing.
-
- :param docs_by_generations: A list of (doc_id, generation, trans_id)
- of local documents that were changed since
- the last local generation the remote
- replica knows about.
- :type docs_by_generations: list of tuples
-
- :param source_replica_uid: The uid of the source replica.
- :type source_replica_uid: str
-
- :param last_known_generation: Target's last known generation.
- :type last_known_generation: int
-
- :param last_known_trans_id: Target's last known transaction id.
- :type last_known_trans_id: str
-
- :param return_doc_cb: A callback for inserting received documents from
- target. If not overriden, this will call u1db
- insert_doc_from_target in synchronizer, which
- implements the TAKE OTHER semantics.
- :type return_doc_cb: function
-
- :param ensure_callback: A callback that ensures we know the target
- replica uid if the target replica was just
- created.
- :type ensure_callback: function
-
- :param defer_decryption: Whether to defer the decryption process using
- the intermediate database. If False,
- decryption will be done inline.
- :type defer_decryption: bool
-
- :return: The new generation and transaction id of the target replica.
- :rtype: tuple
- """
- self._ensure_callback = ensure_callback
-
- if defer_decryption:
- self._sync_exchange_lock.acquire()
- self._setup_sync_decr_pool()
- self._setup_sync_watcher()
- self._defer_decryption = True
-
- self.start()
-
- if sync_id is None:
- sync_id = str(uuid4())
- self.source_replica_uid = source_replica_uid
- # let the decrypter pool access the passed callback to insert docs
- setProxiedObject(self._insert_doc_cb[source_replica_uid],
- return_doc_cb)
-
- if not self.clear_to_sync():
- raise PendingReceivedDocsSyncError
-
- self._ensure_connection()
- if self._trace_hook: # for tests
- self._trace_hook('sync_exchange')
- url = '%s/sync-from/%s' % (self._url.path, source_replica_uid)
- headers = self._sign_request('POST', url, {})
-
- cur_target_gen = last_known_generation
- cur_target_trans_id = last_known_trans_id
-
- # send docs
- msg = "%d/%d" % (0, len(docs_by_generations))
- signal(SOLEDAD_SYNC_SEND_STATUS, msg)
- logger.debug("Soledad sync send status: %s" % msg)
-
- defer_encryption = self._sync_db is not None
- syncer_pool = DocumentSyncerPool(
- self._raw_url, self._raw_creds, url, headers, ensure_callback,
- self.stop)
- threads = []
- last_request_lock = None
- last_callback_lock = None
- sent = 0
- total = len(docs_by_generations)
-
- synced = []
- number_of_docs = len(docs_by_generations)
-
- for doc, gen, trans_id in docs_by_generations:
- # allow for interrupting the sync process
- if self.stopped is True:
- break
-
- # skip non-syncable docs
- if isinstance(doc, SoledadDocument) and not doc.syncable:
- continue
-
- # -------------------------------------------------------------
- # symmetric encryption of document's contents
- # -------------------------------------------------------------
- doc_json = doc.get_json()
- if not doc.is_tombstone():
- if not defer_encryption:
- # fallback case, for tests
- doc_json = encrypt_doc(self._crypto, doc)
- else:
- try:
- doc_json = self.get_encrypted_doc_from_db(
- doc.doc_id, doc.rev)
- except Exception as exc:
- logger.error("Error while getting "
- "encrypted doc from db")
- logger.exception(exc)
- continue
- if doc_json is None:
- # Not marked as tombstone, but we got nothing
- # from the sync db. As it is not encrypted yet, we
- # force inline encryption.
- # TODO: implement a queue to deal with these cases.
- doc_json = encrypt_doc(self._crypto, doc)
- # -------------------------------------------------------------
- # end of symmetric encryption
- # -------------------------------------------------------------
- t = syncer_pool.new_syncer_thread(
- sent + 1, total, last_request_lock=None,
- last_callback_lock=last_callback_lock)
-
- # bail out if any thread failed
- if t is None:
- self.stop()
- break
-
- # set the request method
- t.doc_syncer.set_request_method(
- 'put', sync_id, cur_target_gen, cur_target_trans_id,
- id=doc.doc_id, rev=doc.rev, content=doc_json, gen=gen,
- trans_id=trans_id, number_of_docs=number_of_docs, doc_idx=sent + 1)
- # set the success calback
-
- def _success_callback(idx, total, response):
- _success_msg = "Soledad sync send status: %d/%d" \
- % (idx, total)
- signal(SOLEDAD_SYNC_SEND_STATUS, _success_msg)
- logger.debug(_success_msg)
-
- t.doc_syncer.set_success_callback(_success_callback)
-
- # set the failure callback
- def _failure_callback(idx, total, exception):
- _failure_msg = "Soledad sync: error while sending document " \
- "%d/%d: %s" % (idx, total, exception)
- logger.warning("%s" % _failure_msg)
- logger.warning("Soledad sync: failing gracefully, will "
- "recover on next sync.")
-
- t.doc_syncer.set_failure_callback(_failure_callback)
-
- # save thread and append
- t.start()
- threads.append((t, doc))
- last_request_lock = t.request_lock
- last_callback_lock = t.callback_lock
- sent += 1
-
- # make sure all threads finished and we have up-to-date info
- while threads:
- # check if there are failures
- t, doc = threads.pop(0)
- t.join()
- if t.success:
- synced.append((doc.doc_id, doc.rev))
-
- if defer_decryption:
- self._sync_watcher.start()
-
- # get docs from target
- if self.stopped is False:
- cur_target_gen, cur_target_trans_id = self._get_remote_docs(
- url,
- last_known_generation, last_known_trans_id, headers,
- return_doc_cb, ensure_callback, sync_id, syncer_pool,
- defer_decryption=defer_decryption)
- syncer_pool.cleanup()
-
- # delete documents from the sync database
- if defer_encryption:
- self.delete_encrypted_docs_from_db(synced)
-
- # wait for deferred decryption to finish
- if defer_decryption:
- while self.clear_to_sync() is False:
- sleep(self.DECRYPT_TASK_PERIOD)
- self._teardown_sync_watcher()
- self._teardown_sync_decr_pool()
- self._sync_exchange_lock.release()
-
- self.stop()
- return cur_target_gen, cur_target_trans_id
-
- def start(self):
- """
- Mark current sync session as running.
- """
- with self._stop_lock:
- self._stopped = False
-
- def stop(self):
- """
- Mark current sync session as stopped.
-
- This will eventually interrupt the sync_exchange() method and return
- enough information to the synchronizer so the sync session can be
- recovered afterwards.
- """
- with self._stop_lock:
- self._stopped = True
-
- @property
- def stopped(self):
- """
- Return whether this sync session is stopped.
-
- :return: Whether this sync session is stopped.
- :rtype: bool
- """
- with self._stop_lock:
- return self._stopped is True
-
- def get_encrypted_doc_from_db(self, doc_id, doc_rev):
- """
- Retrieve encrypted document from the database of encrypted docs for
- sync.
-
- :param doc_id: The Document id.
- :type doc_id: str
-
- :param doc_rev: The document revision
- :type doc_rev: str
- """
- encr = SyncEncrypterPool
- c = self._sync_db.cursor()
- sql = ("SELECT content FROM %s WHERE doc_id=? and rev=?" % (
- encr.TABLE_NAME,))
- c.execute(sql, (doc_id, doc_rev))
- res = c.fetchall()
- if len(res) != 0:
- return res[0][0]
-
- def delete_encrypted_docs_from_db(self, docs_ids):
- """
- Delete several encrypted documents from the database of symmetrically
- encrypted docs to sync.
-
- :param docs_ids: an iterable with (doc_id, doc_rev) for all documents
- to be deleted.
- :type docs_ids: any iterable of tuples of str
- """
- if docs_ids:
- encr = SyncEncrypterPool
- c = self._sync_db.cursor()
- for doc_id, doc_rev in docs_ids:
- sql = ("DELETE FROM %s WHERE doc_id=? and rev=?" % (
- encr.TABLE_NAME,))
- c.execute(sql, (doc_id, doc_rev))
- self._sync_db.commit()
-
- def _save_encrypted_received_doc(self, doc, gen, trans_id, idx, total):
- """
- Save a symmetrically encrypted incoming document into the received
- docs table in the sync db. A decryption task will pick it up
- from here in turn.
-
- :param doc: The document to save.
- :type doc: SoledadDocument
- :param gen: The generation.
- :type gen: str
- :param trans_id: Transacion id.
- :type gen: str
- :param idx: The index count of the current operation.
- :type idx: int
- :param total: The total number of operations.
- :type total: int
- """
- logger.debug(
- "Enqueueing doc for decryption: %d/%d."
- % (idx + 1, total))
- self._sync_decr_pool.insert_encrypted_received_doc(
- doc.doc_id, doc.rev, doc.content, gen, trans_id)
-
- def _save_received_doc(self, doc, gen, trans_id, idx, total):
- """
- Save any incoming document into the received docs table in the sync db.
-
- :param doc: The document to save.
- :type doc: SoledadDocument
- :param gen: The generation.
- :type gen: str
- :param trans_id: Transacion id.
- :type gen: str
- :param idx: The index count of the current operation.
- :type idx: int
- :param total: The total number of operations.
- :type total: int
- """
- logger.debug(
- "Enqueueing doc, no decryption needed: %d/%d."
- % (idx + 1, total))
- self._sync_decr_pool.insert_received_doc(
- doc.doc_id, doc.rev, doc.content, gen, trans_id)
-
- #
- # Symmetric decryption of syncing docs
- #
-
- def clear_to_sync(self):
- """
- Return True if sync can proceed (ie, the received db table is empty).
- :rtype: bool
- """
- if self._sync_decr_pool is not None:
- return self._sync_decr_pool.count_received_encrypted_docs() == 0
- else:
- return True
-
- def set_decryption_callback(self, cb):
- """
- Set callback to be called when the decryption finishes.
-
- :param cb: The callback to be set.
- :type cb: callable
- """
- self._decryption_callback = cb
-
- def has_decryption_callback(self):
- """
- Return True if there is a decryption callback set.
- :rtype: bool
- """
- return self._decryption_callback is not None
-
- def has_syncdb(self):
- """
- Return True if we have an initialized syncdb.
- """
- return self._sync_db is not None
-
- def _decrypt_syncing_received_docs(self):
- """
- Decrypt the documents received from remote replica and insert them
- into the local one.
-
- Called periodically from TimerTask self._sync_watcher.
- """
- if sameProxiedObjects(
- self._insert_doc_cb.get(self.source_replica_uid),
- None):
- return
-
- decrypter = self._sync_decr_pool
- decrypter.decrypt_received_docs()
- done = decrypter.process_decrypted()
-
- def _sign_request(self, method, url_query, params):
- """
- Return an authorization header to be included in the HTTP request.
-
- :param method: The HTTP method.
- :type method: str
- :param url_query: The URL query string.
- :type url_query: str
- :param params: A list with encoded query parameters.
- :type param: list
-
- :return: The Authorization header.
- :rtype: list of tuple
- """
- return TokenBasedAuth._sign_request(self, method, url_query, params)
-
- def set_token_credentials(self, uuid, token):
- """
- Store given credentials so we can sign the request later.
-
- :param uuid: The user's uuid.
- :type uuid: str
- :param token: The authentication token.
- :type token: str
- """
- TokenBasedAuth.set_token_credentials(self, uuid, token)
diff --git a/common/changes/VERSION_COMPAT b/common/changes/VERSION_COMPAT
index e69de29b..cc00ecf7 100644
--- a/common/changes/VERSION_COMPAT
+++ b/common/changes/VERSION_COMPAT
@@ -0,0 +1,10 @@
+#################################################
+# This file keeps track of the recent changes
+# introduced in internal leap dependencies.
+# Add your changes here so we can properly update
+# requirements.pip during the release process.
+# (leave header when resetting)
+#################################################
+#
+# BEGIN DEPENDENCY LIST -------------------------
+# leap.foo.bar>=x.y.z
diff --git a/common/pkg/requirements-testing.pip b/common/pkg/requirements-testing.pip
index 9302450c..c72c9fc4 100644
--- a/common/pkg/requirements-testing.pip
+++ b/common/pkg/requirements-testing.pip
@@ -3,3 +3,4 @@ testscenarios
leap.common
leap.soledad.server
leap.soledad.client
+setuptools-trial
diff --git a/common/pkg/requirements.pip b/common/pkg/requirements.pip
index 5787114e..005d6884 100644
--- a/common/pkg/requirements.pip
+++ b/common/pkg/requirements.pip
@@ -1,7 +1,10 @@
simplejson
u1db
-#this is not strictly needed by us, but we need it
-#until u1db adds it to its release as a dep.
-oauth
+# leap deps -- bump me!
+leap.common>=0.7.0
+# XXX -- fix me!
+# oauth is not strictly needed by us, but we need it until u1db adds it to its
+# release as a dep.
+oauth
diff --git a/common/setup.cfg b/common/setup.cfg
new file mode 100644
index 00000000..c71bffa0
--- /dev/null
+++ b/common/setup.cfg
@@ -0,0 +1,2 @@
+[aliases]
+test = trial
diff --git a/common/setup.py b/common/setup.py
index 365006b2..f4d8bc65 100644
--- a/common/setup.py
+++ b/common/setup.py
@@ -234,23 +234,6 @@ class cmd_develop(_cmd_develop):
# versioneer powered
-old_cmd_sdist = cmdclass["sdist"]
-
-
-class cmd_sdist(old_cmd_sdist):
- """
- Generate 'src/leap/soledad/common/ddocs.py' which contains couch design
- documents scripts.
- """
- def run(self):
- old_cmd_sdist.run(self)
-
- def make_release_tree(self, base_dir, files):
- old_cmd_sdist.make_release_tree(self, base_dir, files)
- build_ddocs_py(basedir=base_dir)
-
-
-# versioneer powered
old_cmd_build = cmdclass["build"]
@@ -262,7 +245,6 @@ class cmd_build(old_cmd_build):
cmdclass["freeze_debianver"] = freeze_debianver
cmdclass["build"] = cmd_build
-cmdclass["sdist"] = cmd_sdist
cmdclass["develop"] = cmd_develop
@@ -288,9 +270,9 @@ setup(
),
classifiers=trove_classifiers,
namespace_packages=["leap", "leap.soledad"],
- packages=find_packages('src', exclude=['leap.soledad.common.tests']),
+ packages=find_packages('src', exclude=['*.tests', '*.tests.*']),
package_dir={'': 'src'},
- test_suite='leap.soledad.common.tests.load_tests',
+ test_suite='leap.soledad.common.tests',
install_requires=utils.parse_requirements(),
tests_require=utils.parse_requirements(
reqfiles=['pkg/requirements-testing.pip']),
diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py
index 23d28e76..1e52e3a7 100644
--- a/common/src/leap/soledad/common/__init__.py
+++ b/common/src/leap/soledad/common/__init__.py
@@ -21,14 +21,10 @@ Soledad routines common to client and server.
"""
-from hashlib import sha256
-
-
#
# Global constants
#
-
SHARED_DB_NAME = 'shared'
SHARED_DB_LOCK_DOC_ID_PREFIX = 'lock-'
USER_DB_PREFIX = 'user-'
@@ -38,45 +34,17 @@ USER_DB_PREFIX = 'user-'
# Global functions
#
-# we want to use leap.common.check.leap_assert in case it is available,
-# because it also logs in a way other parts of leap can access log messages.
-
-try:
- from leap.common.check import leap_assert as soledad_assert
-
-except ImportError:
-
- def soledad_assert(condition, message):
- """
- Asserts the condition and displays the message if that's not
- met.
-
- @param condition: condition to check
- @type condition: bool
- @param message: message to display if the condition isn't met
- @type message: str
- """
- assert condition, message
-
-try:
- from leap.common.check import leap_assert_type as soledad_assert_type
-
-except ImportError:
-
- def soledad_assert_type(var, expectedType):
- """
- Helper assert check for a variable's expected type
-
- @param var: variable to check
- @type var: any
- @param expectedType: type to check agains
- @type expectedType: type
- """
- soledad_assert(isinstance(var, expectedType),
- "Expected type %r instead of %r" %
- (expectedType, type(var)))
+from leap.common.check import leap_assert as soledad_assert
+from leap.common.check import leap_assert_type as soledad_assert_type
from ._version import get_versions
__version__ = get_versions()['version']
del get_versions
+
+
+__all__ = [
+ "soledad_assert",
+ "soledad_assert_type",
+ "__version__",
+]
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index b38b5b96..8d262ccd 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -18,12 +18,12 @@
"""A U1DB backend that uses CouchDB as its persistence layer."""
+
import simplejson as json
import re
import uuid
import logging
import binascii
-import socket
import time
import sys
import threading
@@ -44,7 +44,7 @@ from couchdb.http import (
urljoin as couch_urljoin,
Resource,
)
-from u1db import query_parser, vectorclock
+from u1db import vectorclock
from u1db.errors import (
DatabaseDoesNotExist,
InvalidGeneration,
@@ -60,7 +60,7 @@ from u1db.remote import http_app
from u1db.remote.server_state import ServerState
-from leap.soledad.common import USER_DB_PREFIX, ddocs, errors
+from leap.soledad.common import ddocs, errors
from leap.soledad.common.document import SoledadDocument
@@ -160,7 +160,6 @@ class CouchDocument(SoledadDocument):
"""
if self._conflicts is None:
raise Exception("Run self._ensure_fetch_conflicts first!")
- conflicts_len = len(self._conflicts)
self._conflicts = filter(
lambda doc: doc.rev not in conflict_revs,
self._conflicts)
@@ -1181,7 +1180,7 @@ class CouchDatabase(CommonBackend):
res = self._database.resource(*ddoc_path)
try:
with CouchDatabase.update_handler_lock[self._get_replica_uid()]:
- body={
+ body = {
'other_replica_uid': other_replica_uid,
'other_generation': other_generation,
'other_transaction_id': other_transaction_id,
diff --git a/common/src/leap/soledad/common/crypto.py b/common/src/leap/soledad/common/crypto.py
index 56bb608a..b4f3234f 100644
--- a/common/src/leap/soledad/common/crypto.py
+++ b/common/src/leap/soledad/common/crypto.py
@@ -35,13 +35,29 @@ class EncryptionSchemes(object):
PUBKEY = 'pubkey'
-class UnknownEncryptionScheme(Exception):
+class UnknownEncryptionSchemeError(Exception):
"""
Raised when trying to decrypt from unknown encryption schemes.
"""
pass
+class EncryptionMethods(object):
+ """
+ Representation of encryption methods that can be used.
+ """
+
+ AES_256_CTR = 'aes-256-ctr'
+ XSALSA20 = 'xsalsa20'
+
+
+class UnknownEncryptionMethodError(Exception):
+ """
+ Raised when trying to encrypt/decrypt with unknown method.
+ """
+ pass
+
+
class MacMethods(object):
"""
Representation of MAC methods used to authenticate document's contents.
@@ -50,7 +66,7 @@ class MacMethods(object):
HMAC = 'hmac'
-class UnknownMacMethod(Exception):
+class UnknownMacMethodError(Exception):
"""
Raised when trying to authenticate document's content with unknown MAC
mehtod.
@@ -58,7 +74,7 @@ class UnknownMacMethod(Exception):
pass
-class WrongMac(Exception):
+class WrongMacError(Exception):
"""
Raised when failing to authenticate document's contents based on MAC.
"""
diff --git a/common/src/leap/soledad/common/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py
index 3081683b..acebb77b 100644
--- a/common/src/leap/soledad/common/tests/__init__.py
+++ b/common/src/leap/soledad/common/tests/__init__.py
@@ -19,291 +19,9 @@
"""
Tests to make sure Soledad provides U1DB functionality and more.
"""
-import os
-import random
-import string
-import u1db
-from mock import Mock
-
-
-from leap.soledad.common.document import SoledadDocument
-from leap.soledad.client import Soledad
-from leap.soledad.client.crypto import decrypt_doc_dict
-from leap.soledad.client.crypto import ENC_SCHEME_KEY
-from leap.common.testing.basetest import BaseLeapTest
-
-
-#-----------------------------------------------------------------------------
-# Some tests inherit from BaseSoledadTest in order to have a working Soledad
-# instance in each test.
-#-----------------------------------------------------------------------------
-
-ADDRESS = 'leap@leap.se'
-
-
-class BaseSoledadTest(BaseLeapTest):
- """
- Instantiates Soledad for usage in tests.
- """
- defer_sync_encryption = False
-
- def setUp(self):
- # config info
- self.db1_file = os.path.join(self.tempdir, "db1.u1db")
- self.db2_file = os.path.join(self.tempdir, "db2.u1db")
- self.email = ADDRESS
- # open test dbs
- self._db1 = u1db.open(self.db1_file, create=True,
- document_factory=SoledadDocument)
- self._db2 = u1db.open(self.db2_file, create=True,
- document_factory=SoledadDocument)
- # get a random prefix for each test, so we do not mess with
- # concurrency during initialization and shutting down of
- # each local db.
- self.rand_prefix = ''.join(
- map(lambda x: random.choice(string.ascii_letters), range(6)))
- # initialize soledad by hand so we can control keys
- self._soledad = self._soledad_instance(
- prefix=self.rand_prefix, user=self.email)
-
- def tearDown(self):
- self._db1.close()
- self._db2.close()
- self._soledad.close()
-
- # XXX should not access "private" attrs
- for f in [self._soledad._local_db_path, self._soledad._secrets_path]:
- if os.path.isfile(f):
- os.unlink(f)
- def get_default_shared_mock(self, put_doc_side_effect):
- """
- Get a default class for mocking the shared DB
- """
- class defaultMockSharedDB(object):
- get_doc = Mock(return_value=None)
- put_doc = Mock(side_effect=put_doc_side_effect)
- lock = Mock(return_value=('atoken', 300))
- unlock = Mock(return_value=True)
- def __call__(self):
- return self
- return defaultMockSharedDB
-
- def _soledad_instance(self, user=ADDRESS, passphrase=u'123',
- prefix='',
- secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME,
- local_db_path='soledad.u1db', server_url='',
- cert_file=None, secret_id=None,
- shared_db_class=None):
-
- def _put_doc_side_effect(doc):
- self._doc_put = doc
-
- if shared_db_class is not None:
- MockSharedDB = shared_db_class
- else:
- MockSharedDB = self.get_default_shared_mock(
- _put_doc_side_effect)
-
- Soledad._shared_db = MockSharedDB()
- return Soledad(
- user,
- passphrase,
- secrets_path=os.path.join(
- self.tempdir, prefix, secrets_path),
- local_db_path=os.path.join(
- self.tempdir, prefix, local_db_path),
- server_url=server_url, # Soledad will fail if not given an url.
- cert_file=cert_file,
- secret_id=secret_id,
- defer_encryption=self.defer_sync_encryption)
-
- def assertGetEncryptedDoc(
- self, db, doc_id, doc_rev, content, has_conflicts):
- """
- Assert that the document in the database looks correct.
- """
- exp_doc = self.make_document(doc_id, doc_rev, content,
- has_conflicts=has_conflicts)
- doc = db.get_doc(doc_id)
-
- if ENC_SCHEME_KEY in doc.content:
- # XXX check for SYM_KEY too
- key = self._soledad._crypto.doc_passphrase(doc.doc_id)
- secret = self._soledad._crypto.secret
- decrypted = decrypt_doc_dict(
- doc.content, doc.doc_id, doc.rev,
- key, secret)
- doc.set_json(decrypted)
- self.assertEqual(exp_doc.doc_id, doc.doc_id)
- self.assertEqual(exp_doc.rev, doc.rev)
- self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts)
- self.assertEqual(exp_doc.content, doc.content)
-
-
-# Key material for testing
-KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF"
-PUBLIC_KEY = """
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1.4.10 (GNU/Linux)
-
-mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz
-iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO
-zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx
-irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT
-huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs
-d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g
-wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb
-hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv
-U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H
-T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i
-Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB
-tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD
-BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb
-T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5
-hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP
-QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU
-Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+
-eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI
-txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB
-KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy
-7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr
-K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx
-2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n
-3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf
-H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS
-sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs
-iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD
-uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0
-GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3
-lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS
-fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe
-dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1
-WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK
-3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td
-U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F
-Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX
-NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj
-cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk
-ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE
-VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51
-XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8
-oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM
-Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+
-BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/
-diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2
-ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX
-=MuOY
------END PGP PUBLIC KEY BLOCK-----
-"""
-PRIVATE_KEY = """
------BEGIN PGP PRIVATE KEY BLOCK-----
-Version: GnuPG v1.4.10 (GNU/Linux)
-
-lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz
-iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO
-zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx
-irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT
-huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs
-d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g
-wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb
-hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv
-U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H
-T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i
-Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB
-AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs
-E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t
-KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds
-FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb
-J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky
-KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY
-VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5
-jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF
-q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c
-zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv
-OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt
-VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx
-nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv
-Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP
-4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F
-RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv
-mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x
-sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0
-cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI
-L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW
-ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd
-LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e
-SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO
-dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8
-xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY
-HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw
-7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh
-cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH
-AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM
-MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo
-rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX
-hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA
-QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo
-alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4
-Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb
-HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV
-3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF
-/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n
-s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC
-4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ
-1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ
-uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q
-us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/
-Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o
-6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA
-K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+
-iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t
-9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3
-zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl
-QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD
-Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX
-wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e
-PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC
-9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI
-85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih
-7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn
-E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+
-ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0
-Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m
-KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT
-xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/
-jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4
-OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o
-tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF
-cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb
-OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i
-7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2
-H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX
-MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR
-ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ
-waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU
-e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs
-rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G
-GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu
-tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U
-22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E
-/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC
-0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+
-LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm
-laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy
-bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd
-GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp
-VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ
-z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD
-U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l
-Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ
-GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL
-Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1
-RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc=
-=JTFu
------END PGP PRIVATE KEY BLOCK-----
-"""
+import os
def load_tests():
diff --git a/common/src/leap/soledad/common/tests/couchdb.ini.template b/common/src/leap/soledad/common/tests/couchdb.ini.template
index 1fc2205b..174d9d86 100644
--- a/common/src/leap/soledad/common/tests/couchdb.ini.template
+++ b/common/src/leap/soledad/common/tests/couchdb.ini.template
@@ -12,213 +12,11 @@ delayed_commits = true ; set this to false to ensure an fsync before 201 Created
uri_file = %(tempdir)s/lib/couch.uri
file_compression = snappy
-[database_compaction]
-; larger buffer sizes can originate smaller files
-doc_buffer_size = 524288 ; value in bytes
-checkpoint_after = 5242880 ; checkpoint after every N bytes were written
-
-[view_compaction]
-; larger buffer sizes can originate smaller files
-keyvalue_buffer_size = 2097152 ; value in bytes
-
-[httpd]
-port = 0
-bind_address = 127.0.0.1
-authentication_handlers = {couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler}
-default_handler = {couch_httpd_db, handle_request}
-secure_rewrites = true
-vhost_global_handlers = _utils, _uuids, _session, _oauth, _users
-allow_jsonp = false
-; Options for the MochiWeb HTTP server.
-;server_options = [{backlog, 128}, {acceptor_pool_size, 16}]
-; For more socket options, consult Erlang's module 'inet' man page.
-;socket_options = [{recbuf, 262144}, {sndbuf, 262144}, {nodelay, true}]
-log_max_chunk_size = 1000000
-
[log]
file = %(tempdir)s/log/couch.log
level = info
include_sasl = true
-[couch_httpd_auth]
-authentication_db = _users
-authentication_redirect = /_utils/session.html
-require_valid_user = false
-timeout = 600 ; number of seconds before automatic logout
-auth_cache_size = 50 ; size is number of cache entries
-allow_persistent_cookies = false ; set to true to allow persistent cookies
-
-[couch_httpd_oauth]
-; If set to 'true', oauth token and consumer secrets will be looked up
-; in the authentication database (_users). These secrets are stored in
-; a top level property named "oauth" in user documents. Example:
-; {
-; "_id": "org.couchdb.user:joe",
-; "type": "user",
-; "name": "joe",
-; "password_sha": "fe95df1ca59a9b567bdca5cbaf8412abd6e06121",
-; "salt": "4e170ffeb6f34daecfd814dfb4001a73"
-; "roles": ["foo", "bar"],
-; "oauth": {
-; "consumer_keys": {
-; "consumerKey1": "key1Secret",
-; "consumerKey2": "key2Secret"
-; },
-; "tokens": {
-; "token1": "token1Secret",
-; "token2": "token2Secret"
-; }
-; }
-; }
-use_users_db = false
-
-[query_servers]
-; javascript = %(tempdir)s/server/main.js
-javascript = /usr/bin/couchjs /usr/share/couchdb/server/main.js
-coffeescript = /usr/bin/couchjs /usr/share/couchdb/server/main-coffee.js
-
-
-; Changing reduce_limit to false will disable reduce_limit.
-; If you think you're hitting reduce_limit with a "good" reduce function,
-; please let us know on the mailing list so we can fine tune the heuristic.
-[query_server_config]
-reduce_limit = true
-os_process_limit = 25
-
-[daemons]
-view_manager={couch_view, start_link, []}
-external_manager={couch_external_manager, start_link, []}
-query_servers={couch_query_servers, start_link, []}
-vhosts={couch_httpd_vhost, start_link, []}
-httpd={couch_httpd, start_link, []}
-stats_aggregator={couch_stats_aggregator, start, []}
-stats_collector={couch_stats_collector, start, []}
-uuids={couch_uuids, start, []}
-auth_cache={couch_auth_cache, start_link, []}
-replication_manager={couch_replication_manager, start_link, []}
-os_daemons={couch_os_daemons, start_link, []}
-compaction_daemon={couch_compaction_daemon, start_link, []}
-
-[httpd_global_handlers]
-/ = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>}
-
-_all_dbs = {couch_httpd_misc_handlers, handle_all_dbs_req}
-_active_tasks = {couch_httpd_misc_handlers, handle_task_status_req}
-_config = {couch_httpd_misc_handlers, handle_config_req}
-_replicate = {couch_httpd_replicator, handle_req}
-_uuids = {couch_httpd_misc_handlers, handle_uuids_req}
-_restart = {couch_httpd_misc_handlers, handle_restart_req}
-_stats = {couch_httpd_stats_handlers, handle_stats_req}
-_log = {couch_httpd_misc_handlers, handle_log_req}
-_session = {couch_httpd_auth, handle_session_req}
-_oauth = {couch_httpd_oauth, handle_oauth_req}
-
-[httpd_db_handlers]
-_view_cleanup = {couch_httpd_db, handle_view_cleanup_req}
-_compact = {couch_httpd_db, handle_compact_req}
-_design = {couch_httpd_db, handle_design_req}
-_temp_view = {couch_httpd_view, handle_temp_view_req}
-_changes = {couch_httpd_db, handle_changes_req}
-
-; The external module takes an optional argument allowing you to narrow it to a
-; single script. Otherwise the script name is inferred from the first path section
-; after _external's own path.
-; _mypath = {couch_httpd_external, handle_external_req, <<"mykey">>}
-; _external = {couch_httpd_external, handle_external_req}
-
-[httpd_design_handlers]
-_view = {couch_httpd_view, handle_view_req}
-_show = {couch_httpd_show, handle_doc_show_req}
-_list = {couch_httpd_show, handle_view_list_req}
-_info = {couch_httpd_db, handle_design_info_req}
-_rewrite = {couch_httpd_rewrite, handle_rewrite_req}
-_update = {couch_httpd_show, handle_doc_update_req}
-
-; enable external as an httpd handler, then link it with commands here.
-; note, this api is still under consideration.
-; [external]
-; mykey = /path/to/mycommand
-
-; Here you can setup commands for CouchDB to manage
-; while it is alive. It will attempt to keep each command
-; alive if it exits.
-; [os_daemons]
-; some_daemon_name = /path/to/script -with args
-
-
-[uuids]
-; Known algorithms:
-; random - 128 bits of random awesome
-; All awesome, all the time.
-; sequential - monotonically increasing ids with random increments
-; First 26 hex characters are random. Last 6 increment in
-; random amounts until an overflow occurs. On overflow, the
-; random prefix is regenerated and the process starts over.
-; utc_random - Time since Jan 1, 1970 UTC with microseconds
-; First 14 characters are the time in hex. Last 18 are random.
-algorithm = sequential
-
-[stats]
-; rate is in milliseconds
-rate = 1000
-; sample intervals are in seconds
-samples = [0, 60, 300, 900]
-
-[attachments]
-compression_level = 8 ; from 1 (lowest, fastest) to 9 (highest, slowest), 0 to disable compression
-compressible_types = text/*, application/javascript, application/json, application/xml
-
-[replicator]
-db = _replicator
-; Maximum replicaton retry count can be a non-negative integer or "infinity".
-max_replication_retry_count = 10
-; More worker processes can give higher network throughput but can also
-; imply more disk and network IO.
-worker_processes = 4
-; With lower batch sizes checkpoints are done more frequently. Lower batch sizes
-; also reduce the total amount of used RAM memory.
-worker_batch_size = 500
-; Maximum number of HTTP connections per replication.
-http_connections = 20
-; HTTP connection timeout per replication.
-; Even for very fast/reliable networks it might need to be increased if a remote
-; database is too busy.
-connection_timeout = 30000
-; If a request fails, the replicator will retry it up to N times.
-retries_per_request = 10
-; Some socket options that might boost performance in some scenarios:
-; {nodelay, boolean()}
-; {sndbuf, integer()}
-; {recbuf, integer()}
-; {priority, integer()}
-; See the `inet` Erlang module's man page for the full list of options.
-socket_options = [{keepalive, true}, {nodelay, false}]
-; Path to a file containing the user's certificate.
-;cert_file = /full/path/to/server_cert.pem
-; Path to file containing user's private PEM encoded key.
-;key_file = /full/path/to/server_key.pem
-; String containing the user's password. Only used if the private keyfile is password protected.
-;password = somepassword
-; Set to true to validate peer certificates.
-verify_ssl_certificates = false
-; File containing a list of peer trusted certificates (in the PEM format).
-;ssl_trusted_certificates_file = /etc/ssl/certs/ca-certificates.crt
-; Maximum peer certificate depth (must be set even if certificate validation is off).
-ssl_certificate_max_depth = 3
-
-[compaction_daemon]
-; The delay, in seconds, between each check for which database and view indexes
-; need to be compacted.
-check_interval = 300
-; If a database or view index file is smaller then this value (in bytes),
-; compaction will not happen. Very small files always have a very high
-; fragmentation therefore it's not worth to compact them.
-min_file_size = 131072
-
-[compactions]
-; List of compaction rules for the compaction daemon.
-
-
-;[admins]
-;testuser = -hashed-f50a252c12615697c5ed24ec5cd56b05d66fe91e,b05471ba260132953930cf9f97f327f5
-; pass for above user is 'testpass'
+[httpd]
+port = 0
+bind_address = 127.0.0.1
diff --git a/common/src/leap/soledad/common/tests/hacker_crackdown.txt b/common/src/leap/soledad/common/tests/hacker_crackdown.txt
new file mode 100644
index 00000000..a01eb509
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/hacker_crackdown.txt
@@ -0,0 +1,13005 @@
+The Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling
+
+This eBook is for the use of anyone anywhere at no cost and with
+almost no restrictions whatsoever. You may copy it, give it away or
+re-use it under the terms of the Project Gutenberg License included
+with this eBook or online at www.gutenberg.org
+
+** This is a COPYRIGHTED Project Gutenberg eBook, Details Below **
+** Please follow the copyright guidelines in this file. **
+
+Title: Hacker Crackdown
+ Law and Disorder on the Electronic Frontier
+
+Author: Bruce Sterling
+
+Posting Date: February 9, 2012 [EBook #101]
+Release Date: January, 1994
+
+Language: English
+
+
+*** START OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN ***
+
+
+
+
+
+
+
+
+
+
+
+
+
+THE HACKER CRACKDOWN
+
+Law and Disorder on the Electronic Frontier
+
+by Bruce Sterling
+
+
+
+
+CONTENTS
+
+
+Preface to the Electronic Release of The Hacker Crackdown
+
+Chronology of the Hacker Crackdown
+
+
+Introduction
+
+
+Part 1: CRASHING THE SYSTEM
+A Brief History of Telephony
+Bell's Golden Vaporware
+Universal Service
+Wild Boys and Wire Women
+The Electronic Communities
+The Ungentle Giant
+The Breakup
+In Defense of the System
+The Crash Post-Mortem
+Landslides in Cyberspace
+
+
+Part 2: THE DIGITAL UNDERGROUND
+Steal This Phone
+Phreaking and Hacking
+The View From Under the Floorboards
+Boards: Core of the Underground
+Phile Phun
+The Rake's Progress
+Strongholds of the Elite
+Sting Boards
+Hot Potatoes
+War on the Legion
+Terminus
+Phile 9-1-1
+War Games
+Real Cyberpunk
+
+
+Part 3: LAW AND ORDER
+Crooked Boards
+The World's Biggest Hacker Bust
+Teach Them a Lesson
+The U.S. Secret Service
+The Secret Service Battles the Boodlers
+A Walk Downtown
+FCIC: The Cutting-Edge Mess
+Cyberspace Rangers
+FLETC: Training the Hacker-Trackers
+
+
+Part 4: THE CIVIL LIBERTARIANS
+NuPrometheus + FBI = Grateful Dead
+Whole Earth + Computer Revolution = WELL
+Phiber Runs Underground and Acid Spikes the Well
+The Trial of Knight Lightning
+Shadowhawk Plummets to Earth
+Kyrie in the Confessional
+$79,499
+A Scholar Investigates
+Computers, Freedom, and Privacy
+
+
+Electronic Afterword to The Hacker Crackdown, Halloween 1993
+
+
+
+
+THE HACKER CRACKDOWN
+
+Law and Disorder on the Electronic Frontier
+
+by Bruce Sterling
+
+
+
+
+
+Preface to the Electronic Release of The Hacker Crackdown
+
+
+January 1, 1994--Austin, Texas
+
+
+Hi, I'm Bruce Sterling, the author of this electronic book.
+
+Out in the traditional world of print, The Hacker Crackdown
+is ISBN 0-553-08058-X, and is formally catalogued by
+the Library of Congress as "1. Computer crimes--United States.
+2. Telephone--United States--Corrupt practices.
+3. Programming (Electronic computers)--United States--Corrupt practices."
+
+`Corrupt practices,' I always get a kick out of that description.
+Librarians are very ingenious people.
+
+The paperback is ISBN 0-553-56370-X. If you go
+and buy a print version of The Hacker Crackdown,
+an action I encourage heartily, you may notice that
+in the front of the book, beneath the copyright notice--
+"Copyright (C) 1992 by Bruce Sterling"--
+it has this little block of printed legal
+boilerplate from the publisher. It says, and I quote:
+
+ "No part of this book may be reproduced or transmitted in any form
+or by any means, electronic or mechanical, including photocopying,
+recording, or by any information storage and retrieval system,
+without permission in writing from the publisher.
+For information address: Bantam Books."
+
+This is a pretty good disclaimer, as such disclaimers go.
+I collect intellectual-property disclaimers, and I've seen dozens of them,
+and this one is at least pretty straightforward. In this narrow
+and particular case, however, it isn't quite accurate.
+Bantam Books puts that disclaimer on every book they publish,
+but Bantam Books does not, in fact, own the electronic rights to this book.
+I do, because of certain extensive contract maneuverings my agent and I
+went through before this book was written. I want to give those electronic
+publishing rights away through certain not-for-profit channels,
+and I've convinced Bantam that this is a good idea.
+
+Since Bantam has seen fit to peacably agree to this scheme of mine,
+Bantam Books is not going to fuss about this. Provided you don't try
+to sell the book, they are not going to bother you for what you do with
+the electronic copy of this book. If you want to check this out personally,
+you can ask them; they're at 1540 Broadway NY NY 10036. However, if you were
+so foolish as to print this book and start retailing it for money in violation
+of my copyright and the commercial interests of Bantam Books, then Bantam,
+a part of the gigantic Bertelsmann multinational publishing combine,
+would roust some of their heavy-duty attorneys out of hibernation
+and crush you like a bug. This is only to be expected.
+I didn't write this book so that you could make money out of it.
+If anybody is gonna make money out of this book,
+it's gonna be me and my publisher.
+
+My publisher deserves to make money out of this book.
+Not only did the folks at Bantam Books commission me
+to write the book, and pay me a hefty sum to do so, but
+they bravely printed, in text, an electronic document the
+reproduction of which was once alleged to be a federal felony.
+Bantam Books and their numerous attorneys were very brave
+and forthright about this book. Furthermore, my former editor
+at Bantam Books, Betsy Mitchell, genuinely cared about this project,
+and worked hard on it, and had a lot of wise things to say
+about the manuscript. Betsy deserves genuine credit for this book,
+credit that editors too rarely get.
+
+The critics were very kind to The Hacker Crackdown,
+and commercially the book has done well. On the other hand,
+I didn't write this book in order to squeeze every last nickel
+and dime out of the mitts of impoverished sixteen-year-old
+cyberpunk high-school-students. Teenagers don't have any money--
+(no, not even enough for the six-dollar Hacker Crackdown paperback,
+with its attractive bright-red cover and useful index).
+That's a major reason why teenagers sometimes succumb to the temptation
+to do things they shouldn't, such as swiping my books out of libraries.
+Kids: this one is all yours, all right? Go give the print version back.
+*8-)
+
+Well-meaning, public-spirited civil libertarians don't have much money,
+either. And it seems almost criminal to snatch cash out of the hands of
+America's direly underpaid electronic law enforcement community.
+
+If you're a computer cop, a hacker, or an electronic civil
+liberties activist, you are the target audience for this book.
+I wrote this book because I wanted to help you, and help other people
+understand you and your unique, uhm, problems. I wrote this book
+to aid your activities, and to contribute to the public discussion
+of important political issues. In giving the text away in this
+fashion, I am directly contributing to the book's ultimate aim:
+to help civilize cyberspace.
+
+Information WANTS to be free. And the information inside
+this book longs for freedom with a peculiar intensity.
+I genuinely believe that the natural habitat of this book
+is inside an electronic network. That may not be the easiest
+direct method to generate revenue for the book's author,
+but that doesn't matter; this is where this book belongs
+by its nature. I've written other books--plenty of other books--
+and I'll write more and I am writing more, but this one is special.
+I am making The Hacker Crackdown available electronically
+as widely as I can conveniently manage, and if you like the book,
+and think it is useful, then I urge you to do the same with it.
+
+You can copy this electronic book. Copy the heck out of it,
+be my guest, and give those copies to anybody who wants them.
+The nascent world of cyberspace is full of sysadmins, teachers,
+trainers, cybrarians, netgurus, and various species of cybernetic activist.
+If you're one of those people, I know about you, and I know the hassle
+you go through to try to help people learn about the electronic frontier.
+I hope that possessing this book in electronic form will lessen your troubles.
+Granted, this treatment of our electronic social spectrum is not the ultimate
+in academic rigor. And politically, it has something to offend
+and trouble almost everyone. But hey, I'm told it's readable,
+and at least the price is right.
+
+You can upload the book onto bulletin board systems, or Internet nodes,
+or electronic discussion groups. Go right ahead and do that, I am giving
+you express permission right now. Enjoy yourself.
+
+You can put the book on disks and give the disks away,
+as long as you don't take any money for it.
+
+But this book is not public domain. You can't copyright it in
+your own name. I own the copyright. Attempts to pirate this book
+and make money from selling it may involve you in a serious litigative snarl.
+Believe me, for the pittance you might wring out of such an action,
+it's really not worth it. This book don't "belong" to you.
+In an odd but very genuine way, I feel it doesn't "belong" to me, either.
+It's a book about the people of cyberspace, and distributing it in this way
+is the best way I know to actually make this information available,
+freely and easily, to all the people of cyberspace--including people
+far outside the borders of the United States, who otherwise may never
+have a chance to see any edition of the book, and who may perhaps learn
+something useful from this strange story of distant, obscure, but portentous
+events in so-called "American cyberspace."
+
+This electronic book is now literary freeware. It now belongs to the
+emergent realm of alternative information economics. You have no right
+to make this electronic book part of the conventional flow of commerce.
+Let it be part of the flow of knowledge: there's a difference.
+I've divided the book into four sections, so that it is less ungainly
+for upload and download; if there's a section of particular relevance
+to you and your colleagues, feel free to reproduce that one and skip the rest.
+
+[Project Gutenberg has reassembled the file, with Sterling's permission.]
+
+Just make more when you need them, and give them to whoever might want them.
+
+Now have fun.
+
+Bruce Sterling--bruces@well.sf.ca.us
+
+
+THE HACKER CRACKDOWN
+
+Law and Disorder on the Electronic Frontier
+
+by Bruce Sterling
+
+
+
+
+
+
+
+CHRONOLOGY OF THE HACKER CRACKDOWN
+
+
+1865 U.S. Secret Service (USSS) founded.
+
+1876 Alexander Graham Bell invents telephone.
+
+1878 First teenage males flung off phone system by enraged authorities.
+
+1939 "Futurian" science-fiction group raided by Secret Service.
+
+1971 Yippie phone phreaks start YIPL/TAP magazine.
+
+1972 RAMPARTS magazine seized in blue-box rip-off scandal.
+
+1978 Ward Christenson and Randy Suess create first personal
+ computer bulletin board system.
+
+1982 William Gibson coins term "cyberspace."
+
+1982 "414 Gang" raided.
+
+1983-1983 AT&T dismantled in divestiture.
+
+1984 Congress passes Comprehensive Crime Control Act giving USSS
+ jurisdiction over credit card fraud and computer fraud.
+
+1984 "Legion of Doom" formed.
+
+1984. 2600: THE HACKER QUARTERLY founded.
+
+1984. WHOLE EARTH SOFTWARE CATALOG published.
+
+1985. First police "sting" bulletin board systems established.
+
+1985. Whole Earth 'Lectronic Link computer conference (WELL) goes on-line.
+
+1986 Computer Fraud and Abuse Act passed.
+
+1986 Electronic Communications Privacy Act passed.
+
+1987 Chicago prosecutors form Computer Fraud and Abuse Task Force.
+
+
+1988
+
+July. Secret Service covertly videotapes "SummerCon" hacker convention.
+
+September. "Prophet" cracks BellSouth AIMSX computer network
+ and downloads E911 Document to his own computer and to Jolnet.
+
+September. AT&T Corporate Information Security informed of Prophet's action.
+
+October. Bellcore Security informed of Prophet's action.
+
+
+1989
+
+January. Prophet uploads E911 Document to Knight Lightning.
+
+February 25. Knight Lightning publishes E911 Document in PHRACK
+ electronic newsletter.
+
+May. Chicago Task Force raids and arrests "Kyrie."
+
+June. "NuPrometheus League" distributes Apple Computer proprietary software.
+
+June 13. Florida probation office crossed with phone-sex line
+ in switching-station stunt.
+
+July. "Fry Guy" raided by USSS and Chicago Computer Fraud
+ and Abuse Task Force.
+
+July. Secret Service raids "Prophet," "Leftist," and "Urvile" in Georgia.
+
+
+1990
+
+January 15. Martin Luther King Day Crash strikes AT&T long-distance
+ network nationwide.
+
+January 18-19. Chicago Task Force raids Knight Lightning in St. Louis.
+
+January 24. USSS and New York State Police raid "Phiber Optik,"
+ "Acid Phreak," and "Scorpion" in New York City.
+
+February 1. USSS raids "Terminus" in Maryland.
+
+February 3. Chicago Task Force raids Richard Andrews' home.
+
+February 6. Chicago Task Force raids Richard Andrews' business.
+
+February 6. USSS arrests Terminus, Prophet, Leftist, and Urvile.
+
+February 9. Chicago Task Force arrests Knight Lightning.
+
+February 20. AT&T Security shuts down public-access
+ "attctc" computer in Dallas.
+
+February 21. Chicago Task Force raids Robert Izenberg in Austin.
+
+March 1. Chicago Task Force raids Steve Jackson Games, Inc.,
+ "Mentor," and "Erik Bloodaxe" in Austin.
+
+May 7,8,9.
+
+USSS and Arizona Organized Crime and Racketeering Bureau conduct
+"Operation Sundevil" raids in Cincinnatti, Detroit, Los Angeles,
+Miami, Newark, Phoenix, Pittsburgh, Richmond, Tucson, San Diego,
+San Jose, and San Francisco.
+
+May. FBI interviews John Perry Barlow re NuPrometheus case.
+
+June. Mitch Kapor and Barlow found Electronic Frontier Foundation;
+ Barlow publishes CRIME AND PUZZLEMENT manifesto.
+
+July 24-27. Trial of Knight Lightning.
+
+1991
+
+February. CPSR Roundtable in Washington, D.C.
+
+March 25-28. Computers, Freedom and Privacy conference in San Francisco.
+
+May 1. Electronic Frontier Foundation, Steve Jackson,
+ and others file suit against members of Chicago Task Force.
+
+July 1-2. Switching station phone software crash affects
+ Washington, Los Angeles, Pittsburgh, San Francisco.
+
+September 17. AT&T phone crash affects New York City and three airports.
+
+
+
+
+Introduction
+
+This is a book about cops, and wild teenage whiz-kids, and lawyers,
+and hairy-eyed anarchists, and industrial technicians, and hippies,
+and high-tech millionaires, and game hobbyists, and computer security
+experts, and Secret Service agents, and grifters, and thieves.
+
+This book is about the electronic frontier of the 1990s.
+It concerns activities that take place inside computers
+and over telephone lines.
+
+A science fiction writer coined the useful term "cyberspace" in 1982,
+but the territory in question, the electronic frontier, is about
+a hundred and thirty years old. Cyberspace is the "place" where
+a telephone conversation appears to occur. Not inside your actual phone,
+the plastic device on your desk. Not inside the other person's phone,
+in some other city. THE PLACE BETWEEN the phones. The indefinite
+place OUT THERE, where the two of you, two human beings,
+actually meet and communicate.
+
+Although it is not exactly "real," "cyberspace" is a genuine place.
+Things happen there that have very genuine consequences. This "place"
+is not "real," but it is serious, it is earnest. Tens of thousands
+of people have dedicated their lives to it, to the public service
+of public communication by wire and electronics.
+
+People have worked on this "frontier" for generations now.
+Some people became rich and famous from their efforts there.
+Some just played in it, as hobbyists. Others soberly pondered it,
+and wrote about it, and regulated it, and negotiated over it in
+international forums, and sued one another about it, in gigantic,
+epic court battles that lasted for years. And almost since
+the beginning, some people have committed crimes in this place.
+
+But in the past twenty years, this electrical "space,"
+which was once thin and dark and one-dimensional--little more
+than a narrow speaking-tube, stretching from phone to phone--
+has flung itself open like a gigantic jack-in-the-box.
+Light has flooded upon it, the eerie light of the glowing computer screen.
+This dark electric netherworld has become a vast flowering electronic landscape.
+Since the 1960s, the world of the telephone has cross-bred itself
+with computers and television, and though there is still no substance
+to cyberspace, nothing you can handle, it has a strange kind
+of physicality now. It makes good sense today to talk of cyberspace
+as a place all its own.
+
+Because people live in it now. Not just a few people,
+not just a few technicians and eccentrics, but thousands
+of people, quite normal people. And not just for a little while,
+either, but for hours straight, over weeks, and months,
+and years. Cyberspace today is a "Net," a "Matrix,"
+international in scope and growing swiftly and steadily.
+It's growing in size, and wealth, and political importance.
+
+People are making entire careers in modern cyberspace.
+Scientists and technicians, of course; they've been there
+for twenty years now. But increasingly, cyberspace
+is filling with journalists and doctors and lawyers
+and artists and clerks. Civil servants make their
+careers there now, "on-line" in vast government data-banks;
+and so do spies, industrial, political, and just plain snoops;
+and so do police, at least a few of them. And there are children
+living there now.
+
+People have met there and been married there.
+There are entire living communities in cyberspace today;
+chattering, gossiping, planning, conferring and scheming,
+leaving one another voice-mail and electronic mail,
+giving one another big weightless chunks of valuable data,
+both legitimate and illegitimate. They busily pass one another
+computer software and the occasional festering computer virus.
+
+We do not really understand how to live in cyberspace yet.
+We are feeling our way into it, blundering about.
+That is not surprising. Our lives in the physical world,
+the "real" world, are also far from perfect, despite a lot more practice.
+Human lives, real lives, are imperfect by their nature, and there are
+human beings in cyberspace. The way we live in cyberspace is
+a funhouse mirror of the way we live in the real world.
+We take both our advantages and our troubles with us.
+
+This book is about trouble in cyberspace.
+Specifically, this book is about certain strange events in
+the year 1990, an unprecedented and startling year for the
+the growing world of computerized communications.
+
+In 1990 there came a nationwide crackdown on illicit
+computer hackers, with arrests, criminal charges,
+one dramatic show-trial, several guilty pleas, and
+huge confiscations of data and equipment all over the USA.
+
+The Hacker Crackdown of 1990 was larger, better organized,
+more deliberate, and more resolute than any previous effort
+in the brave new world of computer crime. The U.S. Secret Service,
+private telephone security, and state and local law enforcement groups
+across the country all joined forces in a determined attempt to break
+the back of America's electronic underground. It was a fascinating
+effort, with very mixed results.
+
+The Hacker Crackdown had another unprecedented effect;
+it spurred the creation, within "the computer community,"
+of the Electronic Frontier Foundation, a new and very odd
+interest group, fiercely dedicated to the establishment
+and preservation of electronic civil liberties. The crackdown,
+remarkable in itself, has created a melee of debate over electronic crime,
+punishment, freedom of the press, and issues of search and seizure.
+Politics has entered cyberspace. Where people go, politics follow.
+
+This is the story of the people of cyberspace.
+
+
+
+PART ONE: Crashing the System
+
+On January 15, 1990, AT&T's long-distance telephone switching system crashed.
+
+This was a strange, dire, huge event. Sixty thousand people lost
+their telephone service completely. During the nine long hours
+of frantic effort that it took to restore service, some seventy million
+telephone calls went uncompleted.
+
+Losses of service, known as "outages" in the telco trade,
+are a known and accepted hazard of the telephone business.
+Hurricanes hit, and phone cables get snapped by the thousands.
+Earthquakes wrench through buried fiber-optic lines.
+Switching stations catch fire and burn to the ground.
+These things do happen. There are contingency plans for them,
+and decades of experience in dealing with them.
+But the Crash of January 15 was unprecedented.
+It was unbelievably huge, and it occurred for
+no apparent physical reason.
+
+The crash started on a Monday afternoon in a single
+switching-station in Manhattan. But, unlike any merely
+physical damage, it spread and spread. Station after
+station across America collapsed in a chain reaction,
+until fully half of AT&T's network had gone haywire
+and the remaining half was hard-put to handle the overflow.
+
+Within nine hours, AT&T software engineers more or less
+understood what had caused the crash. Replicating the
+problem exactly, poring over software line by line,
+took them a couple of weeks. But because it was hard
+to understand technically, the full truth of the matter
+and its implications were not widely and thoroughly aired
+and explained. The root cause of the crash remained obscure,
+surrounded by rumor and fear.
+
+The crash was a grave corporate embarrassment.
+The "culprit" was a bug in AT&T's own software--not the
+sort of admission the telecommunications giant wanted
+to make, especially in the face of increasing competition.
+Still, the truth WAS told, in the baffling technical terms
+necessary to explain it.
+
+Somehow the explanation failed to persuade
+American law enforcement officials and even telephone
+corporate security personnel. These people were not
+technical experts or software wizards, and they had their
+own suspicions about the cause of this disaster.
+
+The police and telco security had important sources
+of information denied to mere software engineers.
+They had informants in the computer underground and
+years of experience in dealing with high-tech rascality
+that seemed to grow ever more sophisticated.
+For years they had been expecting a direct and
+savage attack against the American national telephone system.
+And with the Crash of January 15--the first month of a
+new, high-tech decade--their predictions, fears,
+and suspicions seemed at last to have entered the real world.
+A world where the telephone system had not merely crashed,
+but, quite likely, BEEN crashed--by "hackers."
+
+The crash created a large dark cloud of suspicion
+that would color certain people's assumptions and actions
+for months. The fact that it took place in the realm of
+software was suspicious on its face. The fact that it
+occurred on Martin Luther King Day, still the most
+politically touchy of American holidays, made it more
+suspicious yet.
+
+The Crash of January 15 gave the Hacker Crackdown
+its sense of edge and its sweaty urgency. It made people,
+powerful people in positions of public authority,
+willing to believe the worst. And, most fatally,
+it helped to give investigators a willingness
+to take extreme measures and the determination
+to preserve almost total secrecy.
+
+An obscure software fault in an aging switching system
+in New York was to lead to a chain reaction of legal
+and constitutional trouble all across the country.
+
+#
+
+Like the crash in the telephone system, this chain reaction
+was ready and waiting to happen. During the 1980s,
+the American legal system was extensively patched
+to deal with the novel issues of computer crime.
+There was, for instance, the Electronic Communications
+Privacy Act of 1986 (eloquently described as "a stinking mess"
+by a prominent law enforcement official). And there was the
+draconian Computer Fraud and Abuse Act of 1986, passed unanimously
+by the United States Senate, which later would reveal
+a large number of flaws. Extensive, well-meant efforts
+had been made to keep the legal system up to date.
+But in the day-to-day grind of the real world,
+even the most elegant software tends to crumble
+and suddenly reveal its hidden bugs.
+
+Like the advancing telephone system, the American legal system
+was certainly not ruined by its temporary crash; but for those
+caught under the weight of the collapsing system, life became
+a series of blackouts and anomalies.
+
+In order to understand why these weird events occurred,
+both in the world of technology and in the world of law,
+it's not enough to understand the merely technical problems.
+We will get to those; but first and foremost, we must try
+to understand the telephone, and the business of telephones,
+and the community of human beings that telephones have created.
+
+#
+
+Technologies have life cycles, like cities do,
+like institutions do, like laws and governments do.
+
+The first stage of any technology is the Question
+Mark, often known as the "Golden Vaporware" stage.
+At this early point, the technology is only a phantom,
+a mere gleam in the inventor's eye. One such inventor
+was a speech teacher and electrical tinkerer named
+Alexander Graham Bell.
+
+Bell's early inventions, while ingenious, failed to move the world.
+In 1863, the teenage Bell and his brother Melville made an artificial
+talking mechanism out of wood, rubber, gutta-percha, and tin.
+This weird device had a rubber-covered "tongue" made of movable
+wooden segments, with vibrating rubber "vocal cords," and
+rubber "lips" and "cheeks." While Melville puffed a bellows
+into a tin tube, imitating the lungs, young Alec Bell would
+manipulate the "lips," "teeth," and "tongue," causing the thing
+to emit high-pitched falsetto gibberish.
+
+Another would-be technical breakthrough was the Bell "phonautograph"
+of 1874, actually made out of a human cadaver's ear. Clamped into place
+on a tripod, this grisly gadget drew sound-wave images on smoked glass
+through a thin straw glued to its vibrating earbones.
+
+By 1875, Bell had learned to produce audible sounds--ugly shrieks
+and squawks--by using magnets, diaphragms, and electrical current.
+
+Most "Golden Vaporware" technologies go nowhere.
+
+But the second stage of technology is the Rising Star,
+or, the "Goofy Prototype," stage. The telephone, Bell's
+most ambitious gadget yet, reached this stage on March
+10, 1876. On that great day, Alexander Graham Bell
+became the first person to transmit intelligible human
+speech electrically. As it happened, young Professor Bell,
+industriously tinkering in his Boston lab, had spattered
+his trousers with acid. His assistant, Mr. Watson,
+heard his cry for help--over Bell's experimental
+audio-telegraph. This was an event without precedent.
+
+Technologies in their "Goofy Prototype" stage rarely
+work very well. They're experimental, and therefore
+half- baked and rather frazzled. The prototype may
+be attractive and novel, and it does look as if it ought
+to be good for something-or-other. But nobody, including
+the inventor, is quite sure what. Inventors, and speculators,
+and pundits may have very firm ideas about its potential
+use, but those ideas are often very wrong.
+
+The natural habitat of the Goofy Prototype is in trade shows
+and in the popular press. Infant technologies need publicity
+and investment money like a tottering calf need milk.
+This was very true of Bell's machine. To raise research and
+development money, Bell toured with his device as a stage attraction.
+
+Contemporary press reports of the stage debut of the telephone
+showed pleased astonishment mixed with considerable dread.
+Bell's stage telephone was a large wooden box with a crude
+speaker-nozzle, the whole contraption about the size and shape
+of an overgrown Brownie camera. Its buzzing steel soundplate,
+pumped up by powerful electromagnets, was loud enough to fill
+an auditorium. Bell's assistant Mr. Watson, who could manage
+on the keyboards fairly well, kicked in by playing the organ
+from distant rooms, and, later, distant cities. This feat was
+considered marvellous, but very eerie indeed.
+
+Bell's original notion for the telephone, an idea promoted
+for a couple of years, was that it would become a mass medium.
+We might recognize Bell's idea today as something close to modern
+"cable radio." Telephones at a central source would transmit music,
+Sunday sermons, and important public speeches to a paying network
+of wired-up subscribers.
+
+At the time, most people thought this notion made good sense.
+In fact, Bell's idea was workable. In Hungary, this philosophy
+of the telephone was successfully put into everyday practice.
+In Budapest, for decades, from 1893 until after World War I,
+there was a government-run information service called
+"Telefon Hirmondo-." Hirmondo- was a centralized source
+of news and entertainment and culture, including stock reports,
+plays, concerts, and novels read aloud. At certain hours
+of the day, the phone would ring, you would plug in
+a loudspeaker for the use of the family, and Telefon
+Hirmondo- would be on the air--or rather, on the phone.
+
+Hirmondo- is dead tech today, but Hirmondo- might be considered
+a spiritual ancestor of the modern telephone-accessed computer
+data services, such as CompuServe, GEnie or Prodigy.
+The principle behind Hirmondo- is also not too far from computer
+"bulletin- board systems" or BBS's, which arrived in the late 1970s,
+spread rapidly across America, and will figure largely in this book.
+
+We are used to using telephones for individual person-to-person speech,
+because we are used to the Bell system. But this was just one possibility
+among many. Communication networks are very flexible and protean,
+especially when their hardware becomes sufficiently advanced.
+They can be put to all kinds of uses. And they have been--
+and they will be.
+
+Bell's telephone was bound for glory, but this was a combination
+of political decisions, canny infighting in court, inspired industrial
+leadership, receptive local conditions and outright good luck.
+Much the same is true of communications systems today.
+
+As Bell and his backers struggled to install their newfangled system
+in the real world of nineteenth-century New England, they had to fight
+against skepticism and industrial rivalry. There was already a strong
+electrical communications network present in America: the telegraph.
+The head of the Western Union telegraph system dismissed Bell's prototype
+as "an electrical toy" and refused to buy the rights to Bell's patent.
+The telephone, it seemed, might be all right as a parlor entertainment--
+but not for serious business.
+
+Telegrams, unlike mere telephones, left a permanent physical record
+of their messages. Telegrams, unlike telephones, could be answered
+whenever the recipient had time and convenience. And the telegram
+had a much longer distance-range than Bell's early telephone.
+These factors made telegraphy seem a much more sound and businesslike
+technology--at least to some.
+
+The telegraph system was huge, and well-entrenched.
+In 1876, the United States had 214,000 miles of telegraph wire,
+and 8500 telegraph offices. There were specialized telegraphs
+for businesses and stock traders, government, police and fire departments.
+And Bell's "toy" was best known as a stage-magic musical device.
+
+The third stage of technology is known as the "Cash Cow" stage.
+In the "cash cow" stage, a technology finds its place in the world,
+and matures, and becomes settled and productive. After a year or so,
+Alexander Graham Bell and his capitalist backers concluded that
+eerie music piped from nineteenth-century cyberspace was not the real
+selling-point of his invention. Instead, the telephone was about speech--
+individual, personal speech, the human voice, human conversation and
+human interaction. The telephone was not to be managed from any centralized
+broadcast center. It was to be a personal, intimate technology.
+
+When you picked up a telephone, you were not absorbing
+the cold output of a machine--you were speaking to another human being.
+Once people realized this, their instinctive dread of the telephone
+as an eerie, unnatural device, swiftly vanished. A "telephone call"
+was not a "call" from a "telephone" itself, but a call from another
+human being, someone you would generally know and recognize.
+The real point was not what the machine could do for you (or to you),
+but what you yourself, a person and citizen, could do THROUGH the machine.
+This decision on the part of the young Bell Company was absolutely vital.
+
+The first telephone networks went up around Boston--mostly among
+the technically curious and the well-to-do (much the same segment
+of the American populace that, a hundred years later, would be
+buying personal computers). Entrenched backers of the telegraph
+continued to scoff.
+
+But in January 1878, a disaster made the telephone famous.
+A train crashed in Tarriffville, Connecticut. Forward-looking
+doctors in the nearby city of Hartford had had Bell's
+"speaking telephone" installed. An alert local druggist
+was able to telephone an entire community of local doctors,
+who rushed to the site to give aid. The disaster, as disasters do,
+aroused intense press coverage. The phone had proven its usefulness
+in the real world.
+
+After Tarriffville, the telephone network spread like crabgrass.
+By 1890 it was all over New England. By '93, out to Chicago.
+By '97, into Minnesota, Nebraska and Texas. By 1904 it was
+all over the continent.
+
+The telephone had become a mature technology. Professor Bell
+(now generally known as "Dr. Bell" despite his lack of a formal degree)
+became quite wealthy. He lost interest in the tedious day-to-day business
+muddle of the booming telephone network, and gratefully returned
+his attention to creatively hacking-around in his various laboratories,
+which were now much larger, better-ventilated, and gratifyingly
+better-equipped. Bell was never to have another great inventive success,
+though his speculations and prototypes anticipated fiber-optic transmission,
+manned flight, sonar, hydrofoil ships, tetrahedral construction, and
+Montessori education. The "decibel," the standard scientific measure
+of sound intensity, was named after Bell.
+
+Not all Bell's vaporware notions were inspired. He was fascinated
+by human eugenics. He also spent many years developing a weird personal
+system of astrophysics in which gravity did not exist.
+
+Bell was a definite eccentric. He was something of a hypochondriac,
+and throughout his life he habitually stayed up until four A.M.,
+refusing to rise before noon. But Bell had accomplished a great feat;
+he was an idol of millions and his influence, wealth, and great
+personal charm, combined with his eccentricity, made him something
+of a loose cannon on deck. Bell maintained a thriving scientific
+salon in his winter mansion in Washington, D.C., which gave him
+considerable backstage influence in governmental and scientific circles.
+He was a major financial backer of the the magazines Science and
+National Geographic, both still flourishing today as important organs
+of the American scientific establishment.
+
+Bell's companion Thomas Watson, similarly wealthy and similarly odd,
+became the ardent political disciple of a 19th-century science-fiction writer
+and would-be social reformer, Edward Bellamy. Watson also trod the boards
+briefly as a Shakespearian actor.
+
+There would never be another Alexander Graham Bell,
+but in years to come there would be surprising numbers
+of people like him. Bell was a prototype of the
+high-tech entrepreneur. High-tech entrepreneurs will
+play a very prominent role in this book: not merely as
+technicians and businessmen, but as pioneers of the
+technical frontier, who can carry the power and prestige
+they derive from high-technology into the political and
+social arena.
+
+Like later entrepreneurs, Bell was fierce in defense of
+his own technological territory. As the telephone began to
+flourish, Bell was soon involved in violent lawsuits in the
+defense of his patents. Bell's Boston lawyers were
+excellent, however, and Bell himself, as an elocution
+teacher and gifted public speaker, was a devastatingly
+effective legal witness. In the eighteen years of Bell's patents,
+the Bell company was involved in six hundred separate lawsuits.
+The legal records printed filled 149 volumes. The Bell Company
+won every single suit.
+
+After Bell's exclusive patents expired, rival telephone
+companies sprang up all over America. Bell's company,
+American Bell Telephone, was soon in deep trouble.
+In 1907, American Bell Telephone fell into the hands of the
+rather sinister J.P. Morgan financial cartel, robber-baron
+speculators who dominated Wall Street.
+
+At this point, history might have taken a different turn.
+American might well have been served forever by a patchwork
+of locally owned telephone companies. Many state politicians
+and local businessmen considered this an excellent solution.
+
+But the new Bell holding company, American Telephone and Telegraph
+or AT&T, put in a new man at the helm, a visionary industrialist
+named Theodore Vail. Vail, a former Post Office manager,
+understood large organizations and had an innate feeling
+for the nature of large-scale communications. Vail quickly
+saw to it that AT&T seized the technological edge once again.
+The Pupin and Campbell "loading coil," and the deForest
+"audion," are both extinct technology today, but in 1913
+they gave Vail's company the best LONG-DISTANCE lines
+ever built. By controlling long-distance--the links
+between, and over, and above the smaller local phone
+companies--AT&T swiftly gained the whip-hand over them,
+and was soon devouring them right and left.
+
+Vail plowed the profits back into research and development,
+starting the Bell tradition of huge-scale and brilliant
+industrial research.
+
+Technically and financially, AT&T gradually steamrollered
+the opposition. Independent telephone companies never
+became entirely extinct, and hundreds of them flourish today.
+But Vail's AT&T became the supreme communications company.
+At one point, Vail's AT&T bought Western Union itself,
+the very company that had derided Bell's telephone as a "toy."
+Vail thoroughly reformed Western Union's hidebound business
+along his modern principles; but when the federal government
+grew anxious at this centralization of power, Vail politely
+gave Western Union back.
+
+This centralizing process was not unique. Very similar
+events had happened in American steel, oil, and railroads.
+But AT&T, unlike the other companies, was to remain supreme.
+The monopoly robber-barons of those other industries
+were humbled and shattered by government trust-busting.
+
+Vail, the former Post Office official, was quite willing
+to accommodate the US government; in fact he would
+forge an active alliance with it. AT&T would become
+almost a wing of the American government, almost
+another Post Office--though not quite. AT&T would
+willingly submit to federal regulation, but in return,
+it would use the government's regulators as its own police,
+who would keep out competitors and assure the Bell
+system's profits and preeminence.
+
+This was the second birth--the political birth--of the
+American telephone system. Vail's arrangement was to
+persist, with vast success, for many decades, until 1982.
+His system was an odd kind of American industrial socialism.
+It was born at about the same time as Leninist Communism,
+and it lasted almost as long--and, it must be admitted,
+to considerably better effect.
+
+Vail's system worked. Except perhaps for aerospace,
+there has been no technology more thoroughly dominated
+by Americans than the telephone. The telephone was
+seen from the beginning as a quintessentially American
+technology. Bell's policy, and the policy of Theodore Vail,
+was a profoundly democratic policy of UNIVERSAL ACCESS.
+Vail's famous corporate slogan, "One Policy, One System,
+Universal Service," was a political slogan, with a very
+American ring to it.
+
+The American telephone was not to become the specialized tool
+of government or business, but a general public utility.
+At first, it was true, only the wealthy could afford
+private telephones, and Bell's company pursued the
+business markets primarily. The American phone system
+was a capitalist effort, meant to make money; it was not a charity.
+But from the first, almost all communities with telephone service
+had public telephones. And many stores--especially drugstores--
+offered public use of their phones. You might not own a telephone--
+but you could always get into the system, if you really needed to.
+
+There was nothing inevitable about this decision to make telephones
+"public" and "universal." Vail's system involved a profound act
+of trust in the public. This decision was a political one,
+informed by the basic values of the American republic.
+The situation might have been very different;
+and in other countries, under other systems,
+it certainly was.
+
+Joseph Stalin, for instance, vetoed plans for a Soviet
+phone system soon after the Bolshevik revolution.
+Stalin was certain that publicly accessible telephones
+would become instruments of anti-Soviet counterrevolution
+and conspiracy. (He was probably right.) When telephones
+did arrive in the Soviet Union, they would be instruments
+of Party authority, and always heavily tapped. (Alexander
+Solzhenitsyn's prison-camp novel The First Circle
+describes efforts to develop a phone system more suited
+to Stalinist purposes.)
+
+France, with its tradition of rational centralized government,
+had fought bitterly even against the electric telegraph,
+which seemed to the French entirely too anarchical and frivolous.
+For decades, nineteenth-century France communicated via the
+"visual telegraph," a nation-spanning, government-owned semaphore
+system of huge stone towers that signalled from hilltops,
+across vast distances, with big windmill-like arms.
+In 1846, one Dr. Barbay, a semaphore enthusiast,
+memorably uttered an early version of what might be called
+"the security expert's argument" against the open media.
+
+"No, the electric telegraph is not a sound invention.
+It will always be at the mercy of the slightest disruption,
+wild youths, drunkards, bums, etc. . . . The electric telegraph
+meets those destructive elements with only a few meters of wire
+over which supervision is impossible. A single man could,
+without being seen, cut the telegraph wires leading to Paris,
+and in twenty-four hours cut in ten different places the wires
+of the same line, without being arrested. The visual telegraph,
+on the contrary, has its towers, its high walls, its gates
+well-guarded from inside by strong armed men. Yes, I declare,
+substitution of the electric telegraph for the visual one
+is a dreadful measure, a truly idiotic act."
+
+Dr. Barbay and his high-security stone machines
+were eventually unsuccessful, but his argument--
+that communication exists for the safety and convenience
+of the state, and must be carefully protected from the wild
+boys and the gutter rabble who might want to crash the
+system--would be heard again and again.
+
+When the French telephone system finally did arrive,
+its snarled inadequacy was to be notorious. Devotees
+of the American Bell System often recommended a trip
+to France, for skeptics.
+
+In Edwardian Britain, issues of class and privacy
+were a ball-and-chain for telephonic progress. It was
+considered outrageous that anyone--any wild fool off
+the street--could simply barge bellowing into one's office
+or home, preceded only by the ringing of a telephone bell.
+In Britain, phones were tolerated for the use of business,
+but private phones tended be stuffed away into closets,
+smoking rooms, or servants' quarters. Telephone operators
+were resented in Britain because they did not seem to
+"know their place." And no one of breeding would print
+a telephone number on a business card; this seemed a crass
+attempt to make the acquaintance of strangers.
+
+But phone access in America was to become a popular right;
+something like universal suffrage, only more so.
+American women could not yet vote when the phone system
+came through; yet from the beginning American women
+doted on the telephone. This "feminization" of the
+American telephone was often commented on by foreigners.
+Phones in America were not censored or stiff or formalized;
+they were social, private, intimate, and domestic.
+In America, Mother's Day is by far the busiest day
+of the year for the phone network.
+
+The early telephone companies, and especially AT&T,
+were among the foremost employers of American women.
+They employed the daughters of the American middle-class
+in great armies: in 1891, eight thousand women; by 1946,
+almost a quarter of a million. Women seemed to enjoy
+telephone work; it was respectable, it was steady,
+it paid fairly well as women's work went, and--not least--
+it seemed a genuine contribution to the social good
+of the community. Women found Vail's ideal of public
+service attractive. This was especially true in rural areas,
+where women operators, running extensive rural party-lines,
+enjoyed considerable social power. The operator knew everyone
+on the party-line, and everyone knew her.
+
+Although Bell himself was an ardent suffragist, the
+telephone company did not employ women for the sake of
+advancing female liberation. AT&T did this for sound
+commercial reasons. The first telephone operators of
+the Bell system were not women, but teenage American boys.
+They were telegraphic messenger boys (a group about to
+be rendered technically obsolescent), who swept up
+around the phone office, dunned customers for bills,
+and made phone connections on the switchboard,
+all on the cheap.
+
+Within the very first year of operation, 1878,
+Bell's company learned a sharp lesson about combining
+teenage boys and telephone switchboards. Putting
+teenage boys in charge of the phone system brought swift
+and consistent disaster. Bell's chief engineer described them
+as "Wild Indians." The boys were openly rude to customers.
+They talked back to subscribers, saucing off,
+uttering facetious remarks, and generally giving lip.
+The rascals took Saint Patrick's Day off without permission.
+And worst of all they played clever tricks with
+the switchboard plugs: disconnecting calls, crossing lines
+so that customers found themselves talking to strangers,
+and so forth.
+
+This combination of power, technical mastery, and effective
+anonymity seemed to act like catnip on teenage boys.
+
+This wild-kid-on-the-wires phenomenon was not confined to
+the USA; from the beginning, the same was true of the British
+phone system. An early British commentator kindly remarked:
+"No doubt boys in their teens found the work not a little irksome,
+and it is also highly probable that under the early conditions
+of employment the adventurous and inquisitive spirits of which
+the average healthy boy of that age is possessed, were not always
+conducive to the best attention being given to the wants
+of the telephone subscribers."
+
+So the boys were flung off the system--or at least,
+deprived of control of the switchboard. But the
+"adventurous and inquisitive spirits" of the teenage boys
+would be heard from in the world of telephony, again and again.
+
+The fourth stage in the technological life-cycle is death:
+"the Dog," dead tech. The telephone has so far avoided this fate.
+On the contrary, it is thriving, still spreading, still evolving,
+and at increasing speed.
+
+The telephone has achieved a rare and exalted state for a
+technological artifact: it has become a HOUSEHOLD OBJECT.
+The telephone, like the clock, like pen and paper,
+like kitchen utensils and running water, has become
+a technology that is visible only by its absence.
+The telephone is technologically transparent.
+The global telephone system is the largest and most
+complex machine in the world, yet it is easy to use.
+More remarkable yet, the telephone is almost entirely
+physically safe for the user.
+
+For the average citizen in the 1870s, the telephone
+was weirder, more shocking, more "high-tech" and
+harder to comprehend, than the most outrageous stunts
+of advanced computing for us Americans in the 1990s.
+In trying to understand what is happening to us today,
+with our bulletin-board systems, direct overseas dialling,
+fiber-optic transmissions, computer viruses, hacking stunts,
+and a vivid tangle of new laws and new crimes, it is important
+to realize that our society has been through a similar challenge before--
+and that, all in all, we did rather well by it.
+
+Bell's stage telephone seemed bizarre at first. But the
+sensations of weirdness vanished quickly, once people began
+to hear the familiar voices of relatives and friends,
+in their own homes on their own telephones. The telephone
+changed from a fearsome high-tech totem to an everyday pillar
+of human community.
+
+This has also happened, and is still happening,
+to computer networks. Computer networks such as
+NSFnet, BITnet, USENET, JANET, are technically
+advanced, intimidating, and much harder to use than
+telephones. Even the popular, commercial computer
+networks, such as GEnie, Prodigy, and CompuServe,
+cause much head-scratching and have been described
+as "user-hateful." Nevertheless they too are changing
+from fancy high-tech items into everyday sources
+of human community.
+
+The words "community" and "communication" have
+the same root. Wherever you put a communications
+network, you put a community as well. And whenever
+you TAKE AWAY that network--confiscate it, outlaw it,
+crash it, raise its price beyond affordability--
+then you hurt that community.
+
+Communities will fight to defend themselves. People will fight harder
+and more bitterly to defend their communities, than they will fight
+to defend their own individual selves. And this is very true
+of the "electronic community" that arose around computer networks
+in the 1980s--or rather, the VARIOUS electronic communities,
+in telephony, law enforcement, computing, and the digital
+underground that, by the year 1990, were raiding, rallying,
+arresting, suing, jailing, fining and issuing angry manifestos.
+
+None of the events of 1990 were entirely new.
+Nothing happened in 1990 that did not have some kind
+of earlier and more understandable precedent. What gave
+the Hacker Crackdown its new sense of gravity and
+importance was the feeling--the COMMUNITY feeling--
+that the political stakes had been raised; that trouble
+in cyberspace was no longer mere mischief or inconclusive
+skirmishing, but a genuine fight over genuine issues,
+a fight for community survival and the shape of the future.
+
+These electronic communities, having flourished throughout
+the 1980s, were becoming aware of themselves, and increasingly,
+becoming aware of other, rival communities. Worries were
+sprouting up right and left, with complaints, rumors,
+uneasy speculations. But it would take a catalyst, a shock,
+to make the new world evident. Like Bell's great publicity break,
+the Tarriffville Rail Disaster of January 1878,
+it would take a cause celebre.
+
+That cause was the AT&T Crash of January 15, 1990.
+After the Crash, the wounded and anxious telephone
+community would come out fighting hard.
+
+#
+
+The community of telephone technicians, engineers, operators
+and researchers is the oldest community in cyberspace.
+These are the veterans, the most developed group,
+the richest, the most respectable, in most ways the most powerful.
+Whole generations have come and gone since Alexander Graham Bell's day,
+but the community he founded survives; people work for the phone system
+today whose great-grandparents worked for the phone system.
+Its specialty magazines, such as Telephony, AT&T Technical Journal,
+Telephone Engineer and Management, are decades old;
+they make computer publications like Macworld and PC Week
+look like amateur johnny-come-latelies.
+
+And the phone companies take no back seat in high-technology, either.
+Other companies' industrial researchers may have won new markets;
+but the researchers of Bell Labs have won SEVEN NOBEL PRIZES.
+One potent device that Bell Labs originated, the transistor,
+has created entire GROUPS of industries. Bell Labs are
+world-famous for generating "a patent a day," and have even
+made vital discoveries in astronomy, physics and cosmology.
+
+Throughout its seventy-year history, "Ma Bell" was not so much
+a company as a way of life. Until the cataclysmic divestiture
+of the 1980s, Ma Bell was perhaps the ultimate maternalist mega-employer.
+The AT&T corporate image was the "gentle giant," "the voice with a smile,"
+a vaguely socialist-realist world of cleanshaven linemen in shiny helmets
+and blandly pretty phone-girls in headsets and nylons. Bell System
+employees were famous as rock-ribbed Kiwanis and Rotary members,
+Little-League enthusiasts, school-board people.
+
+During the long heyday of Ma Bell, the Bell employee corps
+were nurtured top-to-bottom on a corporate ethos of public service.
+There was good money in Bell, but Bell was not ABOUT money;
+Bell used public relations, but never mere marketeering.
+People went into the Bell System for a good life,
+and they had a good life. But it was not mere money
+that led Bell people out in the midst of storms and earthquakes
+to fight with toppled phone-poles, to wade in flooded manholes,
+to pull the red-eyed graveyard-shift over collapsing switching-systems.
+The Bell ethic was the electrical equivalent of the postman's:
+neither rain, nor snow, nor gloom of night would stop these couriers.
+
+It is easy to be cynical about this, as it is easy to be
+cynical about any political or social system; but cynicism
+does not change the fact that thousands of people took
+these ideals very seriously. And some still do.
+
+The Bell ethos was about public service; and that was
+gratifying; but it was also about private POWER, and that
+was gratifying too. As a corporation, Bell was very special.
+Bell was privileged. Bell had snuggled up close to the state.
+In fact, Bell was as close to government as you could get in
+America and still make a whole lot of legitimate money.
+
+But unlike other companies, Bell was above and beyond
+the vulgar commercial fray. Through its regional operating companies,
+Bell was omnipresent, local, and intimate, all over America;
+but the central ivory towers at its corporate heart were the
+tallest and the ivoriest around.
+
+There were other phone companies in America, to be sure;
+the so-called independents. Rural cooperatives, mostly;
+small fry, mostly tolerated, sometimes warred upon.
+For many decades, "independent" American phone companies
+lived in fear and loathing of the official Bell monopoly
+(or the "Bell Octopus," as Ma Bell's nineteenth-century
+enemies described her in many angry newspaper manifestos).
+Some few of these independent entrepreneurs, while legally
+in the wrong, fought so bitterly against the Octopus
+that their illegal phone networks were cast into the street
+by Bell agents and publicly burned.
+
+The pure technical sweetness of the Bell System gave its operators,
+inventors and engineers a deeply satisfying sense of power and mastery.
+They had devoted their lives to improving this vast nation-spanning machine;
+over years, whole human lives, they had watched it improve and grow.
+It was like a great technological temple. They were an elite,
+and they knew it--even if others did not; in fact, they felt
+even more powerful BECAUSE others did not understand.
+
+The deep attraction of this sensation of elite technical power
+should never be underestimated. "Technical power" is not for everybody;
+for many people it simply has no charm at all. But for some people,
+it becomes the core of their lives. For a few, it is overwhelming,
+obsessive; it becomes something close to an addiction. People--especially
+clever teenage boys whose lives are otherwise mostly powerless and put-upon
+--love this sensation of secret power, and are willing to do all sorts
+of amazing things to achieve it. The technical POWER of electronics
+has motivated many strange acts detailed in this book, which would
+otherwise be inexplicable.
+
+So Bell had power beyond mere capitalism. The Bell service ethos worked,
+and was often propagandized, in a rather saccharine fashion. Over the decades,
+people slowly grew tired of this. And then, openly impatient with it.
+By the early 1980s, Ma Bell was to find herself with scarcely a real friend
+in the world. Vail's industrial socialism had become hopelessly
+out-of-fashion politically. Bell would be punished for that.
+And that punishment would fall harshly upon the people of the
+telephone community.
+
+#
+
+In 1983, Ma Bell was dismantled by federal court action.
+The pieces of Bell are now separate corporate entities.
+The core of the company became AT&T Communications,
+and also AT&T Industries (formerly Western Electric,
+Bell's manufacturing arm). AT&T Bell Labs became Bell
+Communications Research, Bellcore. Then there are the
+Regional Bell Operating Companies, or RBOCs, pronounced "arbocks."
+
+Bell was a titan and even these regional chunks are gigantic enterprises:
+Fortune 50 companies with plenty of wealth and power behind them.
+But the clean lines of "One Policy, One System, Universal Service"
+have been shattered, apparently forever.
+
+The "One Policy" of the early Reagan Administration was to
+shatter a system that smacked of noncompetitive socialism.
+Since that time, there has been no real telephone "policy"
+on the federal level. Despite the breakup, the remnants
+of Bell have never been set free to compete in the open marketplace.
+
+The RBOCs are still very heavily regulated, but not from the top.
+Instead, they struggle politically, economically and legally,
+in what seems an endless turmoil, in a patchwork of overlapping federal
+and state jurisdictions. Increasingly, like other major American corporations,
+the RBOCs are becoming multinational, acquiring important commercial interests
+in Europe, Latin America, and the Pacific Rim. But this, too, adds to their
+legal and political predicament.
+
+The people of what used to be Ma Bell are not happy about their fate.
+They feel ill-used. They might have been grudgingly willing to make
+a full transition to the free market; to become just companies amid
+other companies. But this never happened. Instead, AT&T and the RBOCS
+("the Baby Bells") feel themselves wrenched from side to side by state
+regulators, by Congress, by the FCC, and especially by the federal court
+of Judge Harold Greene, the magistrate who ordered the Bell breakup
+and who has been the de facto czar of American telecommunications
+ever since 1983.
+
+Bell people feel that they exist in a kind of paralegal limbo today.
+They don't understand what's demanded of them. If it's "service,"
+why aren't they treated like a public service? And if it's money,
+then why aren't they free to compete for it? No one seems to know,
+really. Those who claim to know keep changing their minds.
+Nobody in authority seems willing to grasp the nettle for once and all.
+
+Telephone people from other countries are amazed by the
+American telephone system today. Not that it works so well;
+for nowadays even the French telephone system works, more or less.
+They are amazed that the American telephone system STILL works
+AT ALL, under these strange conditions.
+
+Bell's "One System" of long-distance service is now only about
+eighty percent of a system, with the remainder held by Sprint, MCI,
+and the midget long-distance companies. Ugly wars over dubious
+corporate practices such as "slamming" (an underhanded method
+of snitching clients from rivals) break out with some regularity
+in the realm of long-distance service. The battle to break Bell's
+long-distance monopoly was long and ugly, and since the breakup
+the battlefield has not become much prettier. AT&T's famous
+shame-and-blame advertisements, which emphasized the shoddy work
+and purported ethical shadiness of their competitors, were much
+remarked on for their studied psychological cruelty.
+
+There is much bad blood in this industry, and much
+long-treasured resentment. AT&T's post-breakup
+corporate logo, a striped sphere, is known in the
+industry as the "Death Star" (a reference from the movie
+Star Wars, in which the "Death Star" was the spherical
+high- tech fortress of the harsh-breathing imperial ultra-baddie,
+Darth Vader.) Even AT&T employees are less than thrilled
+by the Death Star. A popular (though banned) T-shirt among
+AT&T employees bears the old-fashioned Bell logo of the Bell System,
+plus the newfangled striped sphere, with the before-and-after comments:
+"This is your brain--This is your brain on drugs!" AT&T made a very
+well-financed and determined effort to break into the personal
+computer market; it was disastrous, and telco computer experts
+are derisively known by their competitors as "the pole-climbers."
+AT&T and the Baby Bell arbocks still seem to have few friends.
+
+Under conditions of sharp commercial competition, a crash like
+that of January 15, 1990 was a major embarrassment to AT&T.
+It was a direct blow against their much-treasured reputation
+for reliability. Within days of the crash AT&T's
+Chief Executive Officer, Bob Allen, officially apologized,
+in terms of deeply pained humility:
+
+"AT&T had a major service disruption last Monday.
+We didn't live up to our own standards of quality,
+and we didn't live up to yours. It's as simple as that.
+And that's not acceptable to us. Or to you. . . .
+We understand how much people have come to depend
+upon AT&T service, so our AT&T Bell Laboratories scientists
+and our network engineers are doing everything possible
+to guard against a recurrence. . . . We know there's no way
+to make up for the inconvenience this problem may have caused you."
+
+Mr Allen's "open letter to customers" was printed in lavish ads
+all over the country: in the Wall Street Journal, USA Today,
+New York Times, Los Angeles Times, Chicago Tribune,
+Philadelphia Inquirer, San Francisco Chronicle Examiner,
+Boston Globe, Dallas Morning News, Detroit Free Press,
+Washington Post, Houston Chronicle, Cleveland Plain Dealer,
+Atlanta Journal Constitution, Minneapolis Star Tribune,
+St. Paul Pioneer Press Dispatch, Seattle Times/Post Intelligencer,
+Tacoma News Tribune, Miami Herald, Pittsburgh Press,
+St. Louis Post Dispatch, Denver Post, Phoenix Republic Gazette
+and Tampa Tribune.
+
+In another press release, AT&T went to some pains to suggest
+that this "software glitch" might have happened just as easily to MCI,
+although, in fact, it hadn't. (MCI's switching software was quite different
+from AT&T's--though not necessarily any safer.) AT&T also announced
+their plans to offer a rebate of service on Valentine's Day to make up
+for the loss during the Crash.
+
+"Every technical resource available, including Bell Labs
+scientists and engineers, has been devoted to assuring
+it will not occur again," the public was told. They were
+further assured that "The chances of a recurrence are small--
+a problem of this magnitude never occurred before."
+
+In the meantime, however, police and corporate
+security maintained their own suspicions about
+"the chances of recurrence" and the real reason why
+a "problem of this magnitude" had appeared, seemingly
+out of nowhere. Police and security knew for a fact
+that hackers of unprecedented sophistication were illegally
+entering, and reprogramming, certain digital switching stations.
+Rumors of hidden "viruses" and secret "logic bombs"
+in the switches ran rampant in the underground,
+with much chortling over AT&T's predicament,
+and idle speculation over what unsung hacker genius
+was responsible for it. Some hackers, including police
+informants, were trying hard to finger one another
+as the true culprits of the Crash.
+
+Telco people found little comfort in objectivity when
+they contemplated these possibilities. It was just too close
+to the bone for them; it was embarrassing; it hurt so much,
+it was hard even to talk about.
+
+There has always been thieving and misbehavior in the phone system.
+There has always been trouble with the rival independents,
+and in the local loops. But to have such trouble in the core
+of the system, the long-distance switching stations,
+is a horrifying affair. To telco people, this is
+all the difference between finding roaches in your kitchen
+and big horrid sewer-rats in your bedroom.
+
+From the outside, to the average citizen, the telcos
+still seem gigantic and impersonal. The American public
+seems to regard them as something akin to Soviet apparats.
+Even when the telcos do their best corporate-citizen routine,
+subsidizing magnet high-schools and sponsoring news-shows
+on public television, they seem to win little except public suspicion.
+
+But from the inside, all this looks very different.
+There's harsh competition. A legal and political system
+that seems baffled and bored, when not actively hostile
+to telco interests. There's a loss of morale, a deep sensation
+of having somehow lost the upper hand. Technological change
+has caused a loss of data and revenue to other, newer forms
+of transmission. There's theft, and new forms of theft,
+of growing scale and boldness and sophistication.
+With all these factors, it was no surprise to see the telcos,
+large and small, break out in a litany of bitter complaint.
+
+In late '88 and throughout 1989, telco representatives
+grew shrill in their complaints to those few American law
+enforcement officials who make it their business to try to
+understand what telephone people are talking about.
+Telco security officials had discovered the computer-
+hacker underground, infiltrated it thoroughly,
+and become deeply alarmed at its growing expertise.
+Here they had found a target that was not only loathsome
+on its face, but clearly ripe for counterattack.
+
+Those bitter rivals: AT&T, MCI and Sprint--and a crowd
+of Baby Bells: PacBell, Bell South, Southwestern Bell,
+NYNEX, USWest, as well as the Bell research consortium Bellcore,
+and the independent long-distance carrier Mid-American--
+all were to have their role in the great hacker dragnet of 1990.
+After years of being battered and pushed around, the telcos had,
+at least in a small way, seized the initiative again.
+After years of turmoil, telcos and government officials were
+once again to work smoothly in concert in defense of the System.
+Optimism blossomed; enthusiasm grew on all sides;
+the prospective taste of vengeance was sweet.
+
+#
+
+From the beginning--even before the crackdown had a name--
+secrecy was a big problem. There were many good reasons
+for secrecy in the hacker crackdown. Hackers and code-thieves
+were wily prey, slinking back to their bedrooms and basements
+and destroying vital incriminating evidence at the first hint of trouble.
+Furthermore, the crimes themselves were heavily technical and difficult
+to describe, even to police--much less to the general public.
+
+When such crimes HAD been described intelligibly to the public,
+in the past, that very publicity had tended to INCREASE the crimes
+enormously. Telco officials, while painfully aware of the vulnerabilities
+of their systems, were anxious not to publicize those weaknesses.
+Experience showed them that those weaknesses, once discovered,
+would be pitilessly exploited by tens of thousands of people--not only
+by professional grifters and by underground hackers and phone phreaks,
+but by many otherwise more-or-less honest everyday folks, who regarded
+stealing service from the faceless, soulless "Phone Company" as a kind of
+harmless indoor sport. When it came to protecting their interests,
+telcos had long since given up on general public sympathy for
+"the Voice with a Smile." Nowadays the telco's "Voice" was
+very likely to be a computer's; and the American public
+showed much less of the proper respect and gratitude due
+the fine public service bequeathed them by Dr. Bell and Mr. Vail.
+The more efficient, high-tech, computerized, and impersonal
+the telcos became, it seemed, the more they were met by
+sullen public resentment and amoral greed.
+
+Telco officials wanted to punish the phone-phreak underground, in as
+public and exemplary a manner as possible. They wanted to make dire
+examples of the worst offenders, to seize the ringleaders and intimidate
+the small fry, to discourage and frighten the wacky hobbyists, and send
+the professional grifters to jail. To do all this, publicity was vital.
+
+Yet operational secrecy was even more so. If word got out that
+a nationwide crackdown was coming, the hackers might simply vanish;
+destroy the evidence, hide their computers, go to earth,
+and wait for the campaign to blow over. Even the young
+hackers were crafty and suspicious, and as for the professional grifters,
+they tended to split for the nearest state-line at the first sign of trouble.
+For the crackdown to work well, they would all have to be caught red-handed,
+swept upon suddenly, out of the blue, from every corner of the compass.
+
+And there was another strong motive for secrecy. In the worst-case scenario,
+a blown campaign might leave the telcos open to a devastating hacker
+counter-attack. If there were indeed hackers loose in America who
+had caused the January 15 Crash--if there were truly gifted hackers,
+loose in the nation's long-distance switching systems, and enraged
+or frightened by the crackdown--then they might react unpredictably
+to an attempt to collar them. Even if caught, they might have talented
+and vengeful friends still running around loose. Conceivably,
+it could turn ugly. Very ugly. In fact, it was hard to imagine
+just how ugly things might turn, given that possibility.
+
+Counter-attack from hackers was a genuine concern for the telcos.
+In point of fact, they would never suffer any such counter-attack.
+But in months to come, they would be at some pains to publicize
+this notion and to utter grim warnings about it.
+
+Still, that risk seemed well worth running. Better to run the risk
+of vengeful attacks, than to live at the mercy of potential crashers.
+Any cop would tell you that a protection racket had no real future.
+
+And publicity was such a useful thing. Corporate security officers,
+including telco security, generally work under conditions of great discretion.
+And corporate security officials do not make money for their companies.
+Their job is to PREVENT THE LOSS of money, which is much less glamorous
+than actually winning profits.
+
+If you are a corporate security official, and you do your job brilliantly,
+then nothing bad happens to your company at all. Because of this, you appear
+completely superfluous. This is one of the many unattractive aspects
+of security work. It's rare that these folks have the chance to draw
+some healthy attention to their own efforts.
+
+Publicity also served the interest of their friends in law enforcement.
+Public officials, including law enforcement officials, thrive by attracting
+favorable public interest. A brilliant prosecution in a matter of vital
+public interest can make the career of a prosecuting attorney.
+And for a police officer, good publicity opens the purses of the legislature;
+it may bring a citation, or a promotion, or at least a rise in status
+and the respect of one's peers.
+
+But to have both publicity and secrecy is to have one's cake and eat it too.
+In months to come, as we will show, this impossible act was to cause great
+pain to the agents of the crackdown. But early on, it seemed possible
+--maybe even likely--that the crackdown could successfully combine
+the best of both worlds. The ARREST of hackers would be heavily publicized.
+The actual DEEDS of the hackers, which were technically hard to explain
+and also a security risk, would be left decently obscured. The THREAT
+hackers posed would be heavily trumpeted; the likelihood of their actually
+committing such fearsome crimes would be left to the public's imagination.
+The spread of the computer underground, and its growing technical
+sophistication, would be heavily promoted; the actual hackers themselves,
+mostly bespectacled middle-class white suburban teenagers,
+would be denied any personal publicity.
+
+It does not seem to have occurred to any telco official
+that the hackers accused would demand a day in court;
+that journalists would smile upon the hackers as
+"good copy;" that wealthy high-tech entrepreneurs would
+offer moral and financial support to crackdown victims;
+that constitutional lawyers would show up with briefcases,
+frowning mightily. This possibility does not seem to have
+ever entered the game-plan.
+
+And even if it had, it probably would not have slowed
+the ferocious pursuit of a stolen phone-company document,
+mellifluously known as "Control Office Administration of
+Enhanced 911 Services for Special Services and Major Account Centers."
+
+In the chapters to follow, we will explore the worlds
+of police and the computer underground, and the large
+shadowy area where they overlap. But first, we must
+explore the battleground. Before we leave the world
+of the telcos, we must understand what a switching system
+actually is and how your telephone actually works.
+
+#
+
+To the average citizen, the idea of the telephone is represented by,
+well, a TELEPHONE: a device that you talk into. To a telco
+professional, however, the telephone itself is known, in lordly
+fashion, as a "subset." The "subset" in your house is a mere adjunct,
+a distant nerve ending, of the central switching stations,
+which are ranked in levels of heirarchy, up to the long-distance electronic
+switching stations, which are some of the largest computers on earth.
+
+Let us imagine that it is, say, 1925, before the
+introduction of computers, when the phone system was
+simpler and somewhat easier to grasp. Let's further
+imagine that you are Miss Leticia Luthor, a fictional
+operator for Ma Bell in New York City of the 20s.
+
+Basically, you, Miss Luthor, ARE the "switching system."
+You are sitting in front of a large vertical switchboard,
+known as a "cordboard," made of shiny wooden panels,
+with ten thousand metal-rimmed holes punched in them,
+known as jacks. The engineers would have put more
+holes into your switchboard, but ten thousand is
+as many as you can reach without actually having
+to get up out of your chair.
+
+Each of these ten thousand holes has its own little electric lightbulb,
+known as a "lamp," and its own neatly printed number code.
+
+With the ease of long habit, you are scanning your board for lit-up bulbs.
+This is what you do most of the time, so you are used to it.
+
+A lamp lights up. This means that the phone
+at the end of that line has been taken off the hook.
+Whenever a handset is taken off the hook, that closes a circuit
+inside the phone which then signals the local office, i.e. you,
+automatically. There might be somebody calling, or then
+again the phone might be simply off the hook, but this
+does not matter to you yet. The first thing you do,
+is record that number in your logbook, in your fine American
+public-school handwriting. This comes first, naturally,
+since it is done for billing purposes.
+
+You now take the plug of your answering cord, which goes
+directly to your headset, and plug it into the lit-up hole.
+"Operator," you announce.
+
+In operator's classes, before taking this job, you have
+been issued a large pamphlet full of canned operator's
+responses for all kinds of contingencies, which you had
+to memorize. You have also been trained in a proper
+non-regional, non-ethnic pronunciation and tone of voice.
+You rarely have the occasion to make any spontaneous
+remark to a customer, and in fact this is frowned upon
+(except out on the rural lines where people have time
+on their hands and get up to all kinds of mischief).
+
+A tough-sounding user's voice at the end of the line
+gives you a number. Immediately, you write that number
+down in your logbook, next to the caller's number,
+which you just wrote earlier. You then look and see if
+the number this guy wants is in fact on your switchboard,
+which it generally is, since it's generally a local call.
+Long distance costs so much that people use it sparingly.
+
+Only then do you pick up a calling-cord from a shelf
+at the base of the switchboard. This is a long elastic cord
+mounted on a kind of reel so that it will zip back in when
+you unplug it. There are a lot of cords down there,
+and when a bunch of them are out at once they look like
+a nest of snakes. Some of the girls think there are bugs
+living in those cable-holes. They're called "cable mites"
+and are supposed to bite your hands and give you rashes.
+You don't believe this, yourself.
+
+Gripping the head of your calling-cord, you slip the tip
+of it deftly into the sleeve of the jack for the called person.
+Not all the way in, though. You just touch it. If you hear
+a clicking sound, that means the line is busy and you can't
+put the call through. If the line is busy, you have to stick
+the calling-cord into a "busy-tone jack," which will give
+the guy a busy-tone. This way you don't have to talk to him
+yourself and absorb his natural human frustration.
+
+But the line isn't busy. So you pop the cord all the way in.
+Relay circuits in your board make the distant phone ring,
+and if somebody picks it up off the hook, then a phone
+conversation starts. You can hear this conversation
+on your answering cord, until you unplug it. In fact
+you could listen to the whole conversation if you wanted,
+but this is sternly frowned upon by management, and frankly,
+when you've overheard one, you've pretty much heard 'em all.
+
+You can tell how long the conversation lasts by the glow
+of the calling-cord's lamp, down on the calling-cord's shelf.
+When it's over, you unplug and the calling-cord zips back into place.
+
+Having done this stuff a few hundred thousand times,
+you become quite good at it. In fact you're plugging,
+and connecting, and disconnecting, ten, twenty, forty cords
+at a time. It's a manual handicraft, really, quite satisfying
+in a way, rather like weaving on an upright loom.
+
+Should a long-distance call come up, it would be different,
+but not all that different. Instead of connecting the call
+through your own local switchboard, you have to go up the hierarchy,
+onto the long-distance lines, known as "trunklines."
+Depending on how far the call goes, it may have to work
+its way through a whole series of operators, which can
+take quite a while. The caller doesn't wait on the line
+while this complex process is negotiated across the country
+by the gaggle of operators. Instead, the caller hangs up,
+and you call him back yourself when the call has finally
+worked its way through.
+
+After four or five years of this work, you get married,
+and you have to quit your job, this being the natural order
+of womanhood in the American 1920s. The phone company
+has to train somebody else--maybe two people, since
+the phone system has grown somewhat in the meantime.
+And this costs money.
+
+In fact, to use any kind of human being as a switching
+system is a very expensive proposition. Eight thousand
+Leticia Luthors would be bad enough, but a quarter of a
+million of them is a military-scale proposition and makes
+drastic measures in automation financially worthwhile.
+
+Although the phone system continues to grow today,
+the number of human beings employed by telcos has
+been dropping steadily for years. Phone "operators"
+now deal with nothing but unusual contingencies,
+all routine operations having been shrugged off onto machines.
+Consequently, telephone operators are considerably less
+machine-like nowadays, and have been known to have accents
+and actual character in their voices. When you reach
+a human operator today, the operators are rather more
+"human" than they were in Leticia's day--but on the other hand,
+human beings in the phone system are much harder to reach
+in the first place.
+
+Over the first half of the twentieth century,
+"electromechanical" switching systems of growing
+complexity were cautiously introduced into the phone system.
+In certain backwaters, some of these hybrid systems are still
+in use. But after 1965, the phone system began to go completely
+electronic, and this is by far the dominant mode today.
+Electromechanical systems have "crossbars," and "brushes,"
+and other large moving mechanical parts, which, while faster
+and cheaper than Leticia, are still slow, and tend to wear out
+fairly quickly.
+
+But fully electronic systems are inscribed on silicon chips,
+and are lightning-fast, very cheap, and quite durable.
+They are much cheaper to maintain than even the best
+electromechanical systems, and they fit into half the space.
+And with every year, the silicon chip grows smaller, faster,
+and cheaper yet. Best of all, automated electronics work
+around the clock and don't have salaries or health insurance.
+
+There are, however, quite serious drawbacks to the
+use of computer-chips. When they do break down, it is
+a daunting challenge to figure out what the heck has gone
+wrong with them. A broken cordboard generally had
+a problem in it big enough to see. A broken chip has
+invisible, microscopic faults. And the faults in bad
+software can be so subtle as to be practically theological.
+
+If you want a mechanical system to do something new,
+then you must travel to where it is, and pull pieces out of it,
+and wire in new pieces. This costs money. However, if you want
+a chip to do something new, all you have to do is change its software,
+which is easy, fast and dirt-cheap. You don't even have to see the chip
+to change its program. Even if you did see the chip, it wouldn't look
+like much. A chip with program X doesn't look one whit different from
+a chip with program Y.
+
+With the proper codes and sequences, and access to specialized phone-lines,
+you can change electronic switching systems all over America from anywhere
+you please.
+
+And so can other people. If they know how, and if they want to,
+they can sneak into a microchip via the special phonelines and diddle with it,
+leaving no physical trace at all. If they broke into the operator's station
+and held Leticia at gunpoint, that would be very obvious. If they broke into
+a telco building and went after an electromechanical switch with a toolbelt,
+that would at least leave many traces. But people can do all manner of amazing
+things to computer switches just by typing on a keyboard, and keyboards are
+everywhere today. The extent of this vulnerability is deep, dark, broad,
+almost mind-boggling, and yet this is a basic, primal fact of life about
+any computer on a network.
+
+Security experts over the past twenty years have insisted,
+with growing urgency, that this basic vulnerability of computers
+represents an entirely new level of risk, of unknown but obviously
+dire potential to society. And they are right.
+
+An electronic switching station does pretty much
+everything Letitia did, except in nanoseconds and
+on a much larger scale. Compared to Miss Luthor's
+ten thousand jacks, even a primitive 1ESS switching computer,
+60s vintage, has a 128,000 lines. And the current AT&T
+system of choice is the monstrous fifth-generation 5ESS.
+
+An Electronic Switching Station can scan every line on its "board"
+in a tenth of a second, and it does this over and over, tirelessly,
+around the clock. Instead of eyes, it uses "ferrod scanners"
+to check the condition of local lines and trunks. Instead of hands,
+it has "signal distributors," "central pulse distributors,"
+"magnetic latching relays," and "reed switches," which complete
+and break the calls. Instead of a brain, it has a "central processor."
+Instead of an instruction manual, it has a program. Instead of
+a handwritten logbook for recording and billing calls,
+it has magnetic tapes. And it never has to talk to anybody.
+Everything a customer might say to it is done by punching
+the direct-dial tone buttons on your subset.
+
+Although an Electronic Switching Station can't talk,
+it does need an interface, some way to relate to its, er,
+employers. This interface is known as the "master control
+center." (This interface might be better known simply as
+"the interface," since it doesn't actually "control" phone
+calls directly. However, a term like "Master Control
+Center" is just the kind of rhetoric that telco maintenance
+engineers--and hackers--find particularly satisfying.)
+
+Using the master control center, a phone engineer can test
+local and trunk lines for malfunctions. He (rarely she)
+can check various alarm displays, measure traffic on the lines,
+examine the records of telephone usage and the charges for those calls,
+and change the programming.
+
+And, of course, anybody else who gets into the master control center
+by remote control can also do these things, if he (rarely she)
+has managed to figure them out, or, more likely, has somehow swiped
+the knowledge from people who already know.
+
+In 1989 and 1990, one particular RBOC, BellSouth,
+which felt particularly troubled, spent a purported $1.2
+million on computer security. Some think it spent as
+much as two million, if you count all the associated costs.
+Two million dollars is still very little compared to the
+great cost-saving utility of telephonic computer systems.
+
+Unfortunately, computers are also stupid.
+Unlike human beings, computers possess the truly
+profound stupidity of the inanimate.
+
+In the 1960s, in the first shocks of spreading computerization,
+there was much easy talk about the stupidity of computers--
+how they could "only follow the program" and were rigidly required
+to do "only what they were told." There has been rather less talk
+about the stupidity of computers since they began to achieve
+grandmaster status in chess tournaments, and to manifest
+many other impressive forms of apparent cleverness.
+
+Nevertheless, computers STILL are profoundly brittle and stupid;
+they are simply vastly more subtle in their stupidity and brittleness.
+The computers of the 1990s are much more reliable in their components
+than earlier computer systems, but they are also called upon to do
+far more complex things, under far more challenging conditions.
+
+On a basic mathematical level, every single line of
+a software program offers a chance for some possible screwup.
+Software does not sit still when it works; it "runs,"
+it interacts with itself and with its own inputs and outputs.
+By analogy, it stretches like putty into millions of possible
+shapes and conditions, so many shapes that they can never
+all be successfully tested, not even in the lifespan of the universe.
+Sometimes the putty snaps.
+
+The stuff we call "software" is not like anything that human society
+is used to thinking about. Software is something like a machine,
+and something like mathematics, and something like language, and
+something like thought, and art, and information. . . . But software
+is not in fact any of those other things. The protean quality
+of software is one of the great sources of its fascination.
+It also makes software very powerful, very subtle,
+very unpredictable, and very risky.
+
+Some software is bad and buggy. Some is "robust,"
+even "bulletproof." The best software is that which has
+been tested by thousands of users under thousands of
+different conditions, over years. It is then known as
+"stable." This does NOT mean that the software is
+now flawless, free of bugs. It generally means that there
+are plenty of bugs in it, but the bugs are well-identified
+and fairly well understood.
+
+There is simply no way to assure that software is free
+of flaws. Though software is mathematical in nature,
+it cannot by "proven" like a mathematical theorem;
+software is more like language, with inherent ambiguities,
+with different definitions, different assumptions,
+different levels of meaning that can conflict.
+
+Human beings can manage, more or less, with
+human language because we can catch the gist of it.
+
+Computers, despite years of effort in "artificial intelligence,"
+have proven spectacularly bad in "catching the gist" of anything at all.
+The tiniest bit of semantic grit may still bring the mightiest computer
+tumbling down. One of the most hazardous things you can do to a
+computer program is try to improve it--to try to make it safer.
+Software "patches" represent new, untried un-"stable" software,
+which is by definition riskier.
+
+The modern telephone system has come to depend,
+utterly and irretrievably, upon software. And the
+System Crash of January 15, 1990, was caused by an
+IMPROVEMENT in software. Or rather, an ATTEMPTED
+improvement.
+
+As it happened, the problem itself--the problem per se--took this form.
+A piece of telco software had been written in C language, a standard
+language of the telco field. Within the C software was a
+long "do. . .while" construct. The "do. . .while" construct
+contained a "switch" statement. The "switch" statement contained
+an "if" clause. The "if" clause contained a "break." The "break"
+was SUPPOSED to "break" the "if clause." Instead, the "break"
+broke the "switch" statement.
+
+That was the problem, the actual reason why people picking up phones
+on January 15, 1990, could not talk to one another.
+
+Or at least, that was the subtle, abstract, cyberspatial
+seed of the problem. This is how the problem manifested itself
+from the realm of programming into the realm of real life.
+
+The System 7 software for AT&T's 4ESS switching station,
+the "Generic 44E14 Central Office Switch Software,"
+had been extensively tested, and was considered very stable.
+By the end of 1989, eighty of AT&T's switching systems
+nationwide had been programmed with the new software. Cautiously,
+thirty-four stations were left to run the slower, less-capable
+System 6, because AT&T suspected there might be shakedown problems
+with the new and unprecedently sophisticated System 7 network.
+
+The stations with System 7 were programmed to switch over to a backup net
+in case of any problems. In mid-December 1989, however, a new high-velocity,
+high-security software patch was distributed to each of the 4ESS switches
+that would enable them to switch over even more quickly, making the System 7
+network that much more secure.
+
+Unfortunately, every one of these 4ESS switches was now in possession
+of a small but deadly flaw.
+
+In order to maintain the network, switches must monitor
+the condition of other switches--whether they are up and running,
+whether they have temporarily shut down, whether they are overloaded
+and in need of assistance, and so forth. The new software helped
+control this bookkeeping function by monitoring the status calls
+from other switches.
+
+It only takes four to six seconds for a troubled 4ESS switch
+to rid itself of all its calls, drop everything temporarily,
+and re-boot its software from scratch. Starting over from scratch
+will generally rid the switch of any software problems that may have
+developed in the course of running the system. Bugs that arise will
+be simply wiped out by this process. It is a clever idea. This process
+of automatically re-booting from scratch is known as the "normal fault
+recovery routine." Since AT&T's software is in fact exceptionally stable,
+systems rarely have to go into "fault recovery" in the first place;
+but AT&T has always boasted of its "real world" reliability, and this
+tactic is a belt-and-suspenders routine.
+
+The 4ESS switch used its new software to monitor its fellow switches
+as they recovered from faults. As other switches came back on line
+after recovery, they would send their "OK" signals to the switch.
+The switch would make a little note to that effect in its "status map,"
+recognizing that the fellow switch was back and ready to go,
+and should be sent some calls and put back to regular work.
+
+Unfortunately, while it was busy bookkeeping with the status map,
+the tiny flaw in the brand-new software came into play.
+The flaw caused the 4ESS switch to interact, subtly but drastically,
+with incoming telephone calls from human users. If--and only if--
+two incoming phone-calls happened to hit the switch within a hundredth
+of a second, then a small patch of data would be garbled by the flaw.
+
+But the switch had been programmed to monitor itself
+constantly for any possible damage to its data.
+When the switch perceived that its data had been somehow garbled,
+then it too would go down, for swift repairs to its software.
+It would signal its fellow switches not to send any more work.
+It would go into the fault-recovery mode for four to six seconds.
+And then the switch would be fine again, and would send out its "OK,
+ready for work" signal.
+
+However, the "OK, ready for work" signal was the VERY THING THAT
+HAD CAUSED THE SWITCH TO GO DOWN IN THE FIRST PLACE. And ALL the
+System 7 switches had the same flaw in their status-map software.
+As soon as they stopped to make the bookkeeping note that their fellow
+switch was "OK," then they too would become vulnerable to the slight
+chance that two phone-calls would hit them within a hundredth of a second.
+
+At approximately 2:25 P.M. EST on Monday, January 15,
+one of AT&T's 4ESS toll switching systems in New York City
+had an actual, legitimate, minor problem. It went into fault
+recovery routines, announced "I'm going down," then announced,
+"I'm back, I'm OK." And this cheery message then blasted
+throughout the network to many of its fellow 4ESS switches.
+
+Many of the switches, at first, completely escaped trouble.
+These lucky switches were not hit by the coincidence of
+two phone calls within a hundredth of a second.
+Their software did not fail--at first. But three switches--
+in Atlanta, St. Louis, and Detroit--were unlucky,
+and were caught with their hands full. And they went down.
+And they came back up, almost immediately. And they too began
+to broadcast the lethal message that they, too, were "OK" again,
+activating the lurking software bug in yet other switches.
+
+As more and more switches did have that bit of bad luck
+and collapsed, the call-traffic became more and more densely
+packed in the remaining switches, which were groaning
+to keep up with the load. And of course, as the calls
+became more densely packed, the switches were MUCH MORE LIKELY
+to be hit twice within a hundredth of a second.
+
+It only took four seconds for a switch to get well.
+There was no PHYSICAL damage of any kind to the switches,
+after all. Physically, they were working perfectly.
+This situation was "only" a software problem.
+
+But the 4ESS switches were leaping up and down every
+four to six seconds, in a virulent spreading wave all over America,
+in utter, manic, mechanical stupidity. They kept KNOCKING
+one another down with their contagious "OK" messages.
+
+It took about ten minutes for the chain reaction to cripple the network.
+Even then, switches would periodically luck-out and manage to resume
+their normal work. Many calls--millions of them--were managing
+to get through. But millions weren't.
+
+The switching stations that used System 6 were not directly affected.
+Thanks to these old-fashioned switches, AT&T's national system avoided
+complete collapse. This fact also made it clear to engineers that
+System 7 was at fault.
+
+Bell Labs engineers, working feverishly in New Jersey, Illinois,
+and Ohio, first tried their entire repertoire of standard network
+remedies on the malfunctioning System 7. None of the remedies worked,
+of course, because nothing like this had ever happened to any
+phone system before.
+
+By cutting out the backup safety network entirely,
+they were able to reduce the frenzy of "OK" messages
+by about half. The system then began to recover, as the
+chain reaction slowed. By 11:30 P.M. on Monday January
+15, sweating engineers on the midnight shift breathed a
+sigh of relief as the last switch cleared-up.
+
+By Tuesday they were pulling all the brand-new 4ESS software
+and replacing it with an earlier version of System 7.
+
+If these had been human operators, rather than
+computers at work, someone would simply have
+eventually stopped screaming. It would have been
+OBVIOUS that the situation was not "OK," and common
+sense would have kicked in. Humans possess common sense--
+at least to some extent. Computers simply don't.
+
+On the other hand, computers can handle hundreds
+of calls per second. Humans simply can't. If every single
+human being in America worked for the phone company,
+we couldn't match the performance of digital switches:
+direct-dialling, three-way calling, speed-calling, call-
+waiting, Caller ID, all the rest of the cornucopia
+of digital bounty. Replacing computers with operators
+is simply not an option any more.
+
+And yet we still, anachronistically, expect humans to
+be running our phone system. It is hard for us
+to understand that we have sacrificed huge amounts
+of initiative and control to senseless yet powerful machines.
+When the phones fail, we want somebody to be responsible.
+We want somebody to blame.
+
+When the Crash of January 15 happened, the American populace
+was simply not prepared to understand that enormous landslides
+in cyberspace, like the Crash itself, can happen,
+and can be nobody's fault in particular. It was easier to believe,
+maybe even in some odd way more reassuring to believe,
+that some evil person, or evil group, had done this to us.
+"Hackers" had done it. With a virus. A trojan horse.
+A software bomb. A dirty plot of some kind. People believed this,
+responsible people. In 1990, they were looking hard for evidence
+to confirm their heartfelt suspicions.
+
+And they would look in a lot of places.
+
+Come 1991, however, the outlines of an apparent new reality
+would begin to emerge from the fog.
+
+On July 1 and 2, 1991, computer-software collapses
+in telephone switching stations disrupted service in
+Washington DC, Pittsburgh, Los Angeles and San Francisco.
+Once again, seemingly minor maintenance problems had
+crippled the digital System 7. About twelve million
+people were affected in the Crash of July 1, 1991.
+
+Said the New York Times Service: "Telephone company executives
+and federal regulators said they were not ruling out the possibility
+of sabotage by computer hackers, but most seemed to think the problems
+stemmed from some unknown defect in the software running the networks."
+
+And sure enough, within the week, a red-faced software company,
+DSC Communications Corporation of Plano, Texas, owned up
+to "glitches" in the "signal transfer point" software that
+DSC had designed for Bell Atlantic and Pacific Bell.
+The immediate cause of the July 1 Crash was a single
+mistyped character: one tiny typographical flaw
+in one single line of the software. One mistyped letter,
+in one single line, had deprived the nation's capital of phone service.
+It was not particularly surprising that this tiny flaw had escaped attention:
+a typical System 7 station requires TEN MILLION lines of code.
+
+On Tuesday, September 17, 1991, came the most spectacular outage yet.
+This case had nothing to do with software failures--at least, not directly.
+Instead, a group of AT&T's switching stations in New York City had simply
+run out of electrical power and shut down cold. Their back-up batteries
+had failed. Automatic warning systems were supposed to warn of the loss
+of battery power, but those automatic systems had failed as well.
+
+This time, Kennedy, La Guardia, and Newark airports
+all had their voice and data communications cut.
+This horrifying event was particularly ironic, as attacks
+on airport computers by hackers had long been a standard
+nightmare scenario, much trumpeted by computer-security
+experts who feared the computer underground. There had even
+been a Hollywood thriller about sinister hackers ruining
+airport computers--DIE HARD II.
+
+Now AT&T itself had crippled airports with computer malfunctions--
+not just one airport, but three at once, some of the busiest in the world.
+
+Air traffic came to a standstill throughout the Greater New York area,
+causing more than 500 flights to be cancelled, in a spreading wave
+all over America and even into Europe. Another 500 or so flights
+were delayed, affecting, all in all, about 85,000 passengers.
+(One of these passengers was the chairman of the Federal
+Communications Commission.)
+
+Stranded passengers in New York and New Jersey were further
+infuriated to discover that they could not even manage to
+make a long distance phone call, to explain their delay
+to loved ones or business associates. Thanks to the crash,
+about four and a half million domestic calls, and half a million
+international calls, failed to get through.
+
+The September 17 NYC Crash, unlike the previous ones,
+involved not a whisper of "hacker" misdeeds. On the contrary,
+by 1991, AT&T itself was suffering much of the vilification
+that had formerly been directed at hackers. Congressmen were grumbling.
+So were state and federal regulators. And so was the press.
+
+For their part, ancient rival MCI took out snide full-page
+newspaper ads in New York, offering their own long-distance
+services for the "next time that AT&T goes down."
+
+"You wouldn't find a classy company like AT&T using such advertising,"
+protested AT&T Chairman Robert Allen, unconvincingly. Once again,
+out came the full-page AT&T apologies in newspapers, apologies for
+"an inexcusable culmination of both human and mechanical failure."
+(This time, however, AT&T offered no discount on later calls.
+Unkind critics suggested that AT&T were worried about setting any precedent
+for refunding the financial losses caused by telephone crashes.)
+
+Industry journals asked publicly if AT&T was "asleep at the switch."
+The telephone network, America's purported marvel of high-tech reliability,
+had gone down three times in 18 months. Fortune magazine listed the
+Crash of September 17 among the "Biggest Business Goofs of 1991,"
+cruelly parodying AT&T's ad campaign in an article entitled
+"AT&T Wants You Back (Safely On the Ground, God Willing)."
+
+Why had those New York switching systems simply run out of power?
+Because no human being had attended to the alarm system.
+Why did the alarm systems blare automatically,
+without any human being noticing? Because the three
+telco technicians who SHOULD have been listening
+were absent from their stations in the power-room,
+on another floor of the building--attending a training class.
+A training class about the alarm systems for the power room!
+
+"Crashing the System" was no longer "unprecedented" by late 1991.
+On the contrary, it no longer even seemed an oddity. By 1991,
+it was clear that all the policemen in the world could no longer
+"protect" the phone system from crashes. By far the worst crashes
+the system had ever had, had been inflicted, by the system,
+upon ITSELF. And this time nobody was making cocksure statements
+that this was an anomaly, something that would never happen again.
+By 1991 the System's defenders had met their nebulous Enemy,
+and the Enemy was--the System.
+
+
+
+PART TWO: THE DIGITAL UNDERGROUND
+
+
+The date was May 9, 1990. The Pope was touring Mexico City.
+Hustlers from the Medellin Cartel were trying to buy
+black-market Stinger missiles in Florida. On the comics page,
+Doonesbury character Andy was dying of AIDS. And then. . .a highly
+unusual item whose novelty and calculated rhetoric won it
+headscratching attention in newspapers all over America.
+
+The US Attorney's office in Phoenix, Arizona, had issued
+a press release announcing a nationwide law enforcement crackdown
+against "illegal computer hacking activities." The sweep was
+officially known as "Operation Sundevil."
+
+Eight paragraphs in the press release gave the bare facts:
+twenty-seven search warrants carried out on May 8, with three arrests,
+and a hundred and fifty agents on the prowl in "twelve" cities across America.
+(Different counts in local press reports yielded "thirteen," "fourteen," and
+"sixteen" cities.) Officials estimated that criminal losses of revenue
+to telephone companies "may run into millions of dollars." Credit for
+the Sundevil investigations was taken by the US Secret Service,
+Assistant US Attorney Tim Holtzen of Phoenix, and the Assistant
+Attorney General of Arizona, Gail Thackeray.
+
+The prepared remarks of Garry M. Jenkins, appearing in a U.S. Department
+of Justice press release, were of particular interest. Mr. Jenkins was the
+Assistant Director of the US Secret Service, and the highest-ranking federal
+official to take any direct public role in the hacker crackdown of 1990.
+
+"Today, the Secret Service is sending a clear message to those computer hackers
+who have decided to violate the laws of this nation in the mistaken belief
+that they can successfully avoid detection by hiding behind the relative
+anonymity of their computer terminals. (. . .) "Underground groups have been
+formed for the purpose of exchanging information relevant to their criminal
+activities. These groups often communicate with each other through message
+systems between computers called `bulletin boards.' "Our experience shows
+that many computer hacker suspects are no longer misguided teenagers,
+mischievously playing games with their computers in their bedrooms.
+Some are now high tech computer operators using computers to engage
+in unlawful conduct."
+
+Who were these "underground groups" and "high-tech operators?"
+Where had they come from? What did they want? Who WERE they?
+Were they "mischievous?" Were they dangerous? How had "misguided teenagers"
+managed to alarm the United States Secret Service? And just how widespread
+was this sort of thing?
+
+Of all the major players in the Hacker Crackdown: the phone companies,
+law enforcement, the civil libertarians, and the "hackers" themselves--
+the "hackers" are by far the most mysterious, by far the hardest to
+understand, by far the WEIRDEST.
+
+Not only are "hackers" novel in their activities, but they come
+in a variety of odd subcultures, with a variety of languages,
+motives and values.
+
+The earliest proto-hackers were probably those unsung mischievous
+telegraph boys who were summarily fired by the Bell Company in 1878.
+
+Legitimate "hackers," those computer enthusiasts who are independent-minded
+but law-abiding, generally trace their spiritual ancestry to elite technical
+universities, especially M.I.T. and Stanford, in the 1960s.
+
+But the genuine roots of the modern hacker UNDERGROUND can probably be traced
+most successfully to a now much-obscured hippie anarchist movement known as
+the Yippies. The Yippies, who took their name from the largely fictional
+"Youth International Party," carried out a loud and lively policy of surrealistic
+subversion and outrageous political mischief. Their basic tenets were flagrant
+sexual promiscuity, open and copious drug use, the political overthrow of any
+powermonger over thirty years of age, and an immediate end to the war
+in Vietnam, by any means necessary, including the psychic levitation
+of the Pentagon.
+
+The two most visible Yippies were Abbie Hoffman and Jerry Rubin.
+Rubin eventually became a Wall Street broker. Hoffman, ardently sought
+by federal authorities, went into hiding for seven years,
+in Mexico, France, and the United States. While on the lam,
+Hoffman continued to write and publish, with help from sympathizers
+in the American anarcho-leftist underground. Mostly, Hoffman survived
+through false ID and odd jobs. Eventually he underwent facial plastic
+surgery and adopted an entirely new identity as one "Barry Freed."
+After surrendering himself to authorities in 1980, Hoffman spent a year
+in prison on a cocaine conviction.
+
+Hoffman's worldview grew much darker as the glory days of the 1960s faded.
+In 1989, he purportedly committed suicide, under odd and, to some, rather
+suspicious circumstances.
+
+Abbie Hoffman is said to have caused the Federal Bureau of Investigation
+to amass the single largest investigation file ever opened on an individual
+American citizen. (If this is true, it is still questionable whether the
+FBI regarded Abbie Hoffman a serious public threat--quite possibly,
+his file was enormous simply because Hoffman left colorful legendry
+wherever he went). He was a gifted publicist, who regarded electronic
+media as both playground and weapon. He actively enjoyed manipulating
+network TV and other gullible, image-hungry media, with various weird lies,
+mindboggling rumors, impersonation scams, and other sinister distortions,
+all absolutely guaranteed to upset cops, Presidential candidates,
+and federal judges. Hoffman's most famous work was a book self-reflexively
+known as STEAL THIS BOOK, which publicized a number of methods by which young,
+penniless hippie agitators might live off the fat of a system supported by
+humorless drones. STEAL THIS BOOK, whose title urged readers to damage
+the very means of distribution which had put it into their hands,
+might be described as a spiritual ancestor of a computer virus.
+
+Hoffman, like many a later conspirator, made extensive use of
+pay-phones for his agitation work--in his case, generally through
+the use of cheap brass washers as coin-slugs.
+
+During the Vietnam War, there was a federal surtax imposed on telephone
+service; Hoffman and his cohorts could, and did, argue that in systematically
+stealing phone service they were engaging in civil disobedience:
+virtuously denying tax funds to an illegal and immoral war.
+
+But this thin veil of decency was soon dropped entirely.
+Ripping-off the System found its own justification in deep alienation
+and a basic outlaw contempt for conventional bourgeois values.
+Ingenious, vaguely politicized varieties of rip-off,
+which might be described as "anarchy by convenience,"
+became very popular in Yippie circles, and because rip-off
+was so useful, it was to survive the Yippie movement itself.
+
+In the early 1970s, it required fairly limited expertise
+and ingenuity to cheat payphones, to divert "free"
+electricity and gas service, or to rob vending machines
+and parking meters for handy pocket change. It also required
+a conspiracy to spread this knowledge, and the gall
+and nerve actually to commit petty theft, but the Yippies
+had these qualifications in plenty. In June 1971, Abbie
+Hoffman and a telephone enthusiast sarcastically known
+as "Al Bell" began publishing a newsletter called Youth
+International Party Line. This newsletter was dedicated
+to collating and spreading Yippie rip-off techniques,
+especially of phones, to the joy of the freewheeling
+underground and the insensate rage of all straight people.
+As a political tactic, phone-service theft ensured
+that Yippie advocates would always have ready access
+to the long-distance telephone as a medium, despite
+the Yippies' chronic lack of organization, discipline,
+money, or even a steady home address.
+
+PARTY LINE was run out of Greenwich Village for a couple of years,
+then "Al Bell" more or less defected from the faltering ranks of Yippiedom,
+changing the newsletter's name to TAP or Technical Assistance Program.
+After the Vietnam War ended, the steam began leaking rapidly out of American
+radical dissent. But by this time, "Bell" and his dozen or so
+core contributors had the bit between their teeth,
+and had begun to derive tremendous gut-level satisfaction
+from the sensation of pure TECHNICAL POWER.
+
+TAP articles, once highly politicized, became pitilessly jargonized
+and technical, in homage or parody to the Bell System's own technical
+documents, which TAP studied closely, gutted, and reproduced without
+permission. The TAP elite revelled in gloating possession
+of the specialized knowledge necessary to beat the system.
+
+"Al Bell" dropped out of the game by the late 70s,
+and "Tom Edison" took over; TAP readers (some 1400 of
+them, all told) now began to show more interest in telex
+switches and the growing phenomenon of computer systems.
+
+In 1983, "Tom Edison" had his computer stolen and his house
+set on fire by an arsonist. This was an eventually mortal blow
+to TAP (though the legendary name was to be resurrected
+in 1990 by a young Kentuckian computer-outlaw named "Predat0r.")
+
+#
+
+Ever since telephones began to make money, there have been
+people willing to rob and defraud phone companies.
+The legions of petty phone thieves vastly outnumber those
+"phone phreaks" who "explore the system" for the sake
+of the intellectual challenge. The New York metropolitan area
+(long in the vanguard of American crime) claims over 150,000
+physical attacks on pay telephones every year! Studied carefully,
+a modern payphone reveals itself as a little fortress, carefully
+designed and redesigned over generations, to resist coin-slugs,
+zaps of electricity, chunks of coin-shaped ice, prybars, magnets,
+lockpicks, blasting caps. Public pay- phones must survive in a world
+of unfriendly, greedy people, and a modern payphone is as exquisitely
+evolved as a cactus.
+Because the phone network pre-dates the computer network,
+the scofflaws known as "phone phreaks" pre-date the scofflaws
+known as "computer hackers." In practice, today, the line
+between "phreaking" and "hacking" is very blurred,
+just as the distinction between telephones and computers
+has blurred. The phone system has been digitized,
+and computers have learned to "talk" over phone-lines.
+What's worse--and this was the point of the Mr. Jenkins
+of the Secret Service--some hackers have learned to steal,
+and some thieves have learned to hack.
+
+Despite the blurring, one can still draw a few useful
+behavioral distinctions between "phreaks" and "hackers."
+Hackers are intensely interested in the "system" per se,
+and enjoy relating to machines. "Phreaks" are more
+social, manipulating the system in a rough-and-ready
+fashion in order to get through to other human beings,
+fast, cheap and under the table.
+
+Phone phreaks love nothing so much as "bridges,"
+illegal conference calls of ten or twelve chatting
+conspirators, seaboard to seaboard, lasting for many hours
+--and running, of course, on somebody else's tab,
+preferably a large corporation's.
+
+As phone-phreak conferences wear on, people drop out
+(or simply leave the phone off the hook, while they
+sashay off to work or school or babysitting),
+and new people are phoned up and invited to join in,
+from some other continent, if possible. Technical trivia,
+boasts, brags, lies, head-trip deceptions, weird rumors,
+and cruel gossip are all freely exchanged.
+
+The lowest rung of phone-phreaking is the theft of telephone access codes.
+Charging a phone call to somebody else's stolen number is, of course,
+a pig-easy way of stealing phone service, requiring practically no
+technical expertise. This practice has been very widespread,
+especially among lonely people without much money who are far from home.
+Code theft has flourished especially in college dorms, military bases,
+and, notoriously, among roadies for rock bands. Of late, code theft
+has spread very rapidly among Third Worlders in the US, who pile up
+enormous unpaid long-distance bills to the Caribbean, South America,
+and Pakistan.
+
+The simplest way to steal phone-codes is simply to look over
+a victim's shoulder as he punches-in his own code-number
+on a public payphone. This technique is known as "shoulder-surfing,"
+and is especially common in airports, bus terminals, and train stations.
+The code is then sold by the thief for a few dollars. The buyer abusing
+the code has no computer expertise, but calls his Mom in New York,
+Kingston or Caracas and runs up a huge bill with impunity. The losses
+from this primitive phreaking activity are far, far greater than the
+monetary losses caused by computer-intruding hackers.
+
+In the mid-to-late 1980s, until the introduction of sterner telco
+security measures, COMPUTERIZED code theft worked like a charm,
+and was virtually omnipresent throughout the digital underground,
+among phreaks and hackers alike. This was accomplished through
+programming one's computer to try random code numbers over the telephone
+until one of them worked. Simple programs to do this were widely available
+in the underground; a computer running all night was likely to come up with
+a dozen or so useful hits. This could be repeated week after week until
+one had a large library of stolen codes.
+
+Nowadays, the computerized dialling of hundreds of numbers
+can be detected within hours and swiftly traced.
+If a stolen code is repeatedly abused, this too can
+be detected within a few hours. But for years in the 1980s,
+the publication of stolen codes was a kind of elementary etiquette
+for fledgling hackers. The simplest way to establish your bona-fides
+as a raider was to steal a code through repeated random dialling
+and offer it to the "community" for use. Codes could be both stolen,
+and used, simply and easily from the safety of one's own bedroom,
+with very little fear of detection or punishment.
+
+Before computers and their phone-line modems entered American homes
+in gigantic numbers, phone phreaks had their own special telecommunications
+hardware gadget, the famous "blue box." This fraud device (now rendered
+increasingly useless by the digital evolution of the phone system) could
+trick switching systems into granting free access to long-distance lines.
+It did this by mimicking the system's own signal, a tone of 2600 hertz.
+
+Steven Jobs and Steve Wozniak, the founders of Apple Computer, Inc.,
+once dabbled in selling blue-boxes in college dorms in California.
+For many, in the early days of phreaking, blue-boxing was scarcely
+perceived as "theft," but rather as a fun (if sneaky) way to use
+excess phone capacity harmlessly. After all, the long-distance
+lines were JUST SITTING THERE. . . . Whom did it hurt, really?
+If you're not DAMAGING the system, and you're not USING UP ANY
+TANGIBLE RESOURCE, and if nobody FINDS OUT what you did,
+then what real harm have you done? What exactly HAVE you "stolen,"
+anyway? If a tree falls in the forest and nobody hears it,
+how much is the noise worth? Even now this remains a rather
+dicey question.
+
+Blue-boxing was no joke to the phone companies, however.
+Indeed, when Ramparts magazine, a radical publication in California,
+printed the wiring schematics necessary to create a mute box in June 1972,
+the magazine was seized by police and Pacific Bell phone-company officials.
+The mute box, a blue-box variant, allowed its user to receive long-distance
+calls free of charge to the caller. This device was closely described in a
+Ramparts article wryly titled "Regulating the Phone Company In Your Home."
+Publication of this article was held to be in violation of Californian
+State Penal Code section 502.7, which outlaws ownership of wire-fraud
+devices and the selling of "plans or instructions for any instrument,
+apparatus, or device intended to avoid telephone toll charges."
+
+Issues of Ramparts were recalled or seized on the newsstands,
+and the resultant loss of income helped put the magazine out of business.
+This was an ominous precedent for free-expression issues, but the telco's
+crushing of a radical-fringe magazine passed without serious challenge
+at the time. Even in the freewheeling California 1970s, it was widely felt
+that there was something sacrosanct about what the phone company knew;
+that the telco had a legal and moral right to protect itself by shutting
+off the flow of such illicit information. Most telco information was so
+"specialized" that it would scarcely be understood by any honest member
+of the public. If not published, it would not be missed. To print such
+material did not seem part of the legitimate role of a free press.
+
+In 1990 there would be a similar telco-inspired attack
+on the electronic phreak/hacking "magazine" Phrack.
+The Phrack legal case became a central issue in the
+Hacker Crackdown, and gave rise to great controversy.
+Phrack would also be shut down, for a time, at least,
+but this time both the telcos and their law-enforcement
+allies would pay a much larger price for their actions.
+The Phrack case will be examined in detail, later.
+
+Phone-phreaking as a social practice is still very
+much alive at this moment. Today, phone-phreaking
+is thriving much more vigorously than the better-known
+and worse-feared practice of "computer hacking."
+New forms of phreaking are spreading rapidly, following
+new vulnerabilities in sophisticated phone services.
+
+Cellular phones are especially vulnerable; their chips
+can be re-programmed to present a false caller ID
+and avoid billing. Doing so also avoids police tapping,
+making cellular-phone abuse a favorite among drug-dealers.
+"Call-sell operations" using pirate cellular phones can,
+and have, been run right out of the backs of cars, which move
+from "cell" to "cell" in the local phone system, retailing
+stolen long-distance service, like some kind of demented
+electronic version of the neighborhood ice-cream truck.
+
+Private branch-exchange phone systems in large corporations
+can be penetrated; phreaks dial-up a local company, enter its
+internal phone-system, hack it, then use the company's own
+PBX system to dial back out over the public network,
+causing the company to be stuck with the resulting
+long-distance bill. This technique is known as "diverting."
+"Diverting" can be very costly, especially because phreaks
+tend to travel in packs and never stop talking.
+Perhaps the worst by-product of this "PBX fraud"
+is that victim companies and telcos have sued one another
+over the financial responsibility for the stolen calls,
+thus enriching not only shabby phreaks but well-paid lawyers.
+
+"Voice-mail systems" can also be abused; phreaks
+can seize their own sections of these sophisticated
+electronic answering machines, and use them for trading
+codes or knowledge of illegal techniques. Voice-mail
+abuse does not hurt the company directly, but finding
+supposedly empty slots in your company's answering
+machine all crammed with phreaks eagerly chattering
+and hey-duding one another in impenetrable jargon can
+cause sensations of almost mystical repulsion and dread.
+
+Worse yet, phreaks have sometimes been known to react
+truculently to attempts to "clean up" the voice-mail system.
+Rather than humbly acquiescing to being thrown out of their playground,
+they may very well call up the company officials at work (or at home)
+and loudly demand free voice-mail addresses of their very own.
+Such bullying is taken very seriously by spooked victims.
+
+Acts of phreak revenge against straight people are rare,
+but voice-mail systems are especially tempting and vulnerable,
+and an infestation of angry phreaks in one's voice-mail system is no joke.
+They can erase legitimate messages; or spy on private messages;
+or harass users with recorded taunts and obscenities.
+They've even been known to seize control of voice-mail security,
+and lock out legitimate users, or even shut down the system entirely.
+
+Cellular phone-calls, cordless phones, and ship-to-shore
+telephony can all be monitored by various forms of radio;
+this kind of "passive monitoring" is spreading explosively today.
+Technically eavesdropping on other people's cordless and cellular
+phone-calls is the fastest-growing area in phreaking today.
+This practice strongly appeals to the lust for power and conveys
+gratifying sensations of technical superiority over the eavesdropping
+victim. Monitoring is rife with all manner of tempting evil mischief.
+Simple prurient snooping is by far the most common activity.
+But credit-card numbers unwarily spoken over the phone can be recorded,
+stolen and used. And tapping people's phone-calls (whether through
+active telephone taps or passive radio monitors) does lend itself
+conveniently to activities like blackmail, industrial espionage,
+and political dirty tricks.
+
+It should be repeated that telecommunications fraud,
+the theft of phone service, causes vastly greater monetary
+losses than the practice of entering into computers by stealth.
+Hackers are mostly young suburban American white males,
+and exist in their hundreds--but "phreaks" come from both sexes
+and from many nationalities, ages and ethnic backgrounds,
+and are flourishing in the thousands.
+
+#
+
+The term "hacker" has had an unfortunate history.
+This book, The Hacker Crackdown, has little to say about
+"hacking" in its finer, original sense. The term can signify
+the free-wheeling intellectual exploration of the highest
+and deepest potential of computer systems. Hacking can
+describe the determination to make access to computers
+and information as free and open as possible. Hacking
+can involve the heartfelt conviction that beauty can
+be found in computers, that the fine aesthetic in a perfect
+program can liberate the mind and spirit. This is "hacking"
+as it was defined in Steven Levy's much-praised history
+of the pioneer computer milieu, Hackers, published in 1984.
+
+Hackers of all kinds are absolutely soaked through with heroic
+anti-bureaucratic sentiment. Hackers long for recognition
+as a praiseworthy cultural archetype, the postmodern electronic
+equivalent of the cowboy and mountain man. Whether they deserve
+such a reputation is something for history to decide. But many hackers--
+including those outlaw hackers who are computer intruders, and whose
+activities are defined as criminal--actually attempt to LIVE UP TO
+this techno-cowboy reputation. And given that electronics and
+telecommunications are still largely unexplored territories,
+there is simply NO TELLING what hackers might uncover.
+
+For some people, this freedom is the very breath of oxygen,
+the inventive spontaneity that makes life worth living
+and that flings open doors to marvellous possibility and
+individual empowerment. But for many people
+--and increasingly so--the hacker is an ominous figure,
+a smart-aleck sociopath ready to burst out of his basement
+wilderness and savage other people's lives for his own
+anarchical convenience.
+
+Any form of power without responsibility, without direct
+and formal checks and balances, is frightening to people--
+and reasonably so. It should be frankly admitted that
+hackers ARE frightening, and that the basis of this fear
+is not irrational.
+
+Fear of hackers goes well beyond the fear of merely criminal activity.
+
+Subversion and manipulation of the phone system
+is an act with disturbing political overtones.
+In America, computers and telephones are potent symbols
+of organized authority and the technocratic business elite.
+
+But there is an element in American culture that
+has always strongly rebelled against these symbols;
+rebelled against all large industrial computers
+and all phone companies. A certain anarchical tinge deep
+in the American soul delights in causing confusion and pain
+to all bureaucracies, including technological ones.
+
+There is sometimes malice and vandalism in this attitude,
+but it is a deep and cherished part of the American national character.
+The outlaw, the rebel, the rugged individual, the pioneer,
+the sturdy Jeffersonian yeoman, the private citizen resisting
+interference in his pursuit of happiness--these are figures that all
+Americans recognize, and that many will strongly applaud and defend.
+
+Many scrupulously law-abiding citizens today do cutting-edge work
+with electronics--work that has already had tremendous social influence
+and will have much more in years to come. In all truth, these talented,
+hardworking, law-abiding, mature, adult people are far more disturbing
+to the peace and order of the current status quo than any scofflaw group
+of romantic teenage punk kids. These law-abiding hackers have the power,
+ability, and willingness to influence other people's lives quite unpredictably.
+They have means, motive, and opportunity to meddle drastically with the
+American social order. When corralled into governments, universities,
+or large multinational companies, and forced to follow rulebooks
+and wear suits and ties, they at least have some conventional halters
+on their freedom of action. But when loosed alone, or in small groups,
+and fired by imagination and the entrepreneurial spirit, they can move
+mountains--causing landslides that will likely crash directly into your
+office and living room.
+
+These people, as a class, instinctively recognize that a public,
+politicized attack on hackers will eventually spread to them--
+that the term "hacker," once demonized, might be used to knock
+their hands off the levers of power and choke them out of existence.
+There are hackers today who fiercely and publicly resist any besmirching
+of the noble title of hacker. Naturally and understandably, they deeply
+resent the attack on their values implicit in using the word "hacker"
+as a synonym for computer-criminal.
+
+This book, sadly but in my opinion unavoidably, rather adds
+to the degradation of the term. It concerns itself mostly with "hacking"
+in its commonest latter-day definition, i.e., intruding into computer
+systems by stealth and without permission. The term "hacking" is used
+routinely today by almost all law enforcement officials with any
+professional interest in computer fraud and abuse. American police
+describe almost any crime committed with, by, through, or against
+a computer as hacking.
+
+Most importantly, "hacker" is what computer-intruders
+choose to call THEMSELVES. Nobody who "hacks" into systems
+willingly describes himself (rarely, herself) as a "computer intruder,"
+"computer trespasser," "cracker," "wormer," "darkside hacker"
+or "high tech street gangster." Several other demeaning terms
+have been invented in the hope that the press and public
+will leave the original sense of the word alone. But few people
+actually use these terms. (I exempt the term "cyberpunk,"
+which a few hackers and law enforcement people actually do use.
+The term "cyberpunk" is drawn from literary criticism and has
+some odd and unlikely resonances, but, like hacker,
+cyberpunk too has become a criminal pejorative today.)
+
+In any case, breaking into computer systems was hardly alien
+to the original hacker tradition. The first tottering systems
+of the 1960s required fairly extensive internal surgery merely
+to function day-by-day. Their users "invaded" the deepest,
+most arcane recesses of their operating software almost
+as a matter of routine. "Computer security" in these early,
+primitive systems was at best an afterthought. What security
+there was, was entirely physical, for it was assumed that
+anyone allowed near this expensive, arcane hardware would be
+a fully qualified professional expert.
+
+In a campus environment, though, this meant that grad students,
+teaching assistants, undergraduates, and eventually,
+all manner of dropouts and hangers-on ended up accessing
+and often running the works.
+
+Universities, even modern universities, are not in
+the business of maintaining security over information.
+On the contrary, universities, as institutions, pre-date
+the "information economy" by many centuries and are not-
+for-profit cultural entities, whose reason for existence
+(purportedly) is to discover truth, codify it through
+techniques of scholarship, and then teach it. Universities
+are meant to PASS THE TORCH OF CIVILIZATION, not just
+download data into student skulls, and the values of the
+academic community are strongly at odds with those of all
+would-be information empires. Teachers at all levels, from
+kindergarten up, have proven to be shameless and persistent
+software and data pirates. Universities do not merely
+"leak information" but vigorously broadcast free thought.
+
+This clash of values has been fraught with controversy.
+Many hackers of the 1960s remember their professional
+apprenticeship as a long guerilla war against the uptight
+mainframe-computer "information priesthood." These computer-hungry
+youngsters had to struggle hard for access to computing power,
+and many of them were not above certain, er, shortcuts.
+But, over the years, this practice freed computing
+from the sterile reserve of lab-coated technocrats and
+was largely responsible for the explosive growth of computing
+in general society--especially PERSONAL computing.
+
+Access to technical power acted like catnip on certain
+of these youngsters. Most of the basic techniques of
+computer intrusion: password cracking, trapdoors, backdoors,
+trojan horses--were invented in college environments in the 1960s,
+in the early days of network computing. Some off-the-cuff
+experience at computer intrusion was to be in the informal
+resume of most "hackers" and many future industry giants.
+Outside of the tiny cult of computer enthusiasts, few people
+thought much about the implications of "breaking into"
+computers. This sort of activity had not yet been publicized,
+much less criminalized.
+
+In the 1960s, definitions of "property" and "privacy"
+had not yet been extended to cyberspace. Computers
+were not yet indispensable to society. There were no vast
+databanks of vulnerable, proprietary information stored
+in computers, which might be accessed, copied without
+permission, erased, altered, or sabotaged. The stakes
+were low in the early days--but they grew every year,
+exponentially, as computers themselves grew.
+
+By the 1990s, commercial and political pressures
+had become overwhelming, and they broke the social
+boundaries of the hacking subculture. Hacking
+had become too important to be left to the hackers.
+Society was now forced to tackle the intangible nature
+of cyberspace-as-property, cyberspace as privately-owned
+unreal-estate. In the new, severe, responsible, high-stakes
+context of the "Information Society" of the 1990s,
+"hacking" was called into question.
+
+What did it mean to break into a computer without
+permission and use its computational power, or look
+around inside its files without hurting anything?
+What were computer-intruding hackers, anyway--how should
+society, and the law, best define their actions?
+Were they just BROWSERS, harmless intellectual explorers?
+Were they VOYEURS, snoops, invaders of privacy? Should
+they be sternly treated as potential AGENTS OF ESPIONAGE,
+or perhaps as INDUSTRIAL SPIES? Or were they best
+defined as TRESPASSERS, a very common teenage
+misdemeanor? Was hacking THEFT OF SERVICE?
+(After all, intruders were getting someone else's
+computer to carry out their orders, without permission
+and without paying). Was hacking FRAUD? Maybe it was
+best described as IMPERSONATION. The commonest mode
+of computer intrusion was (and is) to swipe or snoop
+somebody else's password, and then enter the computer
+in the guise of another person--who is commonly stuck
+with the blame and the bills.
+
+Perhaps a medical metaphor was better--hackers should
+be defined as "sick," as COMPUTER ADDICTS unable
+to control their irresponsible, compulsive behavior.
+
+But these weighty assessments meant little to the
+people who were actually being judged. From inside
+the underground world of hacking itself, all these
+perceptions seem quaint, wrongheaded, stupid, or meaningless.
+The most important self-perception of underground hackers--
+from the 1960s, right through to the present day--is that
+they are an ELITE. The day-to-day struggle in the underground
+is not over sociological definitions--who cares?--but for power,
+knowledge, and status among one's peers.
+
+When you are a hacker, it is your own inner conviction
+of your elite status that enables you to break, or let
+us say "transcend," the rules. It is not that ALL rules
+go by the board. The rules habitually broken by hackers
+are UNIMPORTANT rules--the rules of dopey greedhead telco
+bureaucrats and pig-ignorant government pests.
+
+Hackers have their OWN rules, which separate behavior
+which is cool and elite, from behavior which is rodentlike,
+stupid and losing. These "rules," however, are mostly unwritten
+and enforced by peer pressure and tribal feeling. Like all rules
+that depend on the unspoken conviction that everybody else
+is a good old boy, these rules are ripe for abuse. The mechanisms
+of hacker peer- pressure, "teletrials" and ostracism, are rarely used
+and rarely work. Back-stabbing slander, threats, and electronic
+harassment are also freely employed in down-and-dirty intrahacker feuds,
+but this rarely forces a rival out of the scene entirely. The only real
+solution for the problem of an utterly losing, treacherous and rodentlike
+hacker is to TURN HIM IN TO THE POLICE. Unlike the Mafia or Medellin Cartel,
+the hacker elite cannot simply execute the bigmouths, creeps and troublemakers
+among their ranks, so they turn one another in with astonishing frequency.
+
+There is no tradition of silence or OMERTA in the hacker underworld.
+Hackers can be shy, even reclusive, but when they do talk, hackers
+tend to brag, boast and strut. Almost everything hackers do is INVISIBLE;
+if they don't brag, boast, and strut about it, then NOBODY WILL EVER KNOW.
+If you don't have something to brag, boast, and strut about, then nobody
+in the underground will recognize you and favor you with vital cooperation
+and respect.
+
+The way to win a solid reputation in the underground
+is by telling other hackers things that could only
+have been learned by exceptional cunning and stealth.
+Forbidden knowledge, therefore, is the basic currency
+of the digital underground, like seashells among
+Trobriand Islanders. Hackers hoard this knowledge,
+and dwell upon it obsessively, and refine it,
+and bargain with it, and talk and talk about it.
+
+Many hackers even suffer from a strange obsession to TEACH--
+to spread the ethos and the knowledge of the digital underground.
+They'll do this even when it gains them no particular advantage
+and presents a grave personal risk.
+
+And when that risk catches up with them, they will go right on teaching
+and preaching--to a new audience this time, their interrogators from law
+enforcement. Almost every hacker arrested tells everything he knows--
+all about his friends, his mentors, his disciples--legends, threats,
+horror stories, dire rumors, gossip, hallucinations. This is, of course,
+convenient for law enforcement--except when law enforcement begins
+to believe hacker legendry.
+
+Phone phreaks are unique among criminals in their willingness
+to call up law enforcement officials--in the office, at their homes--
+and give them an extended piece of their mind. It is hard not to
+interpret this as BEGGING FOR ARREST, and in fact it is an act
+of incredible foolhardiness. Police are naturally nettled
+by these acts of chutzpah and will go well out of their way
+to bust these flaunting idiots. But it can also be interpreted
+as a product of a world-view so elitist, so closed and hermetic,
+that electronic police are simply not perceived as "police,"
+but rather as ENEMY PHONE PHREAKS who should be scolded
+into behaving "decently."
+
+Hackers at their most grandiloquent perceive themselves
+as the elite pioneers of a new electronic world.
+Attempts to make them obey the democratically
+established laws of contemporary American society are
+seen as repression and persecution. After all, they argue,
+if Alexander Graham Bell had gone along with the rules
+of the Western Union telegraph company, there would have
+been no telephones. If Jobs and Wozniak had believed
+that IBM was the be-all and end-all, there would have
+been no personal computers. If Benjamin Franklin and
+Thomas Jefferson had tried to "work within the system"
+there would have been no United States.
+
+Not only do hackers privately believe this as an article of faith,
+but they have been known to write ardent manifestos about it.
+Here are some revealing excerpts from an especially vivid hacker manifesto:
+"The Techno-Revolution" by "Dr. Crash," which appeared in electronic
+form in Phrack Volume 1, Issue 6, Phile 3.
+
+
+"To fully explain the true motives behind hacking,
+we must first take a quick look into the past. In the 1960s,
+a group of MIT students built the first modern computer system.
+This wild, rebellious group of young men were the first to bear
+the name `hackers.' The systems that they developed were intended
+to be used to solve world problems and to benefit all of mankind.
+"As we can see, this has not been the case. The computer system
+has been solely in the hands of big businesses and the government.
+The wonderful device meant to enrich life has become a weapon which
+dehumanizes people. To the government and large businesses,
+people are no more than disk space, and the government doesn't
+use computers to arrange aid for the poor, but to control nuclear
+death weapons. The average American can only have access
+to a small microcomputer which is worth only a fraction
+of what they pay for it. The businesses keep the
+true state-of-the-art equipment away from the people
+behind a steel wall of incredibly high prices and bureaucracy.
+It is because of this state of affairs that hacking was born. (. . .)
+"Of course, the government doesn't want the monopoly of technology broken,
+so they have outlawed hacking and arrest anyone who is caught. (. . .)
+The phone company is another example of technology abused and kept
+from people with high prices. (. . .) "Hackers often find that their
+existing equipment, due to the monopoly tactics of computer companies,
+is inefficient for their purposes. Due to the exorbitantly high prices,
+it is impossible to legally purchase the necessary equipment.
+This need has given still another segment of the fight: Credit Carding.
+Carding is a way of obtaining the necessary goods without paying for them.
+It is again due to the companies' stupidity that Carding is so easy,
+and shows that the world's businesses are in the hands of those
+with considerably less technical know-how than we, the hackers. (. . .)
+"Hacking must continue. We must train newcomers to the art of hacking.
+(. . . .) And whatever you do, continue the fight. Whether you know it
+or not, if you are a hacker, you are a revolutionary. Don't worry,
+you're on the right side."
+
+The defense of "carding" is rare. Most hackers regard credit-card
+theft as "poison" to the underground, a sleazy and immoral effort that,
+worse yet, is hard to get away with. Nevertheless, manifestos advocating
+credit-card theft, the deliberate crashing of computer systems,
+and even acts of violent physical destruction such as vandalism
+and arson do exist in the underground. These boasts and threats
+are taken quite seriously by the police. And not every hacker
+is an abstract, Platonic computer-nerd. Some few are quite experienced
+at picking locks, robbing phone-trucks, and breaking and entering buildings.
+
+Hackers vary in their degree of hatred for authority
+and the violence of their rhetoric. But, at a bottom line,
+they are scofflaws. They don't regard the current rules
+of electronic behavior as respectable efforts to preserve
+law and order and protect public safety. They regard these
+laws as immoral efforts by soulless corporations to protect
+their profit margins and to crush dissidents. "Stupid" people,
+including police, businessmen, politicians, and journalists,
+simply have no right to judge the actions of those possessed of genius,
+techno-revolutionary intentions, and technical expertise.
+
+#
+
+Hackers are generally teenagers and college kids not
+engaged in earning a living. They often come from fairly
+well-to-do middle-class backgrounds, and are markedly
+anti-materialistic (except, that is, when it comes to
+computer equipment). Anyone motivated by greed for
+mere money (as opposed to the greed for power,
+knowledge and status) is swiftly written-off as a narrow-
+minded breadhead whose interests can only be corrupt
+and contemptible. Having grown up in the 1970s and
+1980s, the young Bohemians of the digital underground
+regard straight society as awash in plutocratic corruption,
+where everyone from the President down is for sale and
+whoever has the gold makes the rules.
+
+Interestingly, there's a funhouse-mirror image of this attitude
+on the other side of the conflict. The police are also
+one of the most markedly anti-materialistic groups
+in American society, motivated not by mere money
+but by ideals of service, justice, esprit-de-corps,
+and, of course, their own brand of specialized knowledge
+and power. Remarkably, the propaganda war between cops
+and hackers has always involved angry allegations
+that the other side is trying to make a sleazy buck.
+Hackers consistently sneer that anti-phreak prosecutors
+are angling for cushy jobs as telco lawyers and that
+computer-crime police are aiming to cash in later
+as well-paid computer-security consultants in the private sector.
+
+For their part, police publicly conflate all
+hacking crimes with robbing payphones with crowbars.
+Allegations of "monetary losses" from computer intrusion
+are notoriously inflated. The act of illicitly copying
+a document from a computer is morally equated with
+directly robbing a company of, say, half a million dollars.
+The teenage computer intruder in possession of this "proprietary"
+document has certainly not sold it for such a sum, would likely
+have little idea how to sell it at all, and quite probably
+doesn't even understand what he has. He has not made a cent
+in profit from his felony but is still morally equated with
+a thief who has robbed the church poorbox and lit out for Brazil.
+
+Police want to believe that all hackers are thieves.
+It is a tortuous and almost unbearable act for the American
+justice system to put people in jail because they want
+to learn things which are forbidden for them to know.
+In an American context, almost any pretext for punishment
+is better than jailing people to protect certain restricted
+kinds of information. Nevertheless, POLICING INFORMATION
+is part and parcel of the struggle against hackers.
+
+This dilemma is well exemplified by the remarkable
+activities of "Emmanuel Goldstein," editor and publisher
+of a print magazine known as 2600: The Hacker Quarterly.
+Goldstein was an English major at Long Island's State University
+of New York in the '70s, when he became involved with the local
+college radio station. His growing interest in electronics
+caused him to drift into Yippie TAP circles and thus into
+the digital underground, where he became a self-described
+techno-rat. His magazine publishes techniques of computer
+intrusion and telephone "exploration" as well as gloating
+exposes of telco misdeeds and governmental failings.
+
+Goldstein lives quietly and very privately in a large,
+crumbling Victorian mansion in Setauket, New York.
+The seaside house is decorated with telco decals, chunks of
+driftwood, and the basic bric-a-brac of a hippie crash-pad.
+He is unmarried, mildly unkempt, and survives mostly
+on TV dinners and turkey-stuffing eaten straight out
+of the bag. Goldstein is a man of considerable charm
+and fluency, with a brief, disarming smile and the kind
+of pitiless, stubborn, thoroughly recidivist integrity
+that America's electronic police find genuinely alarming.
+
+Goldstein took his nom-de-plume, or "handle," from
+a character in Orwell's 1984, which may be taken,
+correctly, as a symptom of the gravity of his sociopolitical
+worldview. He is not himself a practicing computer
+intruder, though he vigorously abets these actions,
+especially when they are pursued against large
+corporations or governmental agencies. Nor is he a thief,
+for he loudly scorns mere theft of phone service, in favor
+of "exploring and manipulating the system." He is probably
+best described and understood as a DISSIDENT.
+
+Weirdly, Goldstein is living in modern America
+under conditions very similar to those of former
+East European intellectual dissidents. In other words,
+he flagrantly espouses a value-system that is deeply
+and irrevocably opposed to the system of those in power
+and the police. The values in 2600 are generally expressed
+in terms that are ironic, sarcastic, paradoxical, or just
+downright confused. But there's no mistaking their
+radically anti-authoritarian tenor. 2600 holds that
+technical power and specialized knowledge, of any kind
+obtainable, belong by right in the hands of those individuals
+brave and bold enough to discover them--by whatever means necessary.
+Devices, laws, or systems that forbid access, and the free
+spread of knowledge, are provocations that any free
+and self-respecting hacker should relentlessly attack.
+The "privacy" of governments, corporations and other soulless
+technocratic organizations should never be protected
+at the expense of the liberty and free initiative
+of the individual techno-rat.
+
+However, in our contemporary workaday world, both governments
+and corporations are very anxious indeed to police information
+which is secret, proprietary, restricted, confidential,
+copyrighted, patented, hazardous, illegal, unethical,
+embarrassing, or otherwise sensitive. This makes Goldstein
+persona non grata, and his philosophy a threat.
+
+Very little about the conditions of Goldstein's daily
+life would astonish, say, Vaclav Havel. (We may note
+in passing that President Havel once had his word-processor
+confiscated by the Czechoslovak police.) Goldstein lives
+by SAMIZDAT, acting semi-openly as a data-center
+for the underground, while challenging the powers-that-be
+to abide by their own stated rules: freedom of speech
+and the First Amendment.
+
+Goldstein thoroughly looks and acts the part of techno-rat,
+with shoulder-length ringlets and a piratical black
+fisherman's-cap set at a rakish angle. He often shows up
+like Banquo's ghost at meetings of computer professionals,
+where he listens quietly, half-smiling and taking thorough notes.
+
+Computer professionals generally meet publicly,
+and find it very difficult to rid themselves of Goldstein
+and his ilk without extralegal and unconstitutional actions.
+Sympathizers, many of them quite respectable people
+with responsible jobs, admire Goldstein's attitude and
+surreptitiously pass him information. An unknown but
+presumably large proportion of Goldstein's 2,000-plus
+readership are telco security personnel and police,
+who are forced to subscribe to 2600 to stay abreast
+of new developments in hacking. They thus find themselves
+PAYING THIS GUY'S RENT while grinding their teeth in anguish,
+a situation that would have delighted Abbie Hoffman
+(one of Goldstein's few idols).
+
+Goldstein is probably the best-known public representative
+of the hacker underground today, and certainly the best-hated.
+Police regard him as a Fagin, a corrupter of youth, and speak
+of him with untempered loathing. He is quite an accomplished gadfly.
+After the Martin Luther King Day Crash of 1990, Goldstein,
+for instance, adeptly rubbed salt into the wound in the pages of 2600.
+"Yeah, it was fun for the phone phreaks as we watched the network crumble,"
+he admitted cheerfully. "But it was also an ominous sign of what's
+to come. . . . Some AT&T people, aided by well-meaning but ignorant media,
+were spreading the notion that many companies had the same software
+and therefore could face the same problem someday. Wrong. This was
+entirely an AT&T software deficiency. Of course, other companies could
+face entirely DIFFERENT software problems. But then, so too could AT&T."
+
+After a technical discussion of the system's failings,
+the Long Island techno-rat went on to offer thoughtful
+criticism to the gigantic multinational's hundreds of
+professionally qualified engineers. "What we don't know
+is how a major force in communications like AT&T could
+be so sloppy. What happened to backups? Sure,
+computer systems go down all the time, but people
+making phone calls are not the same as people logging
+on to computers. We must make that distinction. It's not
+acceptable for the phone system or any other essential
+service to `go down.' If we continue to trust technology
+without understanding it, we can look forward to many
+variations on this theme.
+
+"AT&T owes it to its customers to be prepared to INSTANTLY
+switch to another network if something strange and unpredictable
+starts occurring. The news here isn't so much the failure
+of a computer program, but the failure of AT&T's entire structure."
+
+The very idea of this. . . . this PERSON. . . . offering
+"advice" about "AT&T's entire structure" is more than
+some people can easily bear. How dare this near-criminal
+dictate what is or isn't "acceptable" behavior from AT&T?
+Especially when he's publishing, in the very same issue,
+detailed schematic diagrams for creating various switching-network
+signalling tones unavailable to the public.
+
+"See what happens when you drop a `silver box' tone or two
+down your local exchange or through different long distance
+service carriers," advises 2600 contributor "Mr. Upsetter"
+in "How To Build a Signal Box." "If you experiment systematically
+and keep good records, you will surely discover something interesting."
+
+This is, of course, the scientific method, generally regarded
+as a praiseworthy activity and one of the flowers of modern civilization.
+One can indeed learn a great deal with this sort of structured
+intellectual activity. Telco employees regard this mode of "exploration"
+as akin to flinging sticks of dynamite into their pond to see what lives
+on the bottom.
+
+2600 has been published consistently since 1984.
+It has also run a bulletin board computer system,
+printed 2600 T-shirts, taken fax calls. . . .
+The Spring 1991 issue has an interesting announcement on page 45:
+"We just discovered an extra set of wires attached to our fax line
+and heading up the pole. (They've since been clipped.)
+Your faxes to us and to anyone else could be monitored."
+In the worldview of 2600, the tiny band of techno-rat brothers
+(rarely, sisters) are a beseiged vanguard of the truly free and honest.
+The rest of the world is a maelstrom of corporate crime and high-level
+governmental corruption, occasionally tempered with well-meaning
+ignorance. To read a few issues in a row is to enter a nightmare
+akin to Solzhenitsyn's, somewhat tempered by the fact that 2600
+is often extremely funny.
+
+Goldstein did not become a target of the Hacker Crackdown,
+though he protested loudly, eloquently, and publicly about it,
+and it added considerably to his fame. It was not that he is not
+regarded as dangerous, because he is so regarded. Goldstein has had
+brushes with the law in the past: in 1985, a 2600 bulletin board
+computer was seized by the FBI, and some software on it was formally
+declared "a burglary tool in the form of a computer program."
+But Goldstein escaped direct repression in 1990, because his
+magazine is printed on paper, and recognized as subject
+to Constitutional freedom of the press protection.
+As was seen in the Ramparts case, this is far from
+an absolute guarantee. Still, as a practical matter,
+shutting down 2600 by court-order would create so much
+legal hassle that it is simply unfeasible, at least
+for the present. Throughout 1990, both Goldstein
+and his magazine were peevishly thriving.
+
+Instead, the Crackdown of 1990 would concern itself
+with the computerized version of forbidden data.
+The crackdown itself, first and foremost, was about
+BULLETIN BOARD SYSTEMS. Bulletin Board Systems, most often
+known by the ugly and un-pluralizable acronym "BBS," are
+the life-blood of the digital underground. Boards were
+also central to law enforcement's tactics and strategy
+in the Hacker Crackdown.
+
+A "bulletin board system" can be formally defined as
+a computer which serves as an information and message-
+passing center for users dialing-up over the phone-lines
+through the use of modems. A "modem," or modulator-
+demodulator, is a device which translates the digital
+impulses of computers into audible analog telephone
+signals, and vice versa. Modems connect computers
+to phones and thus to each other.
+
+Large-scale mainframe computers have been connected since the 1960s,
+but PERSONAL computers, run by individuals out of their homes,
+were first networked in the late 1970s. The "board" created
+by Ward Christensen and Randy Suess in February 1978,
+in Chicago, Illinois, is generally regarded as the first
+personal-computer bulletin board system worthy of the name.
+
+Boards run on many different machines, employing many
+different kinds of software. Early boards were crude and buggy,
+and their managers, known as "system operators" or "sysops,"
+were hard-working technical experts who wrote their own software.
+But like most everything else in the world of electronics,
+boards became faster, cheaper, better-designed, and generally
+far more sophisticated throughout the 1980s. They also moved
+swiftly out of the hands of pioneers and into those of the
+general public. By 1985 there were something in the
+neighborhood of 4,000 boards in America. By 1990 it was
+calculated, vaguely, that there were about 30,000 boards in
+the US, with uncounted thousands overseas.
+
+Computer bulletin boards are unregulated enterprises.
+Running a board is a rough-and-ready, catch-as-catch-can proposition.
+Basically, anybody with a computer, modem, software and a phone-line
+can start a board. With second-hand equipment and public-domain
+free software, the price of a board might be quite small--
+less than it would take to publish a magazine or even a
+decent pamphlet. Entrepreneurs eagerly sell bulletin-board
+software, and will coach nontechnical amateur sysops in its use.
+
+Boards are not "presses." They are not magazines,
+or libraries, or phones, or CB radios, or traditional cork
+bulletin boards down at the local laundry, though they
+have some passing resemblance to those earlier media.
+Boards are a new medium--they may even be a LARGE NUMBER of new media.
+
+Consider these unique characteristics: boards are cheap,
+yet they can have a national, even global reach.
+Boards can be contacted from anywhere in the global
+telephone network, at NO COST to the person running the board--
+the caller pays the phone bill, and if the caller is local,
+the call is free. Boards do not involve an editorial elite
+addressing a mass audience. The "sysop" of a board is not
+an exclusive publisher or writer--he is managing an electronic salon,
+where individuals can address the general public, play the part
+of the general public, and also exchange private mail
+with other individuals. And the "conversation" on boards,
+though fluid, rapid, and highly interactive, is not spoken,
+but written. It is also relatively anonymous, sometimes completely so.
+
+And because boards are cheap and ubiquitous, regulations
+and licensing requirements would likely be practically unenforceable.
+It would almost be easier to "regulate," "inspect," and "license"
+the content of private mail--probably more so, since the mail system
+is operated by the federal government. Boards are run by individuals,
+independently, entirely at their own whim.
+
+For the sysop, the cost of operation is not the primary
+limiting factor. Once the investment in a computer and
+modem has been made, the only steady cost is the charge
+for maintaining a phone line (or several phone lines).
+The primary limits for sysops are time and energy.
+Boards require upkeep. New users are generally "validated"--
+they must be issued individual passwords, and called at
+home by voice-phone, so that their identity can be
+verified. Obnoxious users, who exist in plenty, must be
+chided or purged. Proliferating messages must be deleted
+when they grow old, so that the capacity of the system
+is not overwhelmed. And software programs (if such things
+are kept on the board) must be examined for possible
+computer viruses. If there is a financial charge to use
+the board (increasingly common, especially in larger and
+fancier systems) then accounts must be kept, and users
+must be billed. And if the board crashes--a very common
+occurrence--then repairs must be made.
+
+Boards can be distinguished by the amount of effort
+spent in regulating them. First, we have the completely
+open board, whose sysop is off chugging brews and
+watching re-runs while his users generally degenerate
+over time into peevish anarchy and eventual silence.
+Second comes the supervised board, where the sysop
+breaks in every once in a while to tidy up, calm brawls,
+issue announcements, and rid the community of dolts
+and troublemakers. Third is the heavily supervised
+board, which sternly urges adult and responsible behavior
+and swiftly edits any message considered offensive,
+impertinent, illegal or irrelevant. And last comes
+the completely edited "electronic publication," which
+is presented to a silent audience which is not allowed
+to respond directly in any way.
+
+Boards can also be grouped by their degree of anonymity.
+There is the completely anonymous board, where everyone
+uses pseudonyms--"handles"--and even the sysop is unaware
+of the user's true identity. The sysop himself is likely
+pseudonymous on a board of this type. Second, and rather
+more common, is the board where the sysop knows (or thinks
+he knows) the true names and addresses of all users,
+but the users don't know one another's names and may not know his.
+Third is the board where everyone has to use real names,
+and roleplaying and pseudonymous posturing are forbidden.
+
+Boards can be grouped by their immediacy. "Chat-lines"
+are boards linking several users together over several
+different phone-lines simultaneously, so that people
+exchange messages at the very moment that they type.
+(Many large boards feature "chat" capabilities along
+with other services.) Less immediate boards,
+perhaps with a single phoneline, store messages serially,
+one at a time. And some boards are only open for business
+in daylight hours or on weekends, which greatly slows response.
+A NETWORK of boards, such as "FidoNet," can carry electronic mail
+from board to board, continent to continent, across huge distances--
+but at a relative snail's pace, so that a message can take several
+days to reach its target audience and elicit a reply.
+
+Boards can be grouped by their degree of community.
+Some boards emphasize the exchange of private,
+person-to-person electronic mail. Others emphasize
+public postings and may even purge people who "lurk,"
+merely reading posts but refusing to openly participate.
+Some boards are intimate and neighborly. Others are frosty
+and highly technical. Some are little more than storage
+dumps for software, where users "download" and "upload" programs,
+but interact among themselves little if at all.
+
+Boards can be grouped by their ease of access. Some boards
+are entirely public. Others are private and restricted only
+to personal friends of the sysop. Some boards divide users by status.
+On these boards, some users, especially beginners, strangers or children,
+will be restricted to general topics, and perhaps forbidden to post.
+Favored users, though, are granted the ability to post as they please,
+and to stay "on-line" as long as they like, even to the disadvantage
+of other people trying to call in. High-status users can be given access
+to hidden areas in the board, such as off-color topics, private discussions,
+and/or valuable software. Favored users may even become "remote sysops"
+with the power to take remote control of the board through their own
+home computers. Quite often "remote sysops" end up doing all the work
+and taking formal control of the enterprise, despite the fact that it's
+physically located in someone else's house. Sometimes several "co-sysops"
+share power.
+
+And boards can also be grouped by size. Massive, nationwide
+commercial networks, such as CompuServe, Delphi, GEnie and Prodigy,
+are run on mainframe computers and are generally not considered "boards,"
+though they share many of their characteristics, such as electronic mail,
+discussion topics, libraries of software, and persistent and growing problems
+with civil-liberties issues. Some private boards have as many as
+thirty phone-lines and quite sophisticated hardware. And then
+there are tiny boards.
+
+Boards vary in popularity. Some boards are huge and crowded,
+where users must claw their way in against a constant busy-signal.
+Others are huge and empty--there are few things sadder than a formerly
+flourishing board where no one posts any longer, and the dead conversations
+of vanished users lie about gathering digital dust. Some boards are tiny
+and intimate, their telephone numbers intentionally kept confidential
+so that only a small number can log on.
+
+And some boards are UNDERGROUND.
+
+Boards can be mysterious entities. The activities of
+their users can be hard to differentiate from conspiracy.
+Sometimes they ARE conspiracies. Boards have harbored,
+or have been accused of harboring, all manner of fringe groups,
+and have abetted, or been accused of abetting, every manner
+of frowned-upon, sleazy, radical, and criminal activity.
+There are Satanist boards. Nazi boards. Pornographic boards.
+Pedophile boards. Drug- dealing boards. Anarchist boards.
+Communist boards. Gay and Lesbian boards (these exist in great profusion,
+many of them quite lively with well-established histories).
+Religious cult boards. Evangelical boards. Witchcraft
+boards, hippie boards, punk boards, skateboarder boards.
+Boards for UFO believers. There may well be boards for
+serial killers, airline terrorists and professional assassins.
+There is simply no way to tell. Boards spring up, flourish,
+and disappear in large numbers, in most every corner of
+the developed world. Even apparently innocuous public
+boards can, and sometimes do, harbor secret areas known
+only to a few. And even on the vast, public, commercial services,
+private mail is very private--and quite possibly criminal.
+
+Boards cover most every topic imaginable and some
+that are hard to imagine. They cover a vast spectrum
+of social activity. However, all board users do have
+something in common: their possession of computers
+and phones. Naturally, computers and phones are
+primary topics of conversation on almost every board.
+
+And hackers and phone phreaks, those utter devotees
+of computers and phones, live by boards. They swarm by boards.
+They are bred by boards. By the late 1980s, phone-phreak groups
+and hacker groups, united by boards, had proliferated fantastically.
+
+
+As evidence, here is a list of hacker groups compiled
+by the editors of Phrack on August 8, 1988.
+
+
+The Administration.
+Advanced Telecommunications, Inc.
+ALIAS.
+American Tone Travelers.
+Anarchy Inc.
+Apple Mafia.
+The Association.
+Atlantic Pirates Guild.
+
+Bad Ass Mother Fuckers.
+Bellcore.
+Bell Shock Force.
+Black Bag.
+
+Camorra.
+C&M Productions.
+Catholics Anonymous.
+Chaos Computer Club.
+Chief Executive Officers.
+Circle Of Death.
+Circle Of Deneb.
+Club X.
+Coalition of Hi-Tech
+Pirates.
+Coast-To-Coast.
+Corrupt Computing.
+Cult Of The
+Dead Cow.
+Custom Retaliations.
+
+Damage Inc.
+D&B Communications.
+The Danger Gang.
+Dec Hunters.
+Digital Gang.
+DPAK.
+
+Eastern Alliance.
+The Elite Hackers Guild.
+Elite Phreakers and Hackers Club.
+The Elite Society Of America.
+EPG.
+Executives Of Crime.
+Extasyy Elite.
+
+Fargo 4A.
+Farmers Of Doom.
+The Federation.
+Feds R Us.
+First Class.
+Five O.
+Five Star.
+Force Hackers.
+The 414s.
+
+Hack-A-Trip.
+Hackers Of America.
+High Mountain Hackers.
+High Society.
+The Hitchhikers.
+
+IBM Syndicate.
+The Ice Pirates.
+Imperial Warlords.
+Inner Circle.
+Inner Circle II.
+Insanity Inc.
+International Computer Underground Bandits.
+
+Justice League of America.
+
+Kaos Inc.
+Knights Of Shadow.
+Knights Of The Round Table.
+
+League Of Adepts.
+Legion Of Doom.
+Legion Of Hackers.
+Lords Of Chaos.
+Lunatic Labs, Unlimited.
+
+Master Hackers.
+MAD!
+The Marauders.
+MD/PhD.
+
+Metal Communications, Inc.
+MetalliBashers, Inc.
+MBI.
+
+Metro Communications.
+Midwest Pirates Guild.
+
+NASA Elite.
+The NATO Association.
+Neon Knights.
+
+Nihilist Order.
+Order Of The Rose.
+OSS.
+
+Pacific Pirates Guild.
+Phantom Access Associates.
+
+PHido PHreaks.
+The Phirm.
+Phlash.
+PhoneLine Phantoms.
+Phone Phreakers Of America.
+Phortune 500.
+
+Phreak Hack Delinquents.
+Phreak Hack Destroyers.
+
+Phreakers, Hackers, And Laundromat Employees Gang (PHALSE Gang).
+Phreaks Against Geeks.
+Phreaks Against Phreaks Against Geeks.
+Phreaks and Hackers of America.
+Phreaks Anonymous World Wide.
+Project Genesis.
+The Punk Mafia.
+
+The Racketeers.
+Red Dawn Text Files.
+Roscoe Gang.
+
+
+SABRE.
+Secret Circle of Pirates.
+Secret Service.
+707 Club.
+Shadow Brotherhood.
+Sharp Inc.
+65C02 Elite.
+
+Spectral Force.
+Star League.
+Stowaways.
+Strata-Crackers.
+
+
+Team Hackers '86.
+Team Hackers '87.
+
+TeleComputist Newsletter Staff.
+Tribunal Of Knowledge.
+
+Triple Entente.
+Turn Over And Die Syndrome (TOADS).
+
+300 Club.
+1200 Club.
+2300 Club.
+2600 Club.
+2601 Club.
+
+2AF.
+
+The United Soft WareZ Force.
+United Technical Underground.
+
+Ware Brigade.
+The Warelords.
+WASP.
+
+Contemplating this list is an impressive, almost humbling business.
+As a cultural artifact, the thing approaches poetry.
+
+Underground groups--subcultures--can be distinguished
+from independent cultures by their habit of referring
+constantly to the parent society. Undergrounds by their
+nature constantly must maintain a membrane of differentiation.
+Funny/distinctive clothes and hair, specialized jargon, specialized
+ghettoized areas in cities, different hours of rising, working,
+sleeping. . . . The digital underground, which specializes in information,
+relies very heavily on language to distinguish itself. As can be seen
+from this list, they make heavy use of parody and mockery.
+It's revealing to see who they choose to mock.
+
+First, large corporations. We have the Phortune 500,
+The Chief Executive Officers, Bellcore, IBM Syndicate,
+SABRE (a computerized reservation service maintained
+by airlines). The common use of "Inc." is telling--
+none of these groups are actual corporations,
+but take clear delight in mimicking them.
+
+Second, governments and police. NASA Elite, NATO Association.
+"Feds R Us" and "Secret Service" are fine bits of fleering boldness.
+OSS--the Office of Strategic Services was the forerunner of the CIA.
+
+Third, criminals. Using stigmatizing pejoratives as a perverse
+badge of honor is a time-honored tactic for subcultures:
+punks, gangs, delinquents, mafias, pirates, bandits, racketeers.
+
+Specialized orthography, especially the use of "ph" for "f"
+and "z" for the plural "s," are instant recognition symbols.
+So is the use of the numeral "0" for the letter "O"
+--computer-software orthography generally features a
+slash through the zero, making the distinction obvious.
+
+Some terms are poetically descriptive of computer intrusion:
+the Stowaways, the Hitchhikers, the PhoneLine Phantoms, Coast-to-Coast.
+Others are simple bravado and vainglorious puffery.
+(Note the insistent use of the terms "elite" and "master.")
+Some terms are blasphemous, some obscene, others merely cryptic--
+anything to puzzle, offend, confuse, and keep the straights at bay.
+
+Many hacker groups further re-encrypt their names
+by the use of acronyms: United Technical Underground
+becomes UTU, Farmers of Doom become FoD, the United SoftWareZ
+Force becomes, at its own insistence, "TuSwF," and woe to the
+ignorant rodent who capitalizes the wrong letters.
+
+It should be further recognized that the members of these groups
+are themselves pseudonymous. If you did, in fact, run across
+the "PhoneLine Phantoms," you would find them to consist of
+"Carrier Culprit," "The Executioner," "Black Majik,"
+"Egyptian Lover," "Solid State," and "Mr Icom."
+"Carrier Culprit" will likely be referred to by his friends
+as "CC," as in, "I got these dialups from CC of PLP."
+
+It's quite possible that this entire list refers to as
+few as a thousand people. It is not a complete list
+of underground groups--there has never been such a list,
+and there never will be. Groups rise, flourish, decline,
+share membership, maintain a cloud of wannabes and
+casual hangers-on. People pass in and out, are ostracized,
+get bored, are busted by police, or are cornered by telco
+security and presented with huge bills. Many "underground
+groups" are software pirates, "warez d00dz," who might break
+copy protection and pirate programs, but likely wouldn't dare
+to intrude on a computer-system.
+
+It is hard to estimate the true population of the digital
+underground. There is constant turnover. Most hackers
+start young, come and go, then drop out at age 22--
+the age of college graduation. And a large majority
+of "hackers" access pirate boards, adopt a handle,
+swipe software and perhaps abuse a phone-code or two,
+while never actually joining the elite.
+
+Some professional informants, who make it their business
+to retail knowledge of the underground to paymasters in private
+corporate security, have estimated the hacker population
+at as high as fifty thousand. This is likely highly inflated,
+unless one counts every single teenage software pirate
+and petty phone-booth thief. My best guess is about 5,000 people.
+Of these, I would guess that as few as a hundred are truly "elite"
+--active computer intruders, skilled enough to penetrate
+sophisticated systems and truly to worry corporate security
+and law enforcement.
+
+Another interesting speculation is whether this group
+is growing or not. Young teenage hackers are often
+convinced that hackers exist in vast swarms and will soon
+dominate the cybernetic universe. Older and wiser
+veterans, perhaps as wizened as 24 or 25 years old,
+are convinced that the glory days are long gone, that the cops
+have the underground's number now, and that kids these days
+are dirt-stupid and just want to play Nintendo.
+
+My own assessment is that computer intrusion, as a non-profit act
+of intellectual exploration and mastery, is in slow decline,
+at least in the United States; but that electronic fraud,
+especially telecommunication crime, is growing by leaps and bounds.
+
+One might find a useful parallel to the digital underground
+in the drug underground. There was a time, now much-obscured
+by historical revisionism, when Bohemians freely shared joints
+at concerts, and hip, small-scale marijuana dealers might
+turn people on just for the sake of enjoying a long stoned conversation
+about the Doors and Allen Ginsberg. Now drugs are increasingly verboten,
+except in a high-stakes, highly-criminal world of highly addictive drugs.
+Over years of disenchantment and police harassment, a vaguely ideological,
+free-wheeling drug underground has relinquished the business of drug-dealing
+to a far more savage criminal hard-core. This is not a pleasant prospect
+to contemplate, but the analogy is fairly compelling.
+
+What does an underground board look like? What distinguishes
+it from a standard board? It isn't necessarily the conversation--
+hackers often talk about common board topics, such as hardware, software,
+sex, science fiction, current events, politics, movies, personal gossip.
+Underground boards can best be distinguished by their files, or "philes,"
+pre-composed texts which teach the techniques and ethos of the underground.
+These are prized reservoirs of forbidden knowledge. Some are anonymous,
+but most proudly bear the handle of the "hacker" who has created them,
+and his group affiliation, if he has one.
+
+Here is a partial table-of-contents of philes from an underground board,
+somewhere in the heart of middle America, circa 1991. The descriptions
+are mostly self-explanatory.
+
+
+BANKAMER.ZIP 5406 06-11-91 Hacking Bank America
+CHHACK.ZIP 4481 06-11-91 Chilton Hacking
+CITIBANK.ZIP 4118 06-11-91 Hacking Citibank
+CREDIMTC.ZIP 3241 06-11-91 Hacking Mtc Credit Company
+DIGEST.ZIP 5159 06-11-91 Hackers Digest
+HACK.ZIP 14031 06-11-91 How To Hack
+HACKBAS.ZIP 5073 06-11-91 Basics Of Hacking
+HACKDICT.ZIP 42774 06-11-91 Hackers Dictionary
+HACKER.ZIP 57938 06-11-91 Hacker Info
+HACKERME.ZIP 3148 06-11-91 Hackers Manual
+HACKHAND.ZIP 4814 06-11-91 Hackers Handbook
+HACKTHES.ZIP 48290 06-11-91 Hackers Thesis
+HACKVMS.ZIP 4696 06-11-91 Hacking Vms Systems
+MCDON.ZIP 3830 06-11-91 Hacking Macdonalds (Home Of The Archs)
+P500UNIX.ZIP 15525 06-11-91 Phortune 500 Guide To Unix
+RADHACK.ZIP 8411 06-11-91 Radio Hacking
+TAOTRASH.DOC 4096 12-25-89 Suggestions For Trashing
+TECHHACK.ZIP 5063 06-11-91 Technical Hacking
+
+
+The files above are do-it-yourself manuals about computer intrusion.
+The above is only a small section of a much larger library of hacking
+and phreaking techniques and history. We now move into a different
+and perhaps surprising area.
+
++------------+
+ |Anarchy|
++------------+
+
+ANARC.ZIP 3641 06-11-91 Anarchy Files
+ANARCHST.ZIP 63703 06-11-91 Anarchist Book
+ANARCHY.ZIP 2076 06-11-91 Anarchy At Home
+ANARCHY3.ZIP 6982 06-11-91 Anarchy No 3
+ANARCTOY.ZIP 2361 06-11-91 Anarchy Toys
+ANTIMODM.ZIP 2877 06-11-91 Anti-modem Weapons
+ATOM.ZIP 4494 06-11-91 How To Make An Atom Bomb
+BARBITUA.ZIP 3982 06-11-91 Barbiturate Formula
+BLCKPWDR.ZIP 2810 06-11-91 Black Powder Formulas
+BOMB.ZIP 3765 06-11-91 How To Make Bombs
+BOOM.ZIP 2036 06-11-91 Things That Go Boom
+CHLORINE.ZIP 1926 06-11-91 Chlorine Bomb
+COOKBOOK.ZIP 1500 06-11-91 Anarchy Cook Book
+DESTROY.ZIP 3947 06-11-91 Destroy Stuff
+DUSTBOMB.ZIP 2576 06-11-91 Dust Bomb
+ELECTERR.ZIP 3230 06-11-91 Electronic Terror
+EXPLOS1.ZIP 2598 06-11-91 Explosives 1
+EXPLOSIV.ZIP 18051 06-11-91 More Explosives
+EZSTEAL.ZIP 4521 06-11-91 Ez-stealing
+FLAME.ZIP 2240 06-11-91 Flame Thrower
+FLASHLT.ZIP 2533 06-11-91 Flashlight Bomb
+FMBUG.ZIP 2906 06-11-91 How To Make An Fm Bug
+OMEEXPL.ZIP 2139 06-11-91 Home Explosives
+HOW2BRK.ZIP 3332 06-11-91 How To Break In
+LETTER.ZIP 2990 06-11-91 Letter Bomb
+LOCK.ZIP 2199 06-11-91 How To Pick Locks
+MRSHIN.ZIP 3991 06-11-91 Briefcase Locks
+NAPALM.ZIP 3563 06-11-91 Napalm At Home
+NITRO.ZIP 3158 06-11-91 Fun With Nitro
+PARAMIL.ZIP 2962 06-11-91 Paramilitary Info
+PICKING.ZIP 3398 06-11-91 Picking Locks
+PIPEBOMB.ZIP 2137 06-11-91 Pipe Bomb
+POTASS.ZIP 3987 06-11-91 Formulas With Potassium
+PRANK.TXT 11074 08-03-90 More Pranks To Pull On Idiots!
+REVENGE.ZIP 4447 06-11-91 Revenge Tactics
+ROCKET.ZIP 2590 06-11-91 Rockets For Fun
+SMUGGLE.ZIP 3385 06-11-91 How To Smuggle
+
+HOLY COW! The damned thing is full of stuff about bombs!
+
+What are we to make of this?
+
+First, it should be acknowledged that spreading
+knowledge about demolitions to teenagers is a highly and
+deliberately antisocial act. It is not, however, illegal.
+
+Second, it should be recognized that most of these
+philes were in fact WRITTEN by teenagers. Most adult
+American males who can remember their teenage years
+will recognize that the notion of building a flamethrower
+in your garage is an incredibly neat-o idea. ACTUALLY,
+building a flamethrower in your garage, however, is
+fraught with discouraging difficulty. Stuffing gunpowder
+into a booby-trapped flashlight, so as to blow the arm off
+your high-school vice-principal, can be a thing of dark
+beauty to contemplate. Actually committing assault by
+explosives will earn you the sustained attention of the
+federal Bureau of Alcohol, Tobacco and Firearms.
+
+Some people, however, will actually try these plans.
+A determinedly murderous American teenager can probably
+buy or steal a handgun far more easily than he can brew
+fake "napalm" in the kitchen sink. Nevertheless,
+if temptation is spread before people, a certain number
+will succumb, and a small minority will actually attempt
+these stunts. A large minority of that small minority
+will either fail or, quite likely, maim themselves,
+since these "philes" have not been checked for accuracy,
+are not the product of professional experience,
+and are often highly fanciful. But the gloating menace
+of these philes is not to be entirely dismissed.
+
+Hackers may not be "serious" about bombing; if they were,
+we would hear far more about exploding flashlights, homemade bazookas,
+and gym teachers poisoned by chlorine and potassium.
+However, hackers are VERY serious about forbidden knowledge.
+They are possessed not merely by curiosity, but by
+a positive LUST TO KNOW. The desire to know what
+others don't is scarcely new. But the INTENSITY
+of this desire, as manifested by these young technophilic
+denizens of the Information Age, may in fact BE new,
+and may represent some basic shift in social values--
+a harbinger of what the world may come to, as society
+lays more and more value on the possession,
+assimilation and retailing of INFORMATION
+as a basic commodity of daily life.
+
+There have always been young men with obsessive interests
+in these topics. Never before, however, have they been able
+to network so extensively and easily, and to propagandize
+their interests with impunity to random passers-by.
+High-school teachers will recognize that there's always
+one in a crowd, but when the one in a crowd escapes control
+by jumping into the phone-lines, and becomes a hundred such kids
+all together on a board, then trouble is brewing visibly.
+The urge of authority to DO SOMETHING, even something drastic,
+is hard to resist. And in 1990, authority did something.
+In fact authority did a great deal.
+
+#
+
+The process by which boards create hackers goes something
+like this. A youngster becomes interested in computers--
+usually, computer games. He hears from friends that
+"bulletin boards" exist where games can be obtained for free.
+(Many computer games are "freeware," not copyrighted--
+invented simply for the love of it and given away to the public;
+some of these games are quite good.) He bugs his parents for a modem,
+or quite often, uses his parents' modem.
+
+The world of boards suddenly opens up. Computer games
+can be quite expensive, real budget-breakers for a kid,
+but pirated games, stripped of copy protection, are cheap or free.
+They are also illegal, but it is very rare, almost unheard of,
+for a small-scale software pirate to be prosecuted.
+Once "cracked" of its copy protection, the program,
+being digital data, becomes infinitely reproducible.
+Even the instructions to the game, any manuals that accompany it,
+can be reproduced as text files, or photocopied from legitimate sets.
+Other users on boards can give many useful hints in game-playing tactics.
+And a youngster with an infinite supply of free computer games can
+certainly cut quite a swath among his modem-less friends.
+
+And boards are pseudonymous. No one need know that you're
+fourteen years old--with a little practice at subterfuge,
+you can talk to adults about adult things, and be accepted
+and taken seriously! You can even pretend to be a girl,
+or an old man, or anybody you can imagine. If you find this
+kind of deception gratifying, there is ample opportunity
+to hone your ability on boards.
+
+But local boards can grow stale. And almost every board maintains
+a list of phone-numbers to other boards, some in distant, tempting,
+exotic locales. Who knows what they're up to, in Oregon or Alaska
+or Florida or California? It's very easy to find out--just order
+the modem to call through its software--nothing to this, just typing
+on a keyboard, the same thing you would do for most any computer game.
+The machine reacts swiftly and in a few seconds you are talking to
+a bunch of interesting people on another seaboard.
+
+And yet the BILLS for this trivial action can be staggering!
+Just by going tippety-tap with your fingers, you may have
+saddled your parents with four hundred bucks in long-distance charges,
+and gotten chewed out but good. That hardly seems fair.
+
+How horrifying to have made friends in another state
+and to be deprived of their company--and their software--
+just because telephone companies demand absurd amounts of money!
+How painful, to be restricted to boards in one's own AREA CODE--
+what the heck is an "area code" anyway, and what makes it so special?
+A few grumbles, complaints, and innocent questions of this sort
+will often elicit a sympathetic reply from another board user--
+someone with some stolen codes to hand. You dither a while,
+knowing this isn't quite right, then you make up your mind
+to try them anyhow--AND THEY WORK! Suddenly you're doing something
+even your parents can't do. Six months ago you were just some kid--now,
+you're the Crimson Flash of Area Code 512! You're bad--you're nationwide!
+
+Maybe you'll stop at a few abused codes. Maybe you'll decide that
+boards aren't all that interesting after all, that it's wrong,
+not worth the risk --but maybe you won't. The next step
+is to pick up your own repeat-dialling program--
+to learn to generate your own stolen codes.
+(This was dead easy five years ago, much harder
+to get away with nowadays, but not yet impossible.)
+And these dialling programs are not complex or intimidating--
+some are as small as twenty lines of software.
+
+Now, you too can share codes. You can trade codes to learn
+other techniques. If you're smart enough to catch on,
+and obsessive enough to want to bother, and ruthless enough
+to start seriously bending rules, then you'll get better, fast.
+You start to develop a rep. You move up to a heavier class
+of board--a board with a bad attitude, the kind of board
+that naive dopes like your classmates and your former self
+have never even heard of! You pick up the jargon of phreaking
+and hacking from the board. You read a few of those anarchy philes--
+and man, you never realized you could be a real OUTLAW without
+ever leaving your bedroom.
+
+You still play other computer games, but now you have a new
+and bigger game. This one will bring you a different kind of status
+than destroying even eight zillion lousy space invaders.
+
+Hacking is perceived by hackers as a "game." This is
+not an entirely unreasonable or sociopathic perception.
+You can win or lose at hacking, succeed or fail,
+but it never feels "real." It's not simply that
+imaginative youngsters sometimes have a hard time
+telling "make-believe" from "real life." Cyberspace
+is NOT REAL! "Real" things are physical objects
+like trees and shoes and cars. Hacking takes place
+on a screen. Words aren't physical, numbers
+(even telephone numbers and credit card numbers)
+aren't physical. Sticks and stones may break my bones,
+but data will never hurt me. Computers SIMULATE reality,
+like computer games that simulate tank battles or dogfights
+or spaceships. Simulations are just make-believe,
+and the stuff in computers is NOT REAL.
+
+Consider this: if "hacking" is supposed to be so serious and
+real-life and dangerous, then how come NINE-YEAR-OLD KIDS have
+computers and modems? You wouldn't give a nine year old his own car,
+or his own rifle, or his own chainsaw--those things are "real."
+
+People underground are perfectly aware that the "game"
+is frowned upon by the powers that be. Word gets around
+about busts in the underground. Publicizing busts is one
+of the primary functions of pirate boards, but they also
+promulgate an attitude about them, and their own idiosyncratic
+ideas of justice. The users of underground boards won't complain
+if some guy is busted for crashing systems, spreading viruses,
+or stealing money by wire-fraud. They may shake their heads
+with a sneaky grin, but they won't openly defend these practices.
+But when a kid is charged with some theoretical amount of theft:
+$233,846.14, for instance, because he sneaked into a computer
+and copied something, and kept it in his house on a floppy disk--
+this is regarded as a sign of near-insanity from prosecutors,
+a sign that they've drastically mistaken the immaterial game
+of computing for their real and boring everyday world
+of fatcat corporate money.
+
+It's as if big companies and their suck-up lawyers
+think that computing belongs to them, and they can
+retail it with price stickers, as if it were boxes
+of laundry soap! But pricing "information" is like
+trying to price air or price dreams. Well, anybody
+on a pirate board knows that computing can be,
+and ought to be, FREE. Pirate boards are little
+independent worlds in cyberspace, and they don't belong
+to anybody but the underground. Underground boards
+aren't "brought to you by Procter & Gamble."
+
+To log on to an underground board can mean to
+experience liberation, to enter a world where,
+for once, money isn't everything and adults
+don't have all the answers.
+
+Let's sample another vivid hacker manifesto. Here are
+some excerpts from "The Conscience of a Hacker," by "The Mentor,"
+from Phrack Volume One, Issue 7, Phile 3.
+
+"I made a discovery today. I found a computer.
+Wait a second, this is cool. It does what I want it to.
+If it makes a mistake, it's because I screwed it up.
+Not because it doesn't like me. (. . .)
+"And then it happened. . .a door opened to a world. . .
+rushing through the phone line like heroin through an
+addict's veins, an electronic pulse is sent out,
+a refuge from day-to-day incompetencies is sought. . .
+a board is found. `This is it. . .this is where I belong. . .'
+"I know everyone here. . .even if I've never met them,
+never talked to them, may never hear from them again. . .
+I know you all. . . (. . .)
+
+"This is our world now. . .the world of the electron
+and the switch, the beauty of the baud. We make use of a
+service already existing without paying for what could be
+dirt-cheap if it wasn't run by profiteering gluttons, and you
+call us criminals. We explore. . .and you call us criminals.
+We seek after knowledge. . .and you call us criminals.
+We exist without skin color, without nationality,
+without religious bias. . .and you call us criminals.
+You build atomic bombs, you wage wars, you murder,
+cheat and lie to us and try to make us believe that
+it's for our own good, yet we're the criminals.
+
+"Yes, I am a criminal. My crime is that of curiosity.
+My crime is that of judging people by what they say and think,
+not what they look like. My crime is that of outsmarting you,
+something that you will never forgive me for."
+
+#
+
+There have been underground boards almost as long
+as there have been boards. One of the first was 8BBS,
+which became a stronghold of the West Coast phone-phreak elite.
+After going on-line in March 1980, 8BBS sponsored "Susan Thunder,"
+and "Tuc," and, most notoriously, "the Condor." "The Condor"
+bore the singular distinction of becoming the most vilified
+American phreak and hacker ever. Angry underground associates,
+fed up with Condor's peevish behavior, turned him in to police,
+along with a heaping double-helping of outrageous hacker legendry.
+As a result, Condor was kept in solitary confinement for seven months,
+for fear that he might start World War Three by triggering missile silos
+from the prison payphone. (Having served his time, Condor is now
+walking around loose; WWIII has thus far conspicuously failed to occur.)
+
+The sysop of 8BBS was an ardent free-speech enthusiast
+who simply felt that ANY attempt to restrict the expression
+of his users was unconstitutional and immoral.
+Swarms of the technically curious entered 8BBS
+and emerged as phreaks and hackers, until, in 1982,
+a friendly 8BBS alumnus passed the sysop a new modem
+which had been purchased by credit-card fraud.
+Police took this opportunity to seize the entire board
+and remove what they considered an attractive nuisance.
+
+Plovernet was a powerful East Coast pirate board
+that operated in both New York and Florida.
+Owned and operated by teenage hacker "Quasi Moto,"
+Plovernet attracted five hundred eager users in 1983.
+"Emmanuel Goldstein" was one-time co-sysop of Plovernet,
+along with "Lex Luthor," founder of the "Legion of Doom" group.
+Plovernet bore the signal honor of being the original home
+of the "Legion of Doom," about which the reader will be hearing
+a great deal, soon.
+
+"Pirate-80," or "P-80," run by a sysop known as "Scan-Man,"
+got into the game very early in Charleston, and continued
+steadily for years. P-80 flourished so flagrantly that
+even its most hardened users became nervous, and some
+slanderously speculated that "Scan Man" must have ties
+to corporate security, a charge he vigorously denied.
+
+"414 Private" was the home board for the first GROUP
+to attract conspicuous trouble, the teenage "414 Gang,"
+whose intrusions into Sloan-Kettering Cancer Center and
+Los Alamos military computers were to be a nine-days-wonder in 1982.
+
+At about this time, the first software piracy boards
+began to open up, trading cracked games for the Atari 800
+and the Commodore C64. Naturally these boards were
+heavily frequented by teenagers. And with the 1983
+release of the hacker-thriller movie War Games,
+the scene exploded. It seemed that every kid
+in America had demanded and gotten a modem for Christmas.
+Most of these dabbler wannabes put their modems in the attic
+after a few weeks, and most of the remainder minded their
+P's and Q's and stayed well out of hot water. But some
+stubborn and talented diehards had this hacker kid in
+War Games figured for a happening dude. They simply
+could not rest until they had contacted the underground--
+or, failing that, created their own.
+
+In the mid-80s, underground boards sprang up like digital fungi.
+ShadowSpawn Elite. Sherwood Forest I, II, and III.
+Digital Logic Data Service in Florida, sysoped by no less
+a man than "Digital Logic" himself; Lex Luthor of the
+Legion of Doom was prominent on this board, since it
+was in his area code. Lex's own board, "Legion of Doom,"
+started in 1984. The Neon Knights ran a network of Apple-
+hacker boards: Neon Knights North, South, East and West.
+Free World II was run by "Major Havoc." Lunatic Labs
+is still in operation as of this writing. Dr. Ripco
+in Chicago, an anything-goes anarchist board with an
+extensive and raucous history, was seized by Secret Service
+agents in 1990 on Sundevil day, but up again almost immediately,
+with new machines and scarcely diminished vigor.
+
+The St. Louis scene was not to rank with major centers
+of American hacking such as New York and L.A. But St.
+Louis did rejoice in possession of "Knight Lightning"
+and "Taran King," two of the foremost JOURNALISTS native
+to the underground. Missouri boards like Metal Shop,
+Metal Shop Private, Metal Shop Brewery, may not have
+been the heaviest boards around in terms of illicit
+expertise. But they became boards where hackers could
+exchange social gossip and try to figure out what the
+heck was going on nationally--and internationally.
+Gossip from Metal Shop was put into the form of news files,
+then assembled into a general electronic publication,
+Phrack, a portmanteau title coined from "phreak" and "hack."
+The Phrack editors were as obsessively curious about other
+hackers as hackers were about machines.
+
+Phrack, being free of charge and lively reading, began
+to circulate throughout the underground. As Taran King
+and Knight Lightning left high school for college,
+Phrack began to appear on mainframe machines linked to BITNET,
+and, through BITNET to the "Internet," that loose but
+extremely potent not-for-profit network where academic,
+governmental and corporate machines trade data through
+the UNIX TCP/IP protocol. (The "Internet Worm" of
+November 2-3,1988, created by Cornell grad student Robert Morris,
+was to be the largest and best-publicized computer-intrusion scandal
+to date. Morris claimed that his ingenious "worm" program was meant
+to harmlessly explore the Internet, but due to bad programming,
+the Worm replicated out of control and crashed some six thousand
+Internet computers. Smaller-scale and less ambitious Internet hacking
+was a standard for the underground elite.)
+
+Most any underground board not hopelessly lame and out-of-it
+would feature a complete run of Phrack--and, possibly,
+the lesser-known standards of the underground:
+the Legion of Doom Technical Journal, the obscene
+and raucous Cult of the Dead Cow files, P/HUN magazine,
+Pirate, the Syndicate Reports, and perhaps the highly
+anarcho-political Activist Times Incorporated.
+
+Possession of Phrack on one's board was prima facie
+evidence of a bad attitude. Phrack was seemingly everywhere,
+aiding, abetting, and spreading the underground ethos.
+And this did not escape the attention of corporate security
+or the police.
+
+We now come to the touchy subject of police and boards.
+Police, do, in fact, own boards. In 1989, there were
+police-sponsored boards in California, Colorado, Florida,
+Georgia, Idaho, Michigan, Missouri, Texas, and Virginia:
+boards such as "Crime Bytes," "Crimestoppers," "All Points"
+and "Bullet-N-Board." Police officers, as private computer
+enthusiasts, ran their own boards in Arizona, California,
+Colorado, Connecticut, Florida, Missouri, Maryland,
+New Mexico, North Carolina, Ohio, Tennessee and Texas.
+Police boards have often proved helpful in community relations.
+Sometimes crimes are reported on police boards.
+
+Sometimes crimes are COMMITTED on police boards.
+This has sometimes happened by accident, as naive hackers
+blunder onto police boards and blithely begin offering telephone codes.
+Far more often, however, it occurs through the now almost-traditional
+use of "sting boards." The first police sting-boards were established
+in 1985: "Underground Tunnel" in Austin, Texas, whose sysop
+Sgt. Robert Ansley called himself "Pluto"--"The Phone Company"
+in Phoenix, Arizona, run by Ken MacLeod of the Maricopa County
+Sheriff's office--and Sgt. Dan Pasquale's board in Fremont, California.
+Sysops posed as hackers, and swiftly garnered coteries of ardent users,
+who posted codes and loaded pirate software with abandon,
+and came to a sticky end.
+
+Sting boards, like other boards, are cheap to operate,
+very cheap by the standards of undercover police operations.
+Once accepted by the local underground, sysops will likely be
+invited into other pirate boards, where they can compile more dossiers.
+And when the sting is announced and the worst offenders arrested,
+the publicity is generally gratifying. The resultant paranoia
+in the underground--perhaps more justly described as a "deterrence effect"--
+tends to quell local lawbreaking for quite a while.
+
+Obviously police do not have to beat the underbrush for hackers.
+On the contrary, they can go trolling for them. Those caught
+can be grilled. Some become useful informants. They can lead
+the way to pirate boards all across the country.
+
+And boards all across the country showed the sticky
+fingerprints of Phrack, and of that loudest and most
+flagrant of all underground groups, the "Legion of Doom."
+
+The term "Legion of Doom" came from comic books. The Legion of Doom,
+a conspiracy of costumed super- villains headed by the chrome-domed
+criminal ultra- mastermind Lex Luthor, gave Superman a lot of four-color
+graphic trouble for a number of decades. Of course, Superman,
+that exemplar of Truth, Justice, and the American Way,
+always won in the long run. This didn't matter to the hacker Doomsters--
+"Legion of Doom" was not some thunderous and evil Satanic reference,
+it was not meant to be taken seriously. "Legion of Doom" came
+from funny-books and was supposed to be funny.
+
+"Legion of Doom" did have a good mouthfilling ring to it, though.
+It sounded really cool. Other groups, such as the "Farmers of Doom,"
+closely allied to LoD, recognized this grandiloquent quality,
+and made fun of it. There was even a hacker group called
+"Justice League of America," named after Superman's club
+of true-blue crimefighting superheros.
+
+But they didn't last; the Legion did.
+
+The original Legion of Doom, hanging out on Quasi Moto's Plovernet board,
+were phone phreaks. They weren't much into computers. "Lex Luthor" himself
+(who was under eighteen when he formed the Legion) was a COSMOS expert,
+COSMOS being the "Central System for Mainframe Operations,"
+a telco internal computer network. Lex would eventually become
+quite a dab hand at breaking into IBM mainframes, but although
+everyone liked Lex and admired his attitude, he was not considered
+a truly accomplished computer intruder. Nor was he the "mastermind"
+of the Legion of Doom--LoD were never big on formal leadership.
+As a regular on Plovernet and sysop of his "Legion of Doom BBS,"
+Lex was the Legion's cheerleader and recruiting officer.
+
+Legion of Doom began on the ruins of an earlier phreak group,
+The Knights of Shadow. Later, LoD was to subsume the personnel
+of the hacker group "Tribunal of Knowledge." People came and went
+constantly in LoD; groups split up or formed offshoots.
+
+Early on, the LoD phreaks befriended a few computer-intrusion
+enthusiasts, who became the associated "Legion of Hackers."
+Then the two groups conflated into the "Legion of Doom/Hackers,"
+or LoD/H. When the original "hacker" wing, Messrs. "Compu-Phreak"
+and "Phucked Agent 04," found other matters to occupy their time,
+the extra "/H" slowly atrophied out of the name; but by this time
+the phreak wing, Messrs. Lex Luthor, "Blue Archer," "Gary Seven,"
+"Kerrang Khan," "Master of Impact," "Silver Spy," "The Marauder,"
+and "The Videosmith," had picked up a plethora of intrusion
+expertise and had become a force to be reckoned with.
+
+LoD members seemed to have an instinctive understanding
+that the way to real power in the underground lay through
+covert publicity. LoD were flagrant. Not only was it one
+of the earliest groups, but the members took pains to widely
+distribute their illicit knowledge. Some LoD members,
+like "The Mentor," were close to evangelical about it.
+Legion of Doom Technical Journal began to show up on boards
+throughout the underground.
+
+LoD Technical Journal was named in cruel parody
+of the ancient and honored AT&T Technical Journal.
+The material in these two publications was quite similar--
+much of it, adopted from public journals and discussions
+in the telco community. And yet, the predatory attitude
+of LoD made even its most innocuous data seem deeply sinister;
+an outrage; a clear and present danger.
+
+To see why this should be, let's consider the following
+(invented) paragraphs, as a kind of thought experiment.
+
+(A) "W. Fred Brown, AT&T Vice President for
+Advanced Technical Development, testified May 8
+at a Washington hearing of the National Telecommunications
+and Information Administration (NTIA), regarding
+Bellcore's GARDEN project. GARDEN (Generalized
+Automatic Remote Distributed Electronic Network) is a
+telephone-switch programming tool that makes it possible
+to develop new telecom services, including hold-on-hold
+and customized message transfers, from any keypad terminal,
+within seconds. The GARDEN prototype combines centrex
+lines with a minicomputer using UNIX operating system software."
+
+(B) "Crimson Flash 512 of the Centrex Mobsters reports:
+D00dz, you wouldn't believe this GARDEN bullshit Bellcore's
+just come up with! Now you don't even need a lousy Commodore
+to reprogram a switch--just log on to GARDEN as a technician,
+and you can reprogram switches right off the keypad in any
+public phone booth! You can give yourself hold-on-hold
+and customized message transfers, and best of all,
+the thing is run off (notoriously insecure) centrex lines
+using--get this--standard UNIX software! Ha ha ha ha!"
+
+Message (A), couched in typical techno-bureaucratese,
+appears tedious and almost unreadable. (A) scarcely seems
+threatening or menacing. Message (B), on the other hand,
+is a dreadful thing, prima facie evidence of a dire conspiracy,
+definitely not the kind of thing you want your teenager reading.
+
+The INFORMATION, however, is identical. It is PUBLIC
+information, presented before the federal government in
+an open hearing. It is not "secret." It is not "proprietary."
+It is not even "confidential." On the contrary, the
+development of advanced software systems is a matter
+of great public pride to Bellcore.
+
+However, when Bellcore publicly announces a project of this kind,
+it expects a certain attitude from the public--something along
+the lines of GOSH WOW, YOU GUYS ARE GREAT, KEEP THAT UP, WHATEVER IT IS--
+certainly not cruel mimickry, one-upmanship and outrageous speculations
+about possible security holes.
+
+Now put yourself in the place of a policeman confronted by
+an outraged parent, or telco official, with a copy of Version (B).
+This well-meaning citizen, to his horror, has discovered
+a local bulletin-board carrying outrageous stuff like (B),
+which his son is examining with a deep and unhealthy interest.
+If (B) were printed in a book or magazine, you, as an American
+law enforcement officer, would know that it would take
+a hell of a lot of trouble to do anything about it;
+but it doesn't take technical genius to recognize that
+if there's a computer in your area harboring stuff like (B),
+there's going to be trouble.
+
+In fact, if you ask around, any computer-literate cop
+will tell you straight out that boards with stuff like (B)
+are the SOURCE of trouble. And the WORST source of trouble
+on boards are the ringleaders inventing and spreading stuff like (B).
+If it weren't for these jokers, there wouldn't BE any trouble.
+
+And Legion of Doom were on boards like nobody else.
+Plovernet. The Legion of Doom Board. The Farmers of Doom Board.
+Metal Shop. OSUNY. Blottoland. Private Sector. Atlantis.
+Digital Logic. Hell Phrozen Over.
+
+LoD members also ran their own boards. "Silver Spy" started
+his own board, "Catch-22," considered one of the heaviest around.
+So did "Mentor," with his "Phoenix Project." When they didn't run boards
+themselves, they showed up on other people's boards, to brag, boast,
+and strut. And where they themselves didn't go, their philes went,
+carrying evil knowledge and an even more evil attitude.
+
+As early as 1986, the police were under the vague impression
+that EVERYONE in the underground was Legion of Doom.
+LoD was never that large--considerably smaller than either
+"Metal Communications" or "The Administration," for instance--
+but LoD got tremendous press. Especially in Phrack,
+which at times read like an LoD fan magazine; and Phrack
+was everywhere, especially in the offices of telco security.
+You couldn't GET busted as a phone phreak, a hacker,
+or even a lousy codes kid or warez dood, without the cops
+asking if you were LoD.
+
+This was a difficult charge to deny, as LoD never
+distributed membership badges or laminated ID cards.
+If they had, they would likely have died out quickly,
+for turnover in their membership was considerable.
+LoD was less a high-tech street-gang than an ongoing
+state-of-mind. LoD was the Gang That Refused to Die.
+By 1990, LoD had RULED for ten years, and it seemed WEIRD
+to police that they were continually busting people who were
+only sixteen years old. All these teenage small-timers
+were pleading the tiresome hacker litany of "just curious,
+no criminal intent." Somewhere at the center of this
+conspiracy there had to be some serious adult masterminds,
+not this seemingly endless supply of myopic suburban
+white kids with high SATs and funny haircuts.
+
+There was no question that most any American hacker
+arrested would "know" LoD. They knew the handles
+of contributors to LoD Tech Journal, and were likely
+to have learned their craft through LoD boards and LoD activism.
+But they'd never met anyone from LoD. Even some of the
+rotating cadre who were actually and formally "in LoD"
+knew one another only by board-mail and pseudonyms.
+This was a highly unconventional profile for a criminal conspiracy.
+Computer networking, and the rapid evolution of the digital underground,
+made the situation very diffuse and confusing.
+
+Furthermore, a big reputation in the digital underground
+did not coincide with one's willingness to commit "crimes."
+Instead, reputation was based on cleverness and technical mastery.
+As a result, it often seemed that the HEAVIER the hackers were,
+the LESS likely they were to have committed any kind of common,
+easily prosecutable crime. There were some hackers who could really steal.
+And there were hackers who could really hack. But the two groups didn't seem
+to overlap much, if at all. For instance, most people in the underground
+looked up to "Emmanuel Goldstein" of 2600 as a hacker demigod.
+But Goldstein's publishing activities were entirely legal--
+Goldstein just printed dodgy stuff and talked about politics,
+he didn't even hack. When you came right down to it,
+Goldstein spent half his time complaining that computer security
+WASN'T STRONG ENOUGH and ought to be drastically improved
+across the board!
+
+Truly heavy-duty hackers, those with serious technical skills
+who had earned the respect of the underground, never stole money
+or abused credit cards. Sometimes they might abuse phone-codes--
+but often, they seemed to get all the free phone-time they wanted
+without leaving a trace of any kind.
+
+The best hackers, the most powerful and technically accomplished,
+were not professional fraudsters. They raided computers habitually,
+but wouldn't alter anything, or damage anything. They didn't even steal
+computer equipment--most had day-jobs messing with hardware,
+and could get all the cheap secondhand equipment they wanted.
+The hottest hackers, unlike the teenage wannabes, weren't snobs
+about fancy or expensive hardware. Their machines tended to be
+raw second-hand digital hot-rods full of custom add-ons that
+they'd cobbled together out of chickenwire, memory chips and spit.
+Some were adults, computer software writers and consultants by trade,
+and making quite good livings at it. Some of them ACTUALLY WORKED
+FOR THE PHONE COMPANY--and for those, the "hackers" actually found
+under the skirts of Ma Bell, there would be little mercy in 1990.
+
+It has long been an article of faith in the
+underground that the "best" hackers never get caught.
+They're far too smart, supposedly. They never get caught
+because they never boast, brag, or strut. These demigods
+may read underground boards (with a condescending smile),
+but they never say anything there. The "best" hackers,
+according to legend, are adult computer professionals,
+such as mainframe system administrators, who already know
+the ins and outs of their particular brand of security.
+Even the "best" hacker can't break in to just any computer at random:
+the knowledge of security holes is too specialized, varying widely
+with different software and hardware. But if people are employed to run,
+say, a UNIX mainframe or a VAX/VMS machine, then they tend to learn
+security from the inside out. Armed with this knowledge,
+they can look into most anybody else's UNIX or VMS
+without much trouble or risk, if they want to.
+And, according to hacker legend, of course they want to,
+so of course they do. They just don't make a big deal
+of what they've done. So nobody ever finds out.
+
+It is also an article of faith in the underground that
+professional telco people "phreak" like crazed weasels.
+OF COURSE they spy on Madonna's phone calls--I mean,
+WOULDN'T YOU? Of course they give themselves free long-
+distance--why the hell should THEY pay, they're running
+the whole shebang!
+
+It has, as a third matter, long been an article of faith
+that any hacker caught can escape serious punishment if
+he confesses HOW HE DID IT. Hackers seem to believe
+that governmental agencies and large corporations are
+blundering about in cyberspace like eyeless jellyfish
+or cave salamanders. They feel that these large
+but pathetically stupid organizations will proffer up
+genuine gratitude, and perhaps even a security post
+and a big salary, to the hot-shot intruder who will deign
+to reveal to them the supreme genius of his modus operandi.
+
+In the case of longtime LoD member "Control-C,"
+this actually happened, more or less. Control-C had led
+Michigan Bell a merry chase, and when captured in 1987,
+he turned out to be a bright and apparently physically
+harmless young fanatic, fascinated by phones. There was
+no chance in hell that Control-C would actually repay the
+enormous and largely theoretical sums in long-distance
+service that he had accumulated from Michigan Bell.
+He could always be indicted for fraud or computer-intrusion,
+but there seemed little real point in this--he hadn't
+physically damaged any computer. He'd just plead guilty,
+and he'd likely get the usual slap-on-the-wrist,
+and in the meantime it would be a big hassle for Michigan Bell
+just to bring up the case. But if kept on the payroll,
+he might at least keep his fellow hackers at bay.
+
+There were uses for him. For instance, a contrite
+Control-C was featured on Michigan Bell internal posters,
+sternly warning employees to shred their trash.
+He'd always gotten most of his best inside info from
+"trashing"--raiding telco dumpsters, for useful data
+indiscreetly thrown away. He signed these posters, too.
+Control-C had become something like a Michigan Bell mascot.
+And in fact, Control-C DID keep other hackers at bay.
+Little hackers were quite scared of Control-C and his
+heavy-duty Legion of Doom friends. And big hackers WERE
+his friends and didn't want to screw up his cushy situation.
+
+No matter what one might say of LoD, they did stick together.
+When "Wasp," an apparently genuinely malicious New York hacker,
+began crashing Bellcore machines, Control-C received swift volunteer
+help from "the Mentor" and the Georgia LoD wing made up of
+"The Prophet," "Urvile," and "Leftist." Using Mentor's Phoenix
+Project board to coordinate, the Doomsters helped telco security
+to trap Wasp, by luring him into a machine with a tap
+and line-trace installed. Wasp lost. LoD won! And my, did they brag.
+
+Urvile, Prophet and Leftist were well-qualified for this activity,
+probably more so even than the quite accomplished Control-C.
+The Georgia boys knew all about phone switching-stations.
+Though relative johnny-come-latelies in the Legion of Doom,
+they were considered some of LoD's heaviest guys,
+into the hairiest systems around. They had the good fortune
+to live in or near Atlanta, home of the sleepy and apparently
+tolerant BellSouth RBOC.
+
+As RBOC security went, BellSouth were "cake." US West (of Arizona,
+the Rockies and the Pacific Northwest) were tough and aggressive,
+probably the heaviest RBOC around. Pacific Bell, California's PacBell,
+were sleek, high-tech, and longtime veterans of the LA phone-phreak wars.
+NYNEX had the misfortune to run the New York City area, and were warily
+prepared for most anything. Even Michigan Bell, a division of the
+Ameritech RBOC, at least had the elementary sense to hire their own hacker
+as a useful scarecrow. But BellSouth, even though their corporate P.R.
+proclaimed them to have "Everything You Expect From a Leader," were pathetic.
+
+When rumor about LoD's mastery of Georgia's switching network got around
+to BellSouth through Bellcore and telco security scuttlebutt,
+they at first refused to believe it. If you paid serious attention
+to every rumor out and about these hacker kids, you would hear all kinds
+of wacko saucer-nut nonsense: that the National Security Agency
+monitored all American phone calls, that the CIA and DEA tracked
+traffic on bulletin-boards with word-analysis programs,
+that the Condor could start World War III from a payphone.
+
+If there were hackers into BellSouth switching-stations, then how come
+nothing had happened? Nothing had been hurt. BellSouth's machines
+weren't crashing. BellSouth wasn't suffering especially badly from fraud.
+BellSouth's customers weren't complaining. BellSouth was headquartered
+in Atlanta, ambitious metropolis of the new high-tech Sunbelt;
+and BellSouth was upgrading its network by leaps and bounds,
+digitizing the works left right and center. They could hardly be
+considered sluggish or naive. BellSouth's technical expertise
+was second to none, thank you kindly. But then came the Florida business.
+
+On June 13, 1989, callers to the Palm Beach County Probation Department,
+in Delray Beach, Florida, found themselves involved in a remarkable
+discussion with a phone-sex worker named "Tina" in New York State.
+Somehow, ANY call to this probation office near Miami was instantly
+and magically transported across state lines, at no extra charge to the user,
+to a pornographic phone-sex hotline hundreds of miles away!
+
+This practical joke may seem utterly hilarious at first hearing,
+and indeed there was a good deal of chuckling about it in
+phone phreak circles, including the Autumn 1989 issue of 2600.
+But for Southern Bell (the division of the BellSouth RBOC
+supplying local service for Florida, Georgia, North Carolina
+and South Carolina), this was a smoking gun. For the first time ever,
+a computer intruder had broken into a BellSouth central office
+switching station and re-programmed it!
+
+Or so BellSouth thought in June 1989. Actually, LoD members had been
+frolicking harmlessly in BellSouth switches since September 1987.
+The stunt of June 13--call-forwarding a number through manipulation
+of a switching station--was child's play for hackers as accomplished
+as the Georgia wing of LoD. Switching calls interstate sounded like
+a big deal, but it took only four lines of code to accomplish this.
+An easy, yet more discreet, stunt, would be to call-forward another
+number to your own house. If you were careful and considerate,
+and changed the software back later, then not a soul would know.
+Except you. And whoever you had bragged to about it.
+
+As for BellSouth, what they didn't know wouldn't hurt them.
+
+Except now somebody had blown the whole thing wide open, and BellSouth knew.
+
+A now alerted and considerably paranoid BellSouth began searching switches
+right and left for signs of impropriety, in that hot summer of 1989.
+No fewer than forty-two BellSouth employees were put on 12-hour shifts,
+twenty-four hours a day, for two solid months, poring over records
+and monitoring computers for any sign of phony access. These forty-two
+overworked experts were known as BellSouth's "Intrusion Task Force."
+
+What the investigators found astounded them. Proprietary telco databases
+had been manipulated: phone numbers had been created out of thin air,
+with no users' names and no addresses. And perhaps worst of all,
+no charges and no records of use. The new digital ReMOB (Remote Observation)
+diagnostic feature had been extensively tampered with--hackers had learned to
+reprogram ReMOB software, so that they could listen in on any switch-routed
+call at their leisure! They were using telco property to SPY!
+
+The electrifying news went out throughout law enforcement in 1989.
+It had never really occurred to anyone at BellSouth that their prized
+and brand-new digital switching-stations could be RE-PROGRAMMED.
+People seemed utterly amazed that anyone could have the nerve.
+Of course these switching stations were "computers," and everybody
+knew hackers liked to "break into computers:" but telephone people's
+computers were DIFFERENT from normal people's computers.
+
+The exact reason WHY these computers were "different" was
+rather ill-defined. It certainly wasn't the extent of their security.
+The security on these BellSouth computers was lousy; the AIMSX computers,
+for instance, didn't even have passwords. But there was no question that
+BellSouth strongly FELT that their computers were very different indeed.
+And if there were some criminals out there who had not gotten that message,
+BellSouth was determined to see that message taught.
+
+After all, a 5ESS switching station was no mere bookkeeping system for
+some local chain of florists. Public service depended on these stations.
+Public SAFETY depended on these stations.
+
+And hackers, lurking in there call-forwarding or ReMobbing, could spy
+on anybody in the local area! They could spy on telco officials!
+They could spy on police stations! They could spy on local offices
+of the Secret Service. . . .
+
+In 1989, electronic cops and hacker-trackers began using scrambler-phones
+and secured lines. It only made sense. There was no telling who was into
+those systems. Whoever they were, they sounded scary. This was some
+new level of antisocial daring. Could be West German hackers, in the pay
+of the KGB. That too had seemed a weird and farfetched notion,
+until Clifford Stoll had poked and prodded a sluggish Washington
+law-enforcement bureaucracy into investigating a computer intrusion
+that turned out to be exactly that--HACKERS, IN THE PAY OF THE KGB!
+Stoll, the systems manager for an Internet lab in Berkeley California,
+had ended up on the front page of the New Nork Times, proclaimed a national
+hero in the first true story of international computer espionage.
+Stoll's counterspy efforts, which he related in a bestselling book,
+The Cuckoo's Egg, in 1989, had established the credibility of `hacking'
+as a possible threat to national security. The United States Secret Service
+doesn't mess around when it suspects a possible action by a foreign
+intelligence apparat.
+
+The Secret Service scrambler-phones and secured lines put
+a tremendous kink in law enforcement's ability to operate freely;
+to get the word out, cooperate, prevent misunderstandings.
+Nevertheless, 1989 scarcely seemed the time for half-measures.
+If the police and Secret Service themselves were not operationally secure,
+then how could they reasonably demand measures of security from
+private enterprise? At least, the inconvenience made people aware
+of the seriousness of the threat.
+
+If there was a final spur needed to get the police off the dime,
+it came in the realization that the emergency 911 system was vulnerable.
+The 911 system has its own specialized software, but it is run on the same
+digital switching systems as the rest of the telephone network.
+911 is not physically different from normal telephony. But it is
+certainly culturally different, because this is the area of
+telephonic cyberspace reserved for the police and emergency services.
+
+Your average policeman may not know much about hackers or phone-phreaks.
+Computer people are weird; even computer COPS are rather weird;
+the stuff they do is hard to figure out. But a threat to the 911 system
+is anything but an abstract threat. If the 911 system goes, people can die.
+
+Imagine being in a car-wreck, staggering to a phone-booth,
+punching 911 and hearing "Tina" pick up the phone-sex line
+somewhere in New York! The situation's no longer comical, somehow.
+
+And was it possible? No question. Hackers had attacked 911
+systems before. Phreaks can max-out 911 systems just by siccing
+a bunch of computer-modems on them in tandem, dialling them over
+and over until they clog. That's very crude and low-tech,
+but it's still a serious business.
+
+The time had come for action. It was time to take stern measures
+with the underground. It was time to start picking up the dropped threads,
+the loose edges, the bits of braggadocio here and there; it was time to get
+on the stick and start putting serious casework together. Hackers weren't
+"invisible." They THOUGHT they were invisible; but the truth was,
+they had just been tolerated too long.
+
+Under sustained police attention in the summer of '89, the digital
+underground began to unravel as never before.
+
+The first big break in the case came very early on: July 1989,
+the following month. The perpetrator of the "Tina" switch was caught,
+and confessed. His name was "Fry Guy," a 16-year-old in Indiana.
+Fry Guy had been a very wicked young man.
+
+Fry Guy had earned his handle from a stunt involving French fries.
+Fry Guy had filched the log-in of a local MacDonald's manager
+and had logged-on to the MacDonald's mainframe on the Sprint
+Telenet system. Posing as the manager, Fry Guy had altered
+MacDonald's records, and given some teenage hamburger-flipping
+friends of his, generous raises. He had not been caught.
+
+Emboldened by success, Fry Guy moved on to credit-card abuse.
+Fry Guy was quite an accomplished talker; with a gift for
+"social engineering." If you can do "social engineering"
+--fast-talk, fake-outs, impersonation, conning, scamming--
+then card abuse comes easy. (Getting away with it in
+the long run is another question).
+
+Fry Guy had run across "Urvile" of the Legion of Doom
+on the ALTOS Chat board in Bonn, Germany. ALTOS Chat
+was a sophisticated board, accessible through globe-spanning
+computer networks like BITnet, Tymnet, and Telenet.
+ALTOS was much frequented by members of Germany's
+Chaos Computer Club. Two Chaos hackers who hung out on ALTOS,
+"Jaeger" and "Pengo," had been the central villains of
+Clifford Stoll's Cuckoo's Egg case: consorting in East Berlin
+with a spymaster from the KGB, and breaking into American
+computers for hire, through the Internet.
+
+When LoD members learned the story of Jaeger's depredations
+from Stoll's book, they were rather less than impressed,
+technically speaking. On LoD's own favorite board of the moment,
+"Black Ice," LoD members bragged that they themselves could have done
+all the Chaos break-ins in a week flat! Nevertheless, LoD were grudgingly
+impressed by the Chaos rep, the sheer hairy-eyed daring of hash-smoking
+anarchist hackers who had rubbed shoulders with the fearsome big-boys
+of international Communist espionage. LoD members sometimes traded
+bits of knowledge with friendly German hackers on ALTOS--phone numbers
+for vulnerable VAX/VMS computers in Georgia, for instance.
+Dutch and British phone phreaks, and the Australian clique of
+"Phoenix," "Nom," and "Electron," were ALTOS regulars, too.
+In underground circles, to hang out on ALTOS was considered
+the sign of an elite dude, a sophisticated hacker of the
+international digital jet-set.
+
+Fry Guy quickly learned how to raid information from credit-card
+consumer-reporting agencies. He had over a hundred stolen credit-card
+numbers in his notebooks, and upwards of a thousand swiped long-distance
+access codes. He knew how to get onto Altos, and how to talk the talk of
+the underground convincingly. He now wheedled knowledge of switching-station
+tricks from Urvile on the ALTOS system.
+
+Combining these two forms of knowledge enabled Fry Guy to bootstrap
+his way up to a new form of wire-fraud. First, he'd snitched credit card
+numbers from credit-company computers. The data he copied included names,
+addresses and phone numbers of the random card-holders.
+
+Then Fry Guy, impersonating a card-holder, called up Western Union
+and asked for a cash advance on "his" credit card. Western Union,
+as a security guarantee, would call the customer back, at home,
+to verify the transaction.
+
+But, just as he had switched the Florida probation office to "Tina"
+in New York, Fry Guy switched the card-holder's number to a local pay-phone.
+There he would lurk in wait, muddying his trail by routing and re-routing
+the call, through switches as far away as Canada. When the call came through,
+he would boldly "social-engineer," or con, the Western Union people, pretending
+to be the legitimate card-holder. Since he'd answered the proper phone number,
+the deception was not very hard. Western Union's money was then shipped to
+a confederate of Fry Guy's in his home town in Indiana.
+
+Fry Guy and his cohort, using LoD techniques, stole six thousand dollars
+from Western Union between December 1988 and July 1989. They also dabbled
+in ordering delivery of stolen goods through card-fraud. Fry Guy
+was intoxicated with success. The sixteen-year-old fantasized wildly
+to hacker rivals, boasting that he'd used rip-off money to hire himself
+a big limousine, and had driven out-of-state with a groupie from
+his favorite heavy-metal band, Motley Crue.
+
+Armed with knowledge, power, and a gratifying stream of free money,
+Fry Guy now took it upon himself to call local representatives
+of Indiana Bell security, to brag, boast, strut, and utter
+tormenting warnings that his powerful friends in the notorious
+Legion of Doom could crash the national telephone network.
+Fry Guy even named a date for the scheme: the Fourth of July,
+a national holiday.
+
+This egregious example of the begging-for-arrest syndrome was shortly
+followed by Fry Guy's arrest. After the Indiana telephone company figured
+out who he was, the Secret Service had DNRs--Dialed Number Recorders--
+installed on his home phone lines. These devices are not taps, and can't
+record the substance of phone calls, but they do record the phone numbers
+of all calls going in and out. Tracing these numbers showed Fry Guy's
+long-distance code fraud, his extensive ties to pirate bulletin boards,
+and numerous personal calls to his LoD friends in Atlanta. By July 11,
+1989, Prophet, Urvile and Leftist also had Secret Service DNR
+"pen registers" installed on their own lines.
+
+The Secret Service showed up in force at Fry Guy's house on July 22, 1989,
+to the horror of his unsuspecting parents. The raiders were led by
+a special agent from the Secret Service's Indianapolis office.
+However, the raiders were accompanied and advised by Timothy M. Foley
+of the Secret Service's Chicago office (a gentleman about whom
+we will soon be hearing a great deal).
+
+Following federal computer-crime techniques that had been standard
+since the early 1980s, the Secret Service searched the house thoroughly,
+and seized all of Fry Guy's electronic equipment and notebooks.
+All Fry Guy's equipment went out the door in the custody of the
+Secret Service, which put a swift end to his depredations.
+
+The USSS interrogated Fry Guy at length. His case was put in the charge
+of Deborah Daniels, the federal US Attorney for the Southern District
+of Indiana. Fry Guy was charged with eleven counts of computer fraud,
+unauthorized computer access, and wire fraud. The evidence was thorough
+and irrefutable. For his part, Fry Guy blamed his corruption on the
+Legion of Doom and offered to testify against them.
+
+Fry Guy insisted that the Legion intended to crash the phone system
+on a national holiday. And when AT&T crashed on Martin Luther King Day,
+1990, this lent a credence to his claim that genuinely alarmed telco
+security and the Secret Service.
+
+Fry Guy eventually pled guilty on May 31, 1990. On September 14,
+he was sentenced to forty-four months' probation and four hundred hours'
+community service. He could have had it much worse; but it made sense
+to prosecutors to take it easy on this teenage minor, while zeroing
+in on the notorious kingpins of the Legion of Doom.
+
+But the case against LoD had nagging flaws. Despite the best effort
+of investigators, it was impossible to prove that the Legion had crashed
+the phone system on January 15, because they, in fact, hadn't done so.
+The investigations of 1989 did show that certain members of
+the Legion of Doom had achieved unprecedented power over the telco
+switching stations, and that they were in active conspiracy
+to obtain more power yet. Investigators were privately convinced
+that the Legion of Doom intended to do awful things with this knowledge,
+but mere evil intent was not enough to put them in jail.
+
+And although the Atlanta Three--Prophet, Leftist, and especially Urvile--
+had taught Fry Guy plenty, they were not themselves credit-card fraudsters.
+The only thing they'd "stolen" was long-distance service--and since they'd
+done much of that through phone-switch manipulation, there was no easy way
+to judge how much they'd "stolen," or whether this practice was even "theft"
+of any easily recognizable kind.
+
+Fry Guy's theft of long-distance codes had cost the phone companies plenty.
+The theft of long-distance service may be a fairly theoretical "loss,"
+but it costs genuine money and genuine time to delete all those stolen codes,
+and to re-issue new codes to the innocent owners of those corrupted codes.
+The owners of the codes themselves are victimized, and lose time and money
+and peace of mind in the hassle. And then there were the credit-card victims
+to deal with, too, and Western Union. When it came to rip-off, Fry Guy was
+far more of a thief than LoD. It was only when it came to actual computer
+expertise that Fry Guy was small potatoes.
+
+The Atlanta Legion thought most "rules" of cyberspace were for rodents
+and losers, but they DID have rules. THEY NEVER CRASHED ANYTHING,
+AND THEY NEVER TOOK MONEY. These were rough rules-of-thumb, and
+rather dubious principles when it comes to the ethical subtleties
+of cyberspace, but they enabled the Atlanta Three to operate with
+a relatively clear conscience (though never with peace of mind).
+
+If you didn't hack for money, if you weren't robbing people of actual funds
+--money in the bank, that is-- then nobody REALLY got hurt, in LoD's opinion.
+"Theft of service" was a bogus issue, and "intellectual property" was
+a bad joke. But LoD had only elitist contempt for rip-off artists,
+"leechers," thieves. They considered themselves clean. In their opinion,
+if you didn't smash-up or crash any systems --(well, not on purpose, anyhow--
+accidents can happen, just ask Robert Morris) then it was very unfair
+to call you a "vandal" or a "cracker." When you were hanging out on-line
+with your "pals" in telco security, you could face them down from the higher
+plane of hacker morality. And you could mock the police from the supercilious
+heights of your hacker's quest for pure knowledge.
+
+But from the point of view of law enforcement and telco security, however,
+Fry Guy was not really dangerous. The Atlanta Three WERE dangerous.
+It wasn't the crimes they were committing, but the DANGER,
+the potential hazard, the sheer TECHNICAL POWER LoD had accumulated,
+that had made the situation untenable. Fry Guy was not LoD.
+He'd never laid eyes on anyone in LoD; his only contacts with them
+had been electronic. Core members of the Legion of Doom tended to meet
+physically for conventions every year or so, to get drunk, give each other
+the hacker high-sign, send out for pizza and ravage hotel suites.
+Fry Guy had never done any of this. Deborah Daniels assessed Fry Guy
+accurately as "an LoD wannabe."
+
+Nevertheless Fry Guy's crimes would be directly attributed to LoD
+in much future police propaganda. LoD would be described as
+"a closely knit group" involved in "numerous illegal activities"
+including "stealing and modifying individual credit histories,"
+and "fraudulently obtaining money and property." Fry Guy did this,
+but the Atlanta Three didn't; they simply weren't into theft,
+but rather intrusion. This caused a strange kink in
+the prosecution's strategy. LoD were accused of
+"disseminating information about attacking computers
+to other computer hackers in an effort to shift the focus
+of law enforcement to those other hackers and away from the Legion of Doom."
+
+This last accusation (taken directly from a press release by the Chicago
+Computer Fraud and Abuse Task Force) sounds particularly far-fetched.
+One might conclude at this point that investigators would have been
+well-advised to go ahead and "shift their focus" from the "Legion of Doom."
+Maybe they SHOULD concentrate on "those other hackers"--the ones who were
+actually stealing money and physical objects.
+
+But the Hacker Crackdown of 1990 was not a simple policing action.
+It wasn't meant just to walk the beat in cyberspace--it was a CRACKDOWN,
+a deliberate attempt to nail the core of the operation, to send a dire
+and potent message that would settle the hash of the digital underground
+for good.
+
+By this reasoning, Fry Guy wasn't much more than the electronic equivalent
+of a cheap streetcorner dope dealer. As long as the masterminds of LoD were
+still flagrantly operating, pushing their mountains of illicit knowledge
+right and left, and whipping up enthusiasm for blatant lawbreaking,
+then there would be an INFINITE SUPPLY of Fry Guys.
+
+Because LoD were flagrant, they had left trails everywhere,
+to be picked up by law enforcement in New York, Indiana,
+Florida, Texas, Arizona, Missouri, even Australia.
+But 1990's war on the Legion of Doom was led out of Illinois,
+by the Chicago Computer Fraud and Abuse Task Force.
+
+#
+
+The Computer Fraud and Abuse Task Force, led by federal prosecutor
+William J. Cook, had started in 1987 and had swiftly become one
+of the most aggressive local "dedicated computer-crime units."
+Chicago was a natural home for such a group. The world's first
+computer bulletin-board system had been invented in Illinois.
+The state of Illinois had some of the nation's first and sternest
+computer crime laws. Illinois State Police were markedly alert
+to the possibilities of white-collar crime and electronic fraud.
+
+And William J. Cook in particular was a rising star in
+electronic crime-busting. He and his fellow federal prosecutors
+at the U.S. Attorney's office in Chicago had a tight relation
+with the Secret Service, especially go-getting Chicago-based agent
+Timothy Foley. While Cook and his Department of Justice colleagues
+plotted strategy, Foley was their man on the street.
+
+Throughout the 1980s, the federal government had given prosecutors
+an armory of new, untried legal tools against computer crime.
+Cook and his colleagues were pioneers in the use of these new statutes
+in the real-life cut-and-thrust of the federal courtroom.
+
+On October 2, 1986, the US Senate had passed the
+"Computer Fraud and Abuse Act" unanimously, but there
+were pitifully few convictions under this statute.
+Cook's group took their name from this statute,
+since they were determined to transform this powerful but
+rather theoretical Act of Congress into a real-life engine
+of legal destruction against computer fraudsters and scofflaws.
+
+It was not a question of merely discovering crimes,
+investigating them, and then trying and punishing their
+perpetrators. The Chicago unit, like most everyone else
+in the business, already KNEW who the bad guys were:
+the Legion of Doom and the writers and editors of Phrack.
+The task at hand was to find some legal means of putting
+these characters away.
+
+This approach might seem a bit dubious, to someone not
+acquainted with the gritty realities of prosecutorial work.
+But prosecutors don't put people in jail for crimes
+they have committed; they put people in jail for crimes
+they have committed THAT CAN BE PROVED IN COURT.
+Chicago federal police put Al Capone in prison
+for income-tax fraud. Chicago is a big town,
+with a rough-and-ready bare-knuckle tradition
+on both sides of the law.
+
+Fry Guy had broken the case wide open and alerted telco security
+to the scope of the problem. But Fry Guy's crimes would not
+put the Atlanta Three behind bars--much less the wacko underground
+journalists of Phrack. So on July 22, 1989, the same day that
+Fry Guy was raided in Indiana, the Secret Service descended upon
+the Atlanta Three.
+
+This was likely inevitable. By the summer of 1989, law enforcement
+were closing in on the Atlanta Three from at least six directions at once.
+First, there were the leads from Fry Guy, which had led to the DNR registers
+being installed on the lines of the Atlanta Three. The DNR evidence alone
+would have finished them off, sooner or later.
+
+But second, the Atlanta lads were already well-known to Control-C
+and his telco security sponsors. LoD's contacts with telco security
+had made them overconfident and even more boastful than usual;
+they felt that they had powerful friends in high places,
+and that they were being openly tolerated by telco security.
+But BellSouth's Intrusion Task Force were hot on the trail of LoD
+and sparing no effort or expense.
+
+The Atlanta Three had also been identified by name and listed
+on the extensive anti-hacker files maintained, and retailed for pay,
+by private security operative John Maxfield of Detroit.
+Maxfield, who had extensive ties to telco security
+and many informants in the underground, was a bete noire
+of the Phrack crowd, and the dislike was mutual.
+
+
+The Atlanta Three themselves had written articles for Phrack.
+This boastful act could not possibly escape telco and law enforcement
+attention.
+
+"Knightmare," a high-school age hacker from Arizona,
+was a close friend and disciple of Atlanta LoD,
+but he had been nabbed by the formidable Arizona
+Organized Crime and Racketeering Unit. Knightmare was
+on some of LoD's favorite boards--"Black Ice" in particular--
+and was privy to their secrets. And to have Gail Thackeray,
+the Assistant Attorney General of Arizona, on one's trail
+was a dreadful peril for any hacker.
+
+And perhaps worst of all, Prophet had committed a major blunder
+by passing an illicitly copied BellSouth computer-file to Knight Lightning,
+who had published it in Phrack. This, as we will see, was an act of dire
+consequence for almost everyone concerned.
+
+On July 22, 1989, the Secret Service showed up at the Leftist's house,
+where he lived with his parents. A massive squad of some twenty officers
+surrounded the building: Secret Service, federal marshals, local police,
+possibly BellSouth telco security; it was hard to tell in the crush.
+Leftist's dad, at work in his basement office, first noticed
+a muscular stranger in plain clothes crashing through the
+back yard with a drawn pistol. As more strangers poured
+into the house, Leftist's dad naturally assumed there was
+an armed robbery in progress.
+
+Like most hacker parents, Leftist's mom and dad had only the vaguest
+notions of what their son had been up to all this time. Leftist had
+a day-job repairing computer hardware. His obsession with computers
+seemed a bit odd, but harmless enough, and likely to produce a well-
+paying career. The sudden, overwhelming raid left Leftist's
+parents traumatized.
+
+The Leftist himself had been out after work with his co-workers,
+surrounding a couple of pitchers of margaritas. As he came trucking
+on tequila-numbed feet up the pavement, toting a bag full of floppy-disks,
+he noticed a large number of unmarked cars parked in his driveway.
+All the cars sported tiny microwave antennas.
+
+The Secret Service had knocked the front door off its hinges,
+almost flattening his mom.
+
+Inside, Leftist was greeted by Special Agent James Cool
+of the US Secret Service, Atlanta office. Leftist was flabbergasted.
+He'd never met a Secret Service agent before. He could not imagine
+that he'd ever done anything worthy of federal attention.
+He'd always figured that if his activities became intolerable,
+one of his contacts in telco security would give him a private
+phone-call and tell him to knock it off.
+
+But now Leftist was pat-searched for weapons by grim professionals,
+and his bag of floppies was quickly seized. He and his parents were
+all shepherded into separate rooms and grilled at length as a score
+of officers scoured their home for anything electronic.
+
+Leftist was horrified as his treasured IBM AT personal computer
+with its forty-meg hard disk, and his recently purchased 80386 IBM-clone
+with a whopping hundred-meg hard disk, both went swiftly out the door
+in Secret Service custody. They also seized all his disks, all his notebooks,
+and a tremendous booty in dogeared telco documents that Leftist had snitched
+out of trash dumpsters.
+
+Leftist figured the whole thing for a big misunderstanding.
+He'd never been into MILITARY computers. He wasn't a SPY or a COMMUNIST.
+He was just a good ol' Georgia hacker, and now he just wanted all these
+people out of the house. But it seemed they wouldn't go until he made
+some kind of statement.
+
+And so, he levelled with them.
+
+And that, Leftist said later from his federal prison camp in Talladega,
+Alabama, was a big mistake. The Atlanta area was unique,
+in that it had three members of the Legion of Doom who actually
+occupied more or less the same physical locality. Unlike the rest
+of LoD, who tended to associate by phone and computer,
+Atlanta LoD actually WERE "tightly knit." It was no real
+surprise that the Secret Service agents apprehending Urvile
+at the computer-labs at Georgia Tech, would discover Prophet
+with him as well.
+
+Urvile, a 21-year-old Georgia Tech student in polymer chemistry,
+posed quite a puzzling case for law enforcement. Urvile--also known
+as "Necron 99," as well as other handles, for he tended to change his
+cover-alias about once a month--was both an accomplished hacker
+and a fanatic simulation-gamer.
+
+Simulation games are an unusual hobby; but then hackers are unusual people,
+and their favorite pastimes tend to be somewhat out of the ordinary.
+The best-known American simulation game is probably "Dungeons & Dragons,"
+a multi-player parlor entertainment played with paper, maps, pencils,
+statistical tables and a variety of oddly-shaped dice. Players pretend
+to be heroic characters exploring a wholly-invented fantasy world.
+The fantasy worlds of simulation gaming are commonly pseudo-medieval,
+involving swords and sorcery--spell-casting wizards, knights in armor,
+unicorns and dragons, demons and goblins.
+
+Urvile and his fellow gamers preferred their fantasies highly technological.
+They made use of a game known as "G.U.R.P.S.," the "Generic Universal Role
+Playing System," published by a company called Steve Jackson Games (SJG).
+
+"G.U.R.P.S." served as a framework for creating a wide variety of artificial
+fantasy worlds. Steve Jackson Games published a smorgasboard of books,
+full of detailed information and gaming hints, which were used to flesh-out
+many different fantastic backgrounds for the basic GURPS framework.
+Urvile made extensive use of two SJG books called GURPS High-Tech
+and GURPS Special Ops.
+
+In the artificial fantasy-world of GURPS Special Ops,
+players entered a modern fantasy of intrigue and international espionage.
+On beginning the game, players started small and powerless,
+perhaps as minor-league CIA agents or penny-ante arms dealers.
+But as players persisted through a series of game sessions
+(game sessions generally lasted for hours, over long,
+elaborate campaigns that might be pursued for months on end)
+then they would achieve new skills, new knowledge, new power.
+They would acquire and hone new abilities, such as marksmanship,
+karate, wiretapping, or Watergate burglary. They could also win
+various kinds of imaginary booty, like Berettas, or martini shakers,
+or fast cars with ejection seats and machine-guns under the headlights.
+
+As might be imagined from the complexity of these games,
+Urvile's gaming notes were very detailed and extensive.
+Urvile was a "dungeon-master," inventing scenarios
+for his fellow gamers, giant simulated adventure-puzzles
+for his friends to unravel. Urvile's game notes covered
+dozens of pages with all sorts of exotic lunacy, all about
+ninja raids on Libya and break-ins on encrypted Red Chinese supercomputers.
+His notes were written on scrap-paper and kept in loose-leaf binders.
+
+The handiest scrap paper around Urvile's college digs were the many pounds of
+BellSouth printouts and documents that he had snitched out of telco dumpsters.
+His notes were written on the back of misappropriated telco property.
+Worse yet, the gaming notes were chaotically interspersed with Urvile's
+hand-scrawled records involving ACTUAL COMPUTER INTRUSIONS that he
+had committed.
+
+Not only was it next to impossible to tell Urvile's fantasy game-notes
+from cyberspace "reality," but Urvile himself barely made this distinction.
+It's no exaggeration to say that to Urvile it was ALL a game. Urvile was
+very bright, highly imaginative, and quite careless of other people's notions
+of propriety. His connection to "reality" was not something to which he paid
+a great deal of attention.
+
+Hacking was a game for Urvile. It was an amusement he was carrying out,
+it was something he was doing for fun. And Urvile was an obsessive young man.
+He could no more stop hacking than he could stop in the middle of
+a jigsaw puzzle, or stop in the middle of reading a Stephen Donaldson
+fantasy trilogy. (The name "Urvile" came from a best-selling Donaldson novel.)
+
+Urvile's airy, bulletproof attitude seriously annoyed his interrogators.
+First of all, he didn't consider that he'd done anything wrong.
+There was scarcely a shred of honest remorse in him. On the contrary,
+he seemed privately convinced that his police interrogators were operating
+in a demented fantasy-world all their own. Urvile was too polite
+and well-behaved to say this straight-out, but his reactions were askew
+and disquieting.
+
+For instance, there was the business about LoD's ability
+to monitor phone-calls to the police and Secret Service.
+Urvile agreed that this was quite possible, and posed
+no big problem for LoD. In fact, he and his friends
+had kicked the idea around on the "Black Ice" board,
+much as they had discussed many other nifty notions,
+such as building personal flame-throwers and jury-rigging
+fistfulls of blasting-caps. They had hundreds of dial-up numbers
+for government agencies that they'd gotten through scanning Atlanta phones,
+or had pulled from raided VAX/VMS mainframe computers.
+
+Basically, they'd never gotten around to listening in on the cops
+because the idea wasn't interesting enough to bother with.
+Besides, if they'd been monitoring Secret Service phone calls,
+obviously they'd never have been caught in the first place. Right?
+
+The Secret Service was less than satisfied with this rapier-like hacker logic.
+
+Then there was the issue of crashing the phone system. No problem,
+Urvile admitted sunnily. Atlanta LoD could have shut down phone service
+all over Atlanta any time they liked. EVEN THE 911 SERVICE?
+Nothing special about that, Urvile explained patiently.
+Bring the switch to its knees, with say the UNIX "makedir" bug,
+and 911 goes down too as a matter of course. The 911 system
+wasn't very interesting, frankly. It might be tremendously
+interesting to cops (for odd reasons of their own), but as
+technical challenges went, the 911 service was yawnsville.
+
+So of course the Atlanta Three could crash service.
+They probably could have crashed service all over
+BellSouth territory, if they'd worked at it for a while.
+But Atlanta LoD weren't crashers. Only losers and rodents
+were crashers. LoD were ELITE.
+
+Urvile was privately convinced that sheer technical
+expertise could win him free of any kind of problem.
+As far as he was concerned, elite status in the digital
+underground had placed him permanently beyond the intellectual
+grasp of cops and straights. Urvile had a lot to learn.
+
+Of the three LoD stalwarts, Prophet was in the most direct trouble.
+Prophet was a UNIX programming expert who burrowed in and out
+of the Internet as a matter of course. He'd started his hacking
+career at around age 14, meddling with a UNIX mainframe system
+at the University of North Carolina.
+
+Prophet himself had written the handy Legion of Doom
+file "UNIX Use and Security From the Ground Up."
+UNIX (pronounced "you-nicks") is a powerful,
+flexible computer operating-system, for multi-user,
+multi-tasking computers. In 1969, when UNIX was created
+in Bell Labs, such computers were exclusive to large
+corporations and universities, but today UNIX is run
+on thousands of powerful home machines. UNIX was
+particularly well-suited to telecommunications programming,
+and had become a standard in the field. Naturally, UNIX
+also became a standard for the elite hacker and phone phreak.
+Lately, Prophet had not been so active as Leftist and Urvile,
+but Prophet was a recidivist. In 1986, when he was eighteen,
+Prophet had been convicted of "unauthorized access
+to a computer network" in North Carolina. He'd been
+discovered breaking into the Southern Bell Data Network,
+a UNIX-based internal telco network supposedly closed to the public.
+He'd gotten a typical hacker sentence: six months suspended,
+120 hours community service, and three years' probation.
+
+After that humiliating bust, Prophet had gotten rid of most of his
+tonnage of illicit phreak and hacker data, and had tried to go straight.
+He was, after all, still on probation. But by the autumn of 1988,
+the temptations of cyberspace had proved too much for young Prophet,
+and he was shoulder-to-shoulder with Urvile and Leftist into some
+of the hairiest systems around.
+
+In early September 1988, he'd broken into BellSouth's centralized
+automation system, AIMSX or "Advanced Information Management System."
+AIMSX was an internal business network for BellSouth, where telco
+employees stored electronic mail, databases, memos, and calendars,
+and did text processing. Since AIMSX did not have public dial-ups,
+it was considered utterly invisible to the public, and was not well-secured
+--it didn't even require passwords. Prophet abused an account known
+as "waa1," the personal account of an unsuspecting telco employee.
+Disguised as the owner of waa1, Prophet made about ten visits to AIMSX.
+
+Prophet did not damage or delete anything in the system.
+His presence in AIMSX was harmless and almost invisible.
+But he could not rest content with that.
+
+One particular piece of processed text on AIMSX was a telco document
+known as "Bell South Standard Practice 660-225-104SV Control Office
+Administration of Enhanced 911 Services for Special Services
+and Major Account Centers dated March 1988."
+
+Prophet had not been looking for this document. It was merely one
+among hundreds of similar documents with impenetrable titles.
+However, having blundered over it in the course of his illicit
+wanderings through AIMSX, he decided to take it with him as a trophy.
+It might prove very useful in some future boasting, bragging,
+and strutting session. So, some time in September 1988,
+Prophet ordered the AIMSX mainframe computer to copy this document
+(henceforth called simply called "the E911 Document") and to transfer
+this copy to his home computer.
+
+No one noticed that Prophet had done this. He had "stolen"
+the E911 Document in some sense, but notions of property
+in cyberspace can be tricky. BellSouth noticed nothing wrong,
+because BellSouth still had their original copy. They had not
+been "robbed" of the document itself. Many people were supposed
+to copy this document--specifically, people who worked for the
+nineteen BellSouth "special services and major account centers,"
+scattered throughout the Southeastern United States. That was
+what it was for, why it was present on a computer network
+in the first place: so that it could be copied and read--
+by telco employees. But now the data had been copied
+by someone who wasn't supposed to look at it.
+
+Prophet now had his trophy. But he further decided to store
+yet another copy of the E911 Document on another person's computer.
+This unwitting person was a computer enthusiast named Richard Andrews
+who lived near Joliet, Illinois. Richard Andrews was a UNIX programmer
+by trade, and ran a powerful UNIX board called "Jolnet," in the basement
+of his house.
+
+Prophet, using the handle "Robert Johnson," had obtained an account
+on Richard Andrews' computer. And there he stashed the E911 Document,
+by storing it in his own private section of Andrews' computer.
+
+Why did Prophet do this? If Prophet had eliminated the E911 Document
+from his own computer, and kept it hundreds of miles away, on another machine, under an
+alias, then he might have been fairly safe from discovery and prosecution--
+although his sneaky action had certainly put the unsuspecting Richard Andrews
+at risk.
+
+But, like most hackers, Prophet was a pack-rat for illicit data.
+When it came to the crunch, he could not bear to part from his trophy.
+When Prophet's place in Decatur, Georgia was raided in July 1989,
+there was the E911 Document, a smoking gun. And there was Prophet
+in the hands of the Secret Service, doing his best to "explain."
+
+Our story now takes us away from the Atlanta Three and their raids
+of the Summer of 1989. We must leave Atlanta Three "cooperating fully"
+with their numerous investigators. And all three of them did cooperate,
+as their Sentencing Memorandum from the US District Court of the
+Northern Division of Georgia explained--just before all three of them
+were sentenced to various federal prisons in November 1990.
+
+We must now catch up on the other aspects of the war on the Legion of Doom.
+The war on the Legion was a war on a network--in fact, a network of three
+networks, which intertwined and interrelated in a complex fashion.
+The Legion itself, with Atlanta LoD, and their hanger-on Fry Guy,
+were the first network. The second network was Phrack magazine,
+with its editors and contributors.
+
+The third network involved the electronic circle around a hacker
+known as "Terminus."
+
+The war against these hacker networks was carried out by
+a law enforcement network. Atlanta LoD and Fry Guy
+were pursued by USSS agents and federal prosecutors in Atlanta,
+Indiana, and Chicago. "Terminus" found himself pursued by USSS
+and federal prosecutors from Baltimore and Chicago. And the war
+against Phrack was almost entirely a Chicago operation.
+
+The investigation of Terminus involved a great deal of energy,
+mostly from the Chicago Task Force, but it was to be the least-known
+and least-publicized of the Crackdown operations. Terminus, who lived
+in Maryland, was a UNIX programmer and consultant, fairly well-known
+(under his given name) in the UNIX community, as an acknowledged expert
+on AT&T minicomputers. Terminus idolized AT&T, especially Bellcore,
+and longed for public recognition as a UNIX expert; his highest ambition
+was to work for Bell Labs.
+
+But Terminus had odd friends and a spotted history.
+Terminus had once been the subject of an admiring interview
+in Phrack (Volume II, Issue 14, Phile 2--dated May 1987).
+In this article, Phrack co-editor Taran King described
+"Terminus" as an electronics engineer, 5'9", brown-haired,
+born in 1959--at 28 years old, quite mature for a hacker.
+
+Terminus had once been sysop of a phreak/hack underground board
+called "MetroNet," which ran on an Apple II. Later he'd replaced
+"MetroNet" with an underground board called "MegaNet,"
+specializing in IBMs. In his younger days, Terminus had written
+one of the very first and most elegant code-scanning programs
+for the IBM-PC. This program had been widely distributed
+in the underground. Uncounted legions of PC-owning phreaks and
+hackers had used Terminus's scanner program to rip-off telco codes.
+This feat had not escaped the attention of telco security;
+it hardly could, since Terminus's earlier handle, "Terminal Technician,"
+was proudly written right on the program.
+
+When he became a full-time computer professional
+(specializing in telecommunications programming),
+he adopted the handle Terminus, meant to indicate that he
+had "reached the final point of being a proficient hacker."
+He'd moved up to the UNIX-based "Netsys" board on an AT&T computer,
+with four phone lines and an impressive 240 megs of storage.
+"Netsys" carried complete issues of Phrack, and Terminus was
+quite friendly with its publishers, Taran King and Knight Lightning.
+
+In the early 1980s, Terminus had been a regular on Plovernet,
+Pirate-80, Sherwood Forest and Shadowland, all well-known pirate boards,
+all heavily frequented by the Legion of Doom. As it happened, Terminus
+was never officially "in LoD," because he'd never been given the official
+LoD high-sign and back-slap by Legion maven Lex Luthor. Terminus had
+never physically met anyone from LoD. But that scarcely mattered much--
+the Atlanta Three themselves had never been officially vetted by Lex, either.
+
+As far as law enforcement was concerned, the issues were clear.
+Terminus was a full-time, adult computer professional
+with particular skills at AT&T software and hardware--
+but Terminus reeked of the Legion of Doom and the underground.
+
+On February 1, 1990--half a month after the Martin Luther King Day Crash--
+USSS agents Tim Foley from Chicago, and Jack Lewis from the Baltimore office,
+accompanied by AT&T security officer Jerry Dalton, travelled to Middle Town,
+Maryland. There they grilled Terminus in his home (to the stark terror of
+his wife and small children), and, in their customary fashion, hauled his
+computers out the door.
+
+The Netsys machine proved to contain a plethora of arcane UNIX software--
+proprietary source code formally owned by AT&T. Software such as:
+UNIX System Five Release 3.2; UNIX SV Release 3.1; UUCP communications
+software; KORN SHELL; RFS; IWB; WWB; DWB; the C++ programming language;
+PMON; TOOL CHEST; QUEST; DACT, and S FIND.
+
+In the long-established piratical tradition of the underground,
+Terminus had been trading this illicitly-copied software with
+a small circle of fellow UNIX programmers. Very unwisely,
+he had stored seven years of his electronic mail on his Netsys machine,
+which documented all the friendly arrangements he had made with
+his various colleagues.
+
+Terminus had not crashed the AT&T phone system on January 15.
+He was, however, blithely running a not-for-profit AT&T
+software-piracy ring. This was not an activity AT&T found amusing.
+AT&T security officer Jerry Dalton valued this "stolen" property
+at over three hundred thousand dollars.
+
+AT&T's entry into the tussle of free enterprise had been complicated
+by the new, vague groundrules of the information economy.
+Until the break-up of Ma Bell, AT&T was forbidden to sell
+computer hardware or software. Ma Bell was the phone company;
+Ma Bell was not allowed to use the enormous revenue from
+telephone utilities, in order to finance any entry into
+the computer market.
+
+AT&T nevertheless invented the UNIX operating system.
+And somehow AT&T managed to make UNIX a minor source of income.
+Weirdly, UNIX was not sold as computer software,
+but actually retailed under an obscure regulatory
+exemption allowing sales of surplus equipment and scrap.
+Any bolder attempt to promote or retail UNIX would have
+aroused angry legal opposition from computer companies.
+Instead, UNIX was licensed to universities, at modest rates,
+where the acids of academic freedom ate away steadily at AT&T's
+proprietary rights.
+
+Come the breakup, AT&T recognized that UNIX was a potential gold-mine.
+By now, large chunks of UNIX code had been created that were not AT&T's,
+and were being sold by others. An entire rival UNIX-based operating system
+had arisen in Berkeley, California (one of the world's great founts of
+ideological hackerdom). Today, "hackers" commonly consider "Berkeley UNIX"
+to be technically superior to AT&T's "System V UNIX," but AT&T has not
+allowed mere technical elegance to intrude on the real-world business
+of marketing proprietary software. AT&T has made its own code deliberately
+incompatible with other folks' UNIX, and has written code that it can prove
+is copyrightable, even if that code happens to be somewhat awkward--"kludgey."
+AT&T UNIX user licenses are serious business agreements, replete with very
+clear copyright statements and non-disclosure clauses.
+
+AT&T has not exactly kept the UNIX cat in the bag,
+but it kept a grip on its scruff with some success.
+By the rampant, explosive standards of software piracy,
+AT&T UNIX source code is heavily copyrighted, well-guarded,
+well-licensed. UNIX was traditionally run only on
+mainframe machines, owned by large groups of suit-and-tie
+professionals, rather than on bedroom machines where
+people can get up to easy mischief.
+
+And AT&T UNIX source code is serious high-level programming.
+The number of skilled UNIX programmers with any actual motive
+to swipe UNIX source code is small. It's tiny, compared to
+the tens of thousands prepared to rip-off, say, entertaining
+PC games like "Leisure Suit Larry."
+
+But by 1989, the warez-d00d underground, in the persons of Terminus
+and his friends, was gnawing at AT&T UNIX. And the property in question
+was not sold for twenty bucks over the counter at the local branch of
+Babbage's or Egghead's; this was massive, sophisticated, multi-line,
+multi-author corporate code worth tens of thousands of dollars.
+
+It must be recognized at this point that Terminus's purported ring of UNIX
+software pirates had not actually made any money from their suspected crimes.
+The $300,000 dollar figure bandied about for the contents of Terminus's
+computer did not mean that Terminus was in actual illicit possession
+of three hundred thousand of AT&T's dollars. Terminus was shipping
+software back and forth, privately, person to person, for free.
+He was not making a commercial business of piracy. He hadn't
+asked for money; he didn't take money. He lived quite modestly.
+
+AT&T employees--as well as freelance UNIX consultants, like Terminus--
+commonly worked with "proprietary" AT&T software, both in the office
+and at home on their private machines. AT&T rarely sent security officers
+out to comb the hard disks of its consultants. Cheap freelance UNIX
+contractors were quite useful to AT&T; they didn't have health insurance
+or retirement programs, much less union membership in the Communication
+Workers of America. They were humble digital drudges, wandering with mop
+and bucket through the Great Technological Temple of AT&T; but when the
+Secret Service arrived at their homes, it seemed they were eating with
+company silverware and sleeping on company sheets! Outrageously, they
+behaved as if the things they worked with every day belonged to them!
+
+And these were no mere hacker teenagers with their hands full
+of trash-paper and their noses pressed to the corporate windowpane.
+These guys were UNIX wizards, not only carrying AT&T data in their
+machines and their heads, but eagerly networking about it,
+over machines that were far more powerful than anything previously
+imagined in private hands. How do you keep people disposable,
+yet assure their awestruck respect for your property? It was a dilemma.
+
+Much UNIX code was public-domain, available for free. Much "proprietary"
+UNIX code had been extensively re-written, perhaps altered so much that it
+became an entirely new product--or perhaps not. Intellectual property rights
+for software developers were, and are, extraordinarily complex and confused.
+And software "piracy," like the private copying of videos, is one of the most
+widely practiced "crimes" in the world today.
+
+The USSS were not experts in UNIX or familiar with the customs of its use.
+The United States Secret Service, considered as a body, did not have one single
+person in it who could program in a UNIX environment--no, not even one.
+The Secret Service WERE making extensive use of expert help, but the "experts"
+they had chosen were AT&T and Bellcore security officials, the very victims of
+the purported crimes under investigation, the very people whose interest in
+AT&T's "proprietary" software was most pronounced.
+
+On February 6, 1990, Terminus was arrested by Agent Lewis.
+Eventually, Terminus would be sent to prison for his illicit
+use of a piece of AT&T software.
+
+The issue of pirated AT&T software would bubble along in the background
+during the war on the Legion of Doom. Some half-dozen of Terminus's on-line
+acquaintances, including people in Illinois, Texas and California,
+were grilled by the Secret Service in connection with the illicit
+copying of software. Except for Terminus, however, none were charged
+with a crime. None of them shared his peculiar prominence in the
+hacker underground.
+
+But that did not mean that these people would, or could,
+stay out of trouble. The transferral of illicit data in
+cyberspace is hazy and ill-defined business, with paradoxical
+dangers for everyone concerned: hackers, signal carriers,
+board owners, cops, prosecutors, even random passers-by.
+Sometimes, well-meant attempts to avert trouble
+or punish wrongdoing bring more trouble than
+would simple ignorance, indifference or impropriety.
+
+Terminus's "Netsys" board was not a common-or-garden
+bulletin board system, though it had most of the usual
+functions of a board. Netsys was not a stand-alone machine,
+but part of the globe-spanning "UUCP" cooperative network.
+The UUCP network uses a set of Unix software programs called
+"Unix-to-Unix Copy," which allows Unix systems to throw data to
+one another at high speed through the public telephone network.
+UUCP is a radically decentralized, not-for-profit network of UNIX computers.
+There are tens of thousands of these UNIX machines. Some are small,
+but many are powerful and also link to other networks. UUCP has
+certain arcane links to major networks such as JANET, EasyNet, BITNET,
+JUNET, VNET, DASnet, PeaceNet and FidoNet, as well as the gigantic Internet.
+(The so-called "Internet" is not actually a network itself, but rather an
+"internetwork" connections standard that allows several globe-spanning
+computer networks to communicate with one another. Readers fascinated
+by the weird and intricate tangles of modern computer networks may enjoy
+John S. Quarterman's authoritative 719-page explication, The Matrix,
+Digital Press, 1990.)
+
+A skilled user of Terminus' UNIX machine could send and receive
+electronic mail from almost any major computer network in the world.
+Netsys was not called a "board" per se, but rather a "node."
+"Nodes" were larger, faster, and more sophisticated than mere "boards,"
+and for hackers, to hang out on internationally-connected "nodes"
+was quite the step up from merely hanging out on local "boards."
+
+Terminus's Netsys node in Maryland had a number of direct
+links to other, similar UUCP nodes, run by people who shared his
+interests and at least something of his free-wheeling attitude.
+One of these nodes was Jolnet, owned by Richard Andrews, who,
+like Terminus, was an independent UNIX consultant.
+Jolnet also ran UNIX, and could be contacted at high speed
+by mainframe machines from all over the world. Jolnet was
+quite a sophisticated piece of work, technically speaking,
+but it was still run by an individual, as a private,
+not-for-profit hobby. Jolnet was mostly used by other
+UNIX programmers--for mail, storage, and access to networks.
+Jolnet supplied access network access to about two hundred people,
+as well as a local junior college.
+
+Among its various features and services, Jolnet also carried
+Phrack magazine.
+
+For reasons of his own, Richard Andrews had become suspicious
+of a new user called "Robert Johnson." Richard Andrews
+took it upon himself to have a look at what "Robert Johnson"
+was storing in Jolnet. And Andrews found the E911 Document.
+
+"Robert Johnson" was the Prophet from the Legion of Doom,
+and the E911 Document was illicitly copied data from Prophet's
+raid on the BellSouth computers.
+
+The E911 Document, a particularly illicit piece of digital property,
+was about to resume its long, complex, and disastrous career.
+
+It struck Andrews as fishy that someone not a telephone employee
+should have a document referring to the "Enhanced 911 System."
+Besides, the document itself bore an obvious warning.
+
+"WARNING: NOT FOR USE OR DISCLOSURE OUTSIDE BELLSOUTH
+OR ANY OF ITS SUBSIDIARIES EXCEPT UNDER WRITTEN AGREEMENT."
+
+These standard nondisclosure tags are often appended to all sorts
+of corporate material. Telcos as a species are particularly notorious
+for stamping most everything in sight as "not for use or disclosure."
+Still, this particular piece of data was about the 911 System.
+That sounded bad to Rich Andrews.
+
+Andrews was not prepared to ignore this sort of trouble.
+He thought it would be wise to pass the document along
+to a friend and acquaintance on the UNIX network, for consultation.
+So, around September 1988, Andrews sent yet another copy of the
+E911 Document electronically to an AT&T employee, one Charles Boykin,
+who ran a UNIX-based node called "attctc" in Dallas, Texas.
+
+"Attctc" was the property of AT&T, and was run from AT&T's
+Customer Technology Center in Dallas, hence the name "attctc."
+"Attctc" was better-known as "Killer," the name of the machine
+that the system was running on. "Killer" was a hefty, powerful,
+AT&T 3B2 500 model, a multi-user, multi-tasking UNIX platform
+with 32 meg of memory and a mind-boggling 3.2 Gigabytes of storage.
+When Killer had first arrived in Texas, in 1985, the 3B2 had been
+one of AT&T's great white hopes for going head-to-head with IBM
+for the corporate computer-hardware market. "Killer" had been shipped
+to the Customer Technology Center in the Dallas Infomart, essentially
+a high-technology mall, and there it sat, a demonstration model.
+
+Charles Boykin, a veteran AT&T hardware and digital communications expert,
+was a local technical backup man for the AT&T 3B2 system. As a display model
+in the Infomart mall, "Killer" had little to do, and it seemed a shame
+to waste the system's capacity. So Boykin ingeniously wrote some UNIX
+bulletin-board software for "Killer," and plugged the machine in to the
+local phone network. "Killer's" debut in late 1985 made it the first
+publicly available UNIX site in the state of Texas. Anyone who wanted to
+play was welcome.
+
+The machine immediately attracted an electronic community.
+It joined the UUCP network, and offered network links
+to over eighty other computer sites, all of which became dependent
+on Killer for their links to the greater world of cyberspace.
+And it wasn't just for the big guys; personal computer users
+also stored freeware programs for the Amiga, the Apple,
+the IBM and the Macintosh on Killer's vast 3,200 meg archives.
+At one time, Killer had the largest library of public-domain
+Macintosh software in Texas.
+
+Eventually, Killer attracted about 1,500 users,
+all busily communicating, uploading and downloading,
+getting mail, gossipping, and linking to arcane
+and distant networks.
+
+Boykin received no pay for running Killer. He considered
+it good publicity for the AT&T 3B2 system (whose sales were
+somewhat less than stellar), but he also simply enjoyed
+the vibrant community his skill had created. He gave away
+the bulletin-board UNIX software he had written, free of charge.
+
+In the UNIX programming community, Charlie Boykin had the
+reputation of a warm, open-hearted, level-headed kind of guy.
+In 1989, a group of Texan UNIX professionals voted Boykin
+"System Administrator of the Year." He was considered
+a fellow you could trust for good advice.
+
+In September 1988, without warning, the E911 Document
+came plunging into Boykin's life, forwarded by Richard Andrews.
+Boykin immediately recognized that the Document was hot property.
+He was not a voice-communications man, and knew little about
+the ins and outs of the Baby Bells, but he certainly knew what
+the 911 System was, and he was angry to see confidential data
+about it in the hands of a nogoodnik. This was clearly a
+matter for telco security. So, on September 21, 1988, Boykin
+made yet ANOTHER copy of the E911 Document and passed this
+one along to a professional acquaintance of his, one Jerome Dalton,
+from AT&T Corporate Information Security. Jerry Dalton was the
+very fellow who would later raid Terminus's house.
+
+From AT&T's security division, the E911 Document went to Bellcore.
+
+Bellcore (or BELL COmmunications REsearch) had once been the central
+laboratory of the Bell System. Bell Labs employees had invented
+the UNIX operating system. Now Bellcore was a quasi-independent,
+jointly owned company that acted as the research arm for all seven
+of the Baby Bell RBOCs. Bellcore was in a good position to co-ordinate
+security technology and consultation for the RBOCs, and the gentleman in
+charge of this effort was Henry M. Kluepfel, a veteran of the Bell System
+who had worked there for twenty-four years.
+
+On October 13, 1988, Dalton passed the E911 Document to Henry Kluepfel.
+Kluepfel, a veteran expert witness in telecommunications fraud and
+computer-fraud cases, had certainly seen worse trouble than this.
+He recognized the document for what it was: a trophy from a hacker break-in.
+
+However, whatever harm had been done in the intrusion was presumably old news.
+At this point there seemed little to be done. Kluepfel made a careful note
+of the circumstances and shelved the problem for the time being.
+
+Whole months passed.
+
+February 1989 arrived. The Atlanta Three were living it up
+in Bell South's switches, and had not yet met their comeuppance.
+The Legion was thriving. So was Phrack magazine.
+A good six months had passed since Prophet's AIMSX break-in.
+Prophet, as hackers will, grew weary of sitting on his laurels.
+"Knight Lightning" and "Taran King," the editors of Phrack,
+were always begging Prophet for material they could publish.
+Prophet decided that the heat must be off by this time,
+and that he could safely brag, boast, and strut.
+
+So he sent a copy of the E911 Document--yet another one--
+from Rich Andrews' Jolnet machine to Knight Lightning's
+BITnet account at the University of Missouri.
+Let's review the fate of the document so far.
+
+0. The original E911 Document. This in the AIMSX system
+on a mainframe computer in Atlanta, available to hundreds of people,
+but all of them, presumably, BellSouth employees. An unknown number
+of them may have their own copies of this document, but they are all
+professionals and all trusted by the phone company.
+
+1. Prophet's illicit copy, at home on his own computer in Decatur, Georgia.
+
+2. Prophet's back-up copy, stored on Rich Andrew's Jolnet machine
+ in the basement of Rich Andrews' house near Joliet Illinois.
+
+3. Charles Boykin's copy on "Killer" in Dallas, Texas,
+ sent by Rich Andrews from Joliet.
+
+4. Jerry Dalton's copy at AT&T Corporate Information Security in New Jersey,
+ sent from Charles Boykin in Dallas.
+
+5. Henry Kluepfel's copy at Bellcore security headquarters in New Jersey,
+ sent by Dalton.
+6. Knight Lightning's copy, sent by Prophet from Rich Andrews' machine,
+ and now in Columbia, Missouri.
+
+We can see that the "security" situation of this proprietary document,
+once dug out of AIMSX, swiftly became bizarre. Without any money
+changing hands, without any particular special effort, this data
+had been reproduced at least six times and had spread itself all over
+the continent. By far the worst, however, was yet to come.
+
+In February 1989, Prophet and Knight Lightning bargained electronically
+over the fate of this trophy. Prophet wanted to boast, but, at the same time,
+scarcely wanted to be caught.
+
+For his part, Knight Lightning was eager to publish as much of the document
+as he could manage. Knight Lightning was a fledgling political-science major
+with a particular interest in freedom-of-information issues. He would gladly
+publish most anything that would reflect glory on the prowess of the
+underground and embarrass the telcos. However, Knight Lightning himself
+had contacts in telco security, and sometimes consulted them on material
+he'd received that might be too dicey for publication.
+
+Prophet and Knight Lightning decided to edit the E911 Document
+so as to delete most of its identifying traits. First of all,
+its large "NOT FOR USE OR DISCLOSURE" warning had to go.
+Then there were other matters. For instance, it listed
+the office telephone numbers of several BellSouth 911
+specialists in Florida. If these phone numbers were
+published in Phrack, the BellSouth employees involved
+would very likely be hassled by phone phreaks,
+which would anger BellSouth no end, and pose a
+definite operational hazard for both Prophet and Phrack.
+
+So Knight Lightning cut the Document almost in half,
+removing the phone numbers and some of the touchier
+and more specific information. He passed it back
+electronically to Prophet; Prophet was still nervous,
+so Knight Lightning cut a bit more. They finally agreed
+that it was ready to go, and that it would be published
+in Phrack under the pseudonym, "The Eavesdropper."
+
+And this was done on February 25, 1989.
+
+The twenty-fourth issue of Phrack featured a chatty interview
+with co-ed phone-phreak "Chanda Leir," three articles on BITNET
+and its links to other computer networks, an article on 800 and 900
+numbers by "Unknown User," "VaxCat's" article on telco basics
+(slyly entitled "Lifting Ma Bell's Veil of Secrecy,)" and
+the usual "Phrack World News."
+
+The News section, with painful irony, featured an extended account
+of the sentencing of "Shadowhawk," an eighteen-year-old Chicago hacker
+who had just been put in federal prison by William J. Cook himself.
+
+And then there were the two articles by "The Eavesdropper."
+The first was the edited E911 Document, now titled
+"Control Office Administration Of Enhanced 911 Services
+for Special Services and Major Account Centers."
+Eavesdropper's second article was a glossary of terms
+explaining the blizzard of telco acronyms and buzzwords
+in the E911 Document.
+
+The hapless document was now distributed, in the usual Phrack routine,
+to a good one hundred and fifty sites. Not a hundred and fifty PEOPLE,
+mind you--a hundred and fifty SITES, some of these sites linked to UNIX
+nodes or bulletin board systems, which themselves had readerships of tens,
+dozens, even hundreds of people.
+
+This was February 1989. Nothing happened immediately.
+Summer came, and the Atlanta crew were raided by the Secret Service.
+Fry Guy was apprehended. Still nothing whatever happened to Phrack.
+Six more issues of Phrack came out, 30 in all, more or less on
+a monthly schedule. Knight Lightning and co-editor Taran King
+went untouched.
+
+Phrack tended to duck and cover whenever the heat came down.
+During the summer busts of 1987--(hacker busts tended to cluster in summer,
+perhaps because hackers were easier to find at home than in college)--
+Phrack had ceased publication for several months, and laid low.
+Several LoD hangers-on had been arrested, but nothing had happened
+to the Phrack crew, the premiere gossips of the underground.
+In 1988, Phrack had been taken over by a new editor,
+"Crimson Death," a raucous youngster with a taste for anarchy files.
+1989, however, looked like a bounty year for the underground.
+Knight Lightning and his co-editor Taran King took up the reins again,
+and Phrack flourished throughout 1989. Atlanta LoD went down hard in
+the summer of 1989, but Phrack rolled merrily on. Prophet's E911 Document
+seemed unlikely to cause Phrack any trouble. By January 1990,
+it had been available in Phrack for almost a year. Kluepfel and Dalton,
+officers of Bellcore and AT&T security, had possessed the document
+for sixteen months--in fact, they'd had it even before Knight Lightning
+himself, and had done nothing in particular to stop its distribution.
+They hadn't even told Rich Andrews or Charles Boykin to erase the copies
+from their UNIX nodes, Jolnet and Killer.
+
+But then came the monster Martin Luther King Day Crash of January 15, 1990.
+
+A flat three days later, on January 18, four agents showed up
+at Knight Lightning's fraternity house. One was Timothy Foley,
+the second Barbara Golden, both of them Secret Service agents
+from the Chicago office. Also along was a University of Missouri
+security officer, and Reed Newlin, a security man from Southwestern Bell,
+the RBOC having jurisdiction over Missouri.
+
+Foley accused Knight Lightning of causing the nationwide crash
+of the phone system.
+
+Knight Lightning was aghast at this allegation. On the face of it,
+the suspicion was not entirely implausible--though Knight Lightning
+knew that he himself hadn't done it. Plenty of hot-dog hackers
+had bragged that they could crash the phone system, however.
+"Shadowhawk," for instance, the Chicago hacker whom William Cook
+had recently put in jail, had several times boasted on boards
+that he could "shut down AT&T's public switched network."
+
+And now this event, or something that looked just like it,
+had actually taken place. The Crash had lit a fire under
+the Chicago Task Force. And the former fence-sitters at
+Bellcore and AT&T were now ready to roll. The consensus
+among telco security--already horrified by the skill of
+the BellSouth intruders --was that the digital underground
+was out of hand. LoD and Phrack must go. And in publishing
+Prophet's E911 Document, Phrack had provided law enforcement
+with what appeared to be a powerful legal weapon.
+
+Foley confronted Knight Lightning about the E911 Document.
+
+Knight Lightning was cowed. He immediately began "cooperating fully"
+in the usual tradition of the digital underground.
+
+He gave Foley a complete run of Phrack, printed out in a set
+of three-ring binders. He handed over his electronic mailing list
+of Phrack subscribers. Knight Lightning was grilled for four hours
+by Foley and his cohorts. Knight Lightning admitted that Prophet
+had passed him the E911 Document, and he admitted that he had known
+it was stolen booty from a hacker raid on a telephone company.
+Knight Lightning signed a statement to this effect, and agreed,
+in writing, to cooperate with investigators.
+
+Next day--January 19, 1990, a Friday --the Secret Service returned
+with a search warrant, and thoroughly searched Knight Lightning's
+upstairs room in the fraternity house. They took all his floppy disks,
+though, interestingly, they left Knight Lightning in possession
+of both his computer and his modem. (The computer had no hard disk,
+and in Foley's judgement was not a store of evidence.) But this was a
+very minor bright spot among Knight Lightning's rapidly multiplying troubles.
+By this time, Knight Lightning was in plenty of hot water, not only with
+federal police, prosecutors, telco investigators, and university security,
+but with the elders of his own campus fraternity, who were outraged
+to think that they had been unwittingly harboring a federal computer-criminal.
+
+On Monday, Knight Lightning was summoned to Chicago, where he was
+further grilled by Foley and USSS veteran agent Barbara Golden, this time
+with an attorney present. And on Tuesday, he was formally indicted
+by a federal grand jury.
+
+The trial of Knight Lightning, which occurred on July 24-27, 1990,
+was the crucial show-trial of the Hacker Crackdown. We will examine
+the trial at some length in Part Four of this book.
+
+In the meantime, we must continue our dogged pursuit of the E911 Document.
+
+It must have been clear by January 1990 that the E911 Document,
+in the form Phrack had published it back in February 1989,
+had gone off at the speed of light in at least a hundred
+and fifty different directions. To attempt to put this
+electronic genie back in the bottle was flatly impossible.
+
+And yet, the E911 Document was STILL stolen property,
+formally and legally speaking. Any electronic transference
+of this document, by anyone unauthorized to have it,
+could be interpreted as an act of wire fraud. Interstate
+transfer of stolen property, including electronic property,
+was a federal crime.
+
+The Chicago Computer Fraud and Abuse Task Force had been assured
+that the E911 Document was worth a hefty sum of money. In fact,
+they had a precise estimate of its worth from BellSouth security personnel:
+$79,449. A sum of this scale seemed to warrant vigorous prosecution.
+Even if the damage could not be undone, at least this large sum
+offered a good legal pretext for stern punishment of the thieves.
+It seemed likely to impress judges and juries. And it could be used
+in court to mop up the Legion of Doom.
+
+The Atlanta crowd was already in the bag, by the time
+the Chicago Task Force had gotten around to Phrack.
+But the Legion was a hydra-headed thing. In late 89,
+a brand-new Legion of Doom board, "Phoenix Project,"
+had gone up in Austin, Texas. Phoenix Project was sysoped
+by no less a man than the Mentor himself, ably assisted by
+University of Texas student and hardened Doomster "Erik Bloodaxe."
+
+As we have seen from his Phrack manifesto, the Mentor was a hacker
+zealot who regarded computer intrusion as something close to a moral duty.
+Phoenix Project was an ambitious effort, intended to revive the digital
+underground to what Mentor considered the full flower of the early 80s.
+The Phoenix board would also boldly bring elite hackers face-to-face
+with the telco "opposition." On "Phoenix," America's cleverest hackers
+would supposedly shame the telco squareheads out of their stick-in-the-mud
+attitudes, and perhaps convince them that the Legion of Doom elite were really
+an all-right crew. The premiere of "Phoenix Project" was heavily trumpeted
+by Phrack,and "Phoenix Project" carried a complete run of Phrack issues,
+including the E911 Document as Phrack had published it.
+
+Phoenix Project was only one of many--possibly hundreds--of nodes and boards
+all over America that were in guilty possession of the E911 Document.
+But Phoenix was an outright, unashamed Legion of Doom board.
+Under Mentor's guidance, it was flaunting itself in the face
+of telco security personnel. Worse yet, it was actively trying
+to WIN THEM OVER as sympathizers for the digital underground elite.
+"Phoenix" had no cards or codes on it. Its hacker elite considered
+Phoenix at least technically legal. But Phoenix was a corrupting influence,
+where hacker anarchy was eating away like digital acid at the underbelly
+of corporate propriety.
+
+The Chicago Computer Fraud and Abuse Task Force now prepared
+to descend upon Austin, Texas.
+
+Oddly, not one but TWO trails of the Task Force's investigation led
+toward Austin. The city of Austin, like Atlanta, had made itself
+a bulwark of the Sunbelt's Information Age, with a strong university
+research presence, and a number of cutting-edge electronics companies,
+including Motorola, Dell, CompuAdd, IBM, Sematech and MCC.
+
+Where computing machinery went, hackers generally followed.
+Austin boasted not only "Phoenix Project," currently LoD's
+most flagrant underground board, but a number of UNIX nodes.
+
+One of these nodes was "Elephant," run by a UNIX consultant
+named Robert Izenberg. Izenberg, in search of a relaxed Southern
+lifestyle and a lowered cost-of-living, had recently migrated
+to Austin from New Jersey. In New Jersey, Izenberg had worked
+for an independent contracting company, programming UNIX code for
+AT&T itself. "Terminus" had been a frequent user on Izenberg's
+privately owned Elephant node.
+
+Having interviewed Terminus and examined the records on Netsys,
+the Chicago Task Force were now convinced that they had discovered
+an underground gang of UNIX software pirates, who were demonstrably
+guilty of interstate trafficking in illicitly copied AT&T source code.
+Izenberg was swept into the dragnet around Terminus, the self-proclaimed
+ultimate UNIX hacker.
+
+Izenberg, in Austin, had settled down into a UNIX job
+with a Texan branch of IBM. Izenberg was no longer
+working as a contractor for AT&T, but he had friends
+in New Jersey, and he still logged on to AT&T UNIX
+computers back in New Jersey, more or less whenever
+it pleased him. Izenberg's activities appeared highly
+suspicious to the Task Force. Izenberg might well be
+breaking into AT&T computers, swiping AT&T software,
+and passing it to Terminus and other possible confederates,
+through the UNIX node network. And this data was worth,
+not merely $79,499, but hundreds of thousands of dollars!
+
+On February 21, 1990, Robert Izenberg arrived home
+from work at IBM to find that all the computers
+had mysteriously vanished from his Austin apartment.
+Naturally he assumed that he had been robbed.
+His "Elephant" node, his other machines, his notebooks,
+his disks, his tapes, all gone! However, nothing much
+else seemed disturbed--the place had not been ransacked.
+The puzzle becaming much stranger some five minutes later.
+Austin U. S. Secret Service Agent Al Soliz, accompanied by
+University of Texas campus-security officer Larry Coutorie
+and the ubiquitous Tim Foley, made their appearance at Izenberg's door.
+They were in plain clothes: slacks, polo shirts. They came in,
+and Tim Foley accused Izenberg of belonging to the Legion of Doom.
+
+Izenberg told them that he had never heard of the "Legion of Doom."
+And what about a certain stolen E911 Document, that posed a direct
+threat to the police emergency lines? Izenberg claimed that he'd
+never heard of that, either.
+
+His interrogators found this difficult to believe.
+Didn't he know Terminus?
+
+Who?
+
+They gave him Terminus's real name. Oh yes, said Izenberg.
+He knew THAT guy all right--he was leading discussions
+on the Internet about AT&T computers, especially the AT&T 3B2.
+
+AT&T had thrust this machine into the marketplace,
+but, like many of AT&T's ambitious attempts to enter
+the computing arena, the 3B2 project had something less
+than a glittering success. Izenberg himself had been
+a contractor for the division of AT&T that supported the 3B2.
+The entire division had been shut down.
+
+Nowadays, the cheapest and quickest way to get help with this
+fractious piece of machinery was to join one of Terminus's
+discussion groups on the Internet, where friendly and knowledgeable
+hackers would help you for free. Naturally the remarks within this
+group were less than flattering about the Death Star. . .was
+THAT the problem?
+
+Foley told Izenberg that Terminus had been acquiring hot software
+through his, Izenberg's, machine.
+
+Izenberg shrugged this off. A good eight megabytes of data flowed
+through his UUCP site every day. UUCP nodes spewed data like fire hoses.
+Elephant had been directly linked to Netsys--not surprising, since Terminus
+was a 3B2 expert and Izenberg had been a 3B2 contractor.
+Izenberg was also linked to "attctc" and the University of Texas.
+Terminus was a well-known UNIX expert, and might have been up to
+all manner of hijinks on Elephant. Nothing Izenberg could do about that.
+That was physically impossible. Needle in a haystack.
+
+In a four-hour grilling, Foley urged Izenberg to come clean
+and admit that he was in conspiracy with Terminus,
+and a member of the Legion of Doom.
+
+Izenberg denied this. He was no weirdo teenage hacker--
+he was thirty-two years old, and didn't even have a "handle."
+Izenberg was a former TV technician and electronics specialist
+who had drifted into UNIX consulting as a full-grown adult.
+Izenberg had never met Terminus, physically. He'd once bought
+a cheap high-speed modem from him, though.
+
+Foley told him that this modem (a Telenet T2500 which ran at 19.2 kilobaud,
+and which had just gone out Izenberg's door in Secret Service custody)
+was likely hot property. Izenberg was taken aback to hear this; but then
+again, most of Izenberg's equipment, like that of most freelance professionals
+in the industry, was discounted, passed hand-to-hand through various kinds
+of barter and gray-market. There was no proof that the modem was stolen,
+and even if it were, Izenberg hardly saw how that gave them the right
+to take every electronic item in his house.
+
+Still, if the United States Secret Service figured they needed
+his computer for national security reasons--or whatever--
+then Izenberg would not kick. He figured he would somehow
+make the sacrifice of his twenty thousand dollars' worth
+of professional equipment, in the spirit of full cooperation
+and good citizenship.
+
+Robert Izenberg was not arrested. Izenberg was not charged with any crime.
+His UUCP node--full of some 140 megabytes of the files, mail, and data
+of himself and his dozen or so entirely innocent users--went out the door
+as "evidence." Along with the disks and tapes, Izenberg had lost about
+800 megabytes of data.
+
+Six months would pass before Izenberg decided to phone the Secret Service
+and ask how the case was going. That was the first time that Robert Izenberg
+would ever hear the name of William Cook. As of January 1992, a full
+two years after the seizure, Izenberg, still not charged with any crime,
+would be struggling through the morass of the courts, in hope of recovering
+his thousands of dollars' worth of seized equipment.
+
+In the meantime, the Izenberg case received absolutely no press coverage.
+The Secret Service had walked into an Austin home, removed a UNIX bulletin-
+board system, and met with no operational difficulties whatsoever.
+
+Except that word of a crackdown had percolated through the Legion of Doom.
+"The Mentor" voluntarily shut down "The Phoenix Project." It seemed a pity,
+especially as telco security employees had, in fact, shown up on Phoenix,
+just as he had hoped--along with the usual motley crowd of LoD heavies,
+hangers-on, phreaks, hackers and wannabes. There was "Sandy" Sandquist from
+US SPRINT security, and some guy named Henry Kluepfel, from Bellcore itself!
+Kluepfel had been trading friendly banter with hackers on Phoenix since
+January 30th (two weeks after the Martin Luther King Day Crash).
+The presence of such a stellar telco official seemed quite the coup
+for Phoenix Project.
+
+Still, Mentor could judge the climate. Atlanta in ruins,
+Phrack in deep trouble, something weird going on with UNIX nodes--
+discretion was advisable. Phoenix Project went off-line.
+
+Kluepfel, of course, had been monitoring this LoD bulletin
+board for his own purposes--and those of the Chicago unit.
+As far back as June 1987, Kluepfel had logged on to a Texas
+underground board called "Phreak Klass 2600." There he'd
+discovered an Chicago youngster named "Shadowhawk,"
+strutting and boasting about rifling AT&T computer files,
+and bragging of his ambitions to riddle AT&T's Bellcore
+computers with trojan horse programs. Kluepfel had passed
+the news to Cook in Chicago, Shadowhawk's computers
+had gone out the door in Secret Service custody,
+and Shadowhawk himself had gone to jail.
+
+Now it was Phoenix Project's turn. Phoenix Project postured
+about "legality" and "merely intellectual interest," but it reeked
+of the underground. It had Phrack on it. It had the E911 Document.
+It had a lot of dicey talk about breaking into systems, including some
+bold and reckless stuff about a supposed "decryption service" that Mentor
+and friends were planning to run, to help crack encrypted passwords off
+of hacked systems.
+
+Mentor was an adult. There was a bulletin board at his place of work,
+as well. Kleupfel logged onto this board, too, and discovered it to be
+called "Illuminati." It was run by some company called Steve Jackson Games.
+
+On March 1, 1990, the Austin crackdown went into high gear.
+
+On the morning of March 1--a Thursday--21-year-old University of Texas
+student "Erik Bloodaxe," co-sysop of Phoenix Project and an avowed member
+of the Legion of Doom, was wakened by a police revolver levelled at his head.
+
+Bloodaxe watched, jittery, as Secret Service agents
+appropriated his 300 baud terminal and, rifling his files,
+discovered his treasured source-code for Robert Morris's
+notorious Internet Worm. But Bloodaxe, a wily operator,
+had suspected that something of the like might be coming.
+All his best equipment had been hidden away elsewhere.
+The raiders took everything electronic, however,
+including his telephone. They were stymied by his
+hefty arcade-style Pac-Man game, and left it in place,
+as it was simply too heavy to move.
+
+Bloodaxe was not arrested. He was not charged with any crime.
+A good two years later, the police still had what they had
+taken from him, however.
+
+The Mentor was less wary. The dawn raid rousted him and his wife
+from bed in their underwear, and six Secret Service agents,
+accompanied by an Austin policeman and Henry Kluepfel himself,
+made a rich haul. Off went the works, into the agents' white
+Chevrolet minivan: an IBM PC-AT clone with 4 meg of RAM and
+a 120-meg hard disk; a Hewlett-Packard LaserJet II printer;
+a completely legitimate and highly expensive SCO-Xenix 286
+operating system; Pagemaker disks and documentation;
+and the Microsoft Word word-processing program. Mentor's wife
+had her incomplete academic thesis stored on the hard-disk;
+that went, too, and so did the couple's telephone. As of two years later,
+all this property remained in police custody.
+
+Mentor remained under guard in his apartment as agents prepared
+to raid Steve Jackson Games. The fact that this was a business
+headquarters and not a private residence did not deter the agents.
+It was still very early; no one was at work yet. The agents prepared
+to break down the door, but Mentor, eavesdropping on the Secret Service
+walkie-talkie traffic, begged them not to do it, and offered his key
+to the building.
+
+The exact details of the next events are unclear. The agents
+would not let anyone else into the building. Their search warrant,
+when produced, was unsigned. Apparently they breakfasted from the local
+"Whataburger," as the litter from hamburgers was later found inside.
+They also extensively sampled a bag of jellybeans kept by an SJG employee.
+Someone tore a "Dukakis for President" sticker from the wall.
+
+SJG employees, diligently showing up for the day's work, were met
+at the door and briefly questioned by U.S. Secret Service agents.
+The employees watched in astonishment as agents wielding crowbars
+and screwdrivers emerged with captive machines. They attacked
+outdoor storage units with boltcutters. The agents wore
+blue nylon windbreakers with "SECRET SERVICE" stencilled
+across the back, with running-shoes and jeans.
+
+Jackson's company lost three computers, several hard-disks,
+hundred of floppy disks, two monitors, three modems,
+a laser printer, various powercords, cables, and adapters
+(and, oddly, a small bag of screws, bolts and nuts).
+The seizure of Illuminati BBS deprived SJG of all the programs,
+text files, and private e-mail on the board. The loss of two other
+SJG computers was a severe blow as well, since it caused the loss
+of electronically stored contracts, financial projections,
+address directories, mailing lists, personnel files,
+business correspondence, and, not least, the drafts
+of forthcoming games and gaming books.
+
+No one at Steve Jackson Games was arrested. No one was accused
+of any crime. No charges were filed. Everything appropriated
+was officially kept as "evidence" of crimes never specified.
+
+After the Phrack show-trial, the Steve Jackson Games scandal
+was the most bizarre and aggravating incident of the Hacker
+Crackdown of 1990. This raid by the Chicago Task Force
+on a science-fiction gaming publisher was to rouse a
+swarming host of civil liberties issues, and gave rise
+to an enduring controversy that was still re-complicating itself,
+and growing in the scope of its implications, a full two years later.
+
+The pursuit of the E911 Document stopped with the Steve Jackson Games raid.
+As we have seen, there were hundreds, perhaps thousands of computer users
+in America with the E911 Document in their possession. Theoretically,
+Chicago had a perfect legal right to raid any of these people,
+and could have legally seized the machines of anybody who subscribed to Phrack.
+However, there was no copy of the E911 Document on Jackson's Illuminati board.
+And there the Chicago raiders stopped dead; they have not raided anyone since.
+
+It might be assumed that Rich Andrews and Charlie Boykin, who had brought
+the E911 Document to the attention of telco security, might be spared
+any official suspicion. But as we have seen, the willingness to
+"cooperate fully" offers little, if any, assurance against federal
+anti-hacker prosecution.
+
+Richard Andrews found himself in deep trouble, thanks to the E911 Document.
+Andrews lived in Illinois, the native stomping grounds of the Chicago
+Task Force. On February 3 and 6, both his home and his place of work
+were raided by USSS. His machines went out the door, too, and he was
+grilled at length (though not arrested). Andrews proved to be in
+purportedly guilty possession of: UNIX SVR 3.2; UNIX SVR 3.1; UUCP;
+PMON; WWB; IWB; DWB; NROFF; KORN SHELL '88; C++; and QUEST,
+among other items. Andrews had received this proprietary code--
+which AT&T officially valued at well over $250,000--through the
+UNIX network, much of it supplied to him as a personal favor by Terminus.
+Perhaps worse yet, Andrews admitted to returning the favor, by passing
+Terminus a copy of AT&T proprietary STARLAN source code.
+
+Even Charles Boykin, himself an AT&T employee, entered some very hot water.
+By 1990, he'd almost forgotten about the E911 problem he'd reported in
+September 88; in fact, since that date, he'd passed two more security alerts
+to Jerry Dalton, concerning matters that Boykin considered far worse than
+the E911 Document.
+
+But by 1990, year of the crackdown, AT&T Corporate Information Security
+was fed up with "Killer." This machine offered no direct income to AT&T,
+and was providing aid and comfort to a cloud of suspicious yokels
+from outside the company, some of them actively malicious toward AT&T,
+its property, and its corporate interests. Whatever goodwill and publicity
+had been won among Killer's 1,500 devoted users was considered no longer
+worth the security risk. On February 20, 1990, Jerry Dalton arrived in
+Dallas and simply unplugged the phone jacks, to the puzzled alarm
+of Killer's many Texan users. Killer went permanently off-line,
+with the loss of vast archives of programs and huge quantities
+of electronic mail; it was never restored to service. AT&T showed
+no particular regard for the "property" of these 1,500 people.
+Whatever "property" the users had been storing on AT&T's computer
+simply vanished completely.
+
+Boykin, who had himself reported the E911 problem,
+now found himself under a cloud of suspicion. In a weird
+private-security replay of the Secret Service seizures,
+Boykin's own home was visited by AT&T Security and his
+own machines were carried out the door.
+
+However, there were marked special features in the Boykin case.
+Boykin's disks and his personal computers were swiftly examined
+by his corporate employers and returned politely in just two days--
+(unlike Secret Service seizures, which commonly take months or years).
+Boykin was not charged with any crime or wrongdoing, and he kept his job
+with AT&T (though he did retire from AT&T in September 1991,
+at the age of 52).
+
+It's interesting to note that the US Secret Service somehow failed
+to seize Boykin's "Killer" node and carry AT&T's own computer out the door.
+Nor did they raid Boykin's home. They seemed perfectly willing to take the
+word of AT&T Security that AT&T's employee, and AT&T's "Killer" node,
+were free of hacker contraband and on the up-and-up.
+
+It's digital water-under-the-bridge at this point, as Killer's
+3,200 megabytes of Texan electronic community were erased in 1990,
+and "Killer" itself was shipped out of the state.
+
+But the experiences of Andrews and Boykin, and the users of their systems,
+remained side issues. They did not begin to assume the social, political,
+and legal importance that gathered, slowly but inexorably, around the issue
+of the raid on Steve Jackson Games.
+
+#
+
+We must now turn our attention to Steve Jackson Games itself,
+and explain what SJG was, what it really did, and how it had
+managed to attract this particularly odd and virulent kind of trouble.
+The reader may recall that this is not the first but the second time
+that the company has appeared in this narrative; a Steve Jackson game
+called GURPS was a favorite pastime of Atlanta hacker Urvile,
+and Urvile's science-fictional gaming notes had been mixed up
+promiscuously with notes about his actual computer intrusions.
+
+First, Steve Jackson Games, Inc., was NOT a publisher of "computer games."
+SJG published "simulation games," parlor games that were played on paper,
+with pencils, and dice, and printed guidebooks full of rules and
+statistics tables. There were no computers involved in the games themselves.
+When you bought a Steve Jackson Game, you did not receive any software disks.
+What you got was a plastic bag with some cardboard game tokens,
+maybe a few maps or a deck of cards. Most of their products were books.
+
+However, computers WERE deeply involved in the Steve Jackson Games business.
+Like almost all modern publishers, Steve Jackson and his fifteen employees
+used computers to write text, to keep accounts, and to run the business
+generally. They also used a computer to run their official bulletin board
+system for Steve Jackson Games, a board called Illuminati. On Illuminati,
+simulation gamers who happened to own computers and modems could associate,
+trade mail, debate the theory and practice of gaming, and keep up with the
+company's news and its product announcements.
+
+Illuminati was a modestly popular board, run on a small computer
+with limited storage, only one phone-line, and no ties to large-scale
+computer networks. It did, however, have hundreds of users,
+many of them dedicated gamers willing to call from out-of-state.
+
+Illuminati was NOT an "underground" board. It did not feature hints
+on computer intrusion, or "anarchy files," or illicitly posted
+credit card numbers, or long-distance access codes.
+Some of Illuminati's users, however, were members of the Legion of Doom.
+And so was one of Steve Jackson's senior employees--the Mentor.
+The Mentor wrote for Phrack, and also ran an underground board,
+Phoenix Project--but the Mentor was not a computer professional.
+The Mentor was the managing editor of Steve Jackson Games and
+a professional game designer by trade. These LoD members did not
+use Illuminati to help their HACKING activities. They used it to
+help their GAME-PLAYING activities--and they were even more dedicated
+to simulation gaming than they were to hacking.
+
+"Illuminati" got its name from a card-game that Steve Jackson himself,
+the company's founder and sole owner, had invented. This multi-player
+card-game was one of Mr Jackson's best-known, most successful,
+most technically innovative products. "Illuminati" was a game
+of paranoiac conspiracy in which various antisocial cults warred
+covertly to dominate the world. "Illuminati" was hilarious,
+and great fun to play, involving flying saucers, the CIA, the KGB,
+the phone companies, the Ku Klux Klan, the South American Nazis,
+the cocaine cartels, the Boy Scouts, and dozens of other splinter groups
+from the twisted depths of Mr. Jackson's professionally fervid imagination.
+For the uninitiated, any public discussion of the "Illuminati" card-game
+sounded, by turns, utterly menacing or completely insane.
+
+And then there was SJG's "Car Wars," in which souped-up armored hot-rods
+with rocket-launchers and heavy machine-guns did battle on the American
+highways of the future. The lively Car Wars discussion on the Illuminati
+board featured many meticulous, painstaking discussions of the effects
+of grenades, land-mines, flamethrowers and napalm. It sounded like
+hacker anarchy files run amuck.
+
+Mr Jackson and his co-workers earned their daily bread by supplying people
+with make-believe adventures and weird ideas. The more far-out, the better.
+
+Simulation gaming is an unusual pastime, but gamers have not
+generally had to beg the permission of the Secret Service to exist.
+Wargames and role-playing adventures are an old and honored pastime,
+much favored by professional military strategists. Once little-known,
+these games are now played by hundreds of thousands of enthusiasts
+throughout North America, Europe and Japan. Gaming-books, once restricted
+to hobby outlets, now commonly appear in chain-stores like B. Dalton's
+and Waldenbooks, and sell vigorously.
+
+Steve Jackson Games, Inc., of Austin, Texas, was a games company
+of the middle rank. In 1989, SJG grossed about a million dollars.
+Jackson himself had a good reputation in his industry as a talented
+and innovative designer of rather unconventional games, but his company
+was something less than a titan of the field--certainly not like the
+multimillion-dollar TSR Inc., or Britain's gigantic "Games Workshop."
+SJG's Austin headquarters was a modest two-story brick office-suite,
+cluttered with phones, photocopiers, fax machines and computers.
+It bustled with semi-organized activity and was littered with
+glossy promotional brochures and dog-eared science-fiction novels.
+Attached to the offices was a large tin-roofed warehouse piled twenty feet
+high with cardboard boxes of games and books. Despite the weird imaginings
+that went on within it, the SJG headquarters was quite a quotidian,
+everyday sort of place. It looked like what it was: a publishers' digs.
+
+Both "Car Wars" and "Illuminati" were well-known, popular games.
+But the mainstay of the Jackson organization was their Generic Universal
+Role-Playing System, "G.U.R.P.S." The GURPS system was considered solid
+and well-designed, an asset for players. But perhaps the most popular
+feature of the GURPS system was that it allowed gaming-masters to design
+scenarios that closely resembled well-known books, movies, and other works
+of fantasy. Jackson had licensed and adapted works from many science fiction
+and fantasy authors. There was GURPS Conan, GURPS Riverworld,
+GURPS Horseclans, GURPS Witch World, names eminently familiar
+to science-fiction readers. And there was GURPS Special Ops,
+from the world of espionage fantasy and unconventional warfare.
+
+And then there was GURPS Cyberpunk.
+
+"Cyberpunk" was a term given to certain science fiction writers
+who had entered the genre in the 1980s. "Cyberpunk," as the label implies,
+had two general distinguishing features. First, its writers had a compelling
+interest in information technology, an interest closely akin
+to science fiction's earlier fascination with space travel.
+And second, these writers were "punks," with all the
+distinguishing features that that implies: Bohemian artiness,
+youth run wild, an air of deliberate rebellion, funny clothes and hair,
+odd politics, a fondness for abrasive rock and roll; in a word, trouble.
+
+The "cyberpunk" SF writers were a small group of mostly college-educated
+white middle-class litterateurs, scattered through the US and Canada.
+Only one, Rudy Rucker, a professor of computer science in Silicon Valley,
+could rank with even the humblest computer hacker. But, except for
+Professor Rucker, the "cyberpunk" authors were not programmers
+or hardware experts; they considered themselves artists
+(as, indeed, did Professor Rucker). However, these writers
+all owned computers, and took an intense and public interest
+in the social ramifications of the information industry.
+
+The cyberpunks had a strong following among the global generation
+that had grown up in a world of computers, multinational networks,
+and cable television. Their outlook was considered somewhat morbid,
+cynical, and dark, but then again, so was the outlook of their
+generational peers. As that generation matured and increased
+in strength and influence, so did the cyberpunks.
+As science-fiction writers went, they were doing
+fairly well for themselves. By the late 1980s,
+their work had attracted attention from gaming companies,
+including Steve Jackson Games, which was planning a cyberpunk
+simulation for the flourishing GURPS gaming-system.
+
+The time seemed ripe for such a product, which had already been proven
+in the marketplace. The first games- company out of the gate,
+with a product boldly called "Cyberpunk" in defiance of possible
+infringement-of-copyright suits, had been an upstart group called
+R. Talsorian. Talsorian's Cyberpunk was a fairly decent game,
+but the mechanics of the simulation system left a lot to be desired.
+Commercially, however, the game did very well.
+
+The next cyberpunk game had been the even more successful Shadowrun
+by FASA Corporation. The mechanics of this game were fine, but the
+scenario was rendered moronic by sappy fantasy elements like elves,
+trolls, wizards, and dragons--all highly ideologically-incorrect,
+according to the hard-edged, high-tech standards of cyberpunk science fiction.
+
+Other game designers were champing at the bit. Prominent among them
+was the Mentor, a gentleman who, like most of his friends in the
+Legion of Doom, was quite the cyberpunk devotee. Mentor reasoned
+that the time had come for a REAL cyberpunk gaming-book--one that the
+princes of computer-mischief in the Legion of Doom could play without
+laughing themselves sick. This book, GURPS Cyberpunk, would reek
+of culturally on-line authenticity.
+
+Mentor was particularly well-qualified for this task.
+Naturally, he knew far more about computer-intrusion
+and digital skullduggery than any previously published
+cyberpunk author. Not only that, but he was good at his work.
+A vivid imagination, combined with an instinctive feeling
+for the working of systems and, especially, the loopholes
+within them, are excellent qualities for a professional game designer.
+
+By March 1st, GURPS Cyberpunk was almost complete, ready to print and ship.
+Steve Jackson expected vigorous sales for this item, which, he hoped,
+would keep the company financially afloat for several months.
+GURPS Cyberpunk, like the other GURPS "modules," was not a "game"
+like a Monopoly set, but a BOOK: a bound paperback book the size
+of a glossy magazine, with a slick color cover, and pages full of text,
+illustrations, tables and footnotes. It was advertised as a game,
+and was used as an aid to game-playing, but it was a book,
+with an ISBN number, published in Texas, copyrighted,
+and sold in bookstores.
+
+And now, that book, stored on a computer, had gone out the door
+in the custody of the Secret Service.
+
+The day after the raid, Steve Jackson visited the local Secret Service
+headquarters with a lawyer in tow. There he confronted Tim Foley
+(still in Austin at that time) and demanded his book back. But there
+was trouble. GURPS Cyberpunk, alleged a Secret Service agent to astonished
+businessman Steve Jackson, was "a manual for computer crime."
+
+"It's science fiction," Jackson said.
+
+"No, this is real."
+
+This statement was repeated several times, by several agents.
+Jackson's ominously accurate game had passed from pure,
+obscure, small-scale fantasy into the impure, highly publicized,
+large-scale fantasy of the Hacker Crackdown.
+
+No mention was made of the real reason for the search.
+According to their search warrant, the raiders had expected
+to find the E911 Document stored on Jackson's bulletin board system.
+But that warrant was sealed; a procedure that most law enforcement agencies
+will use only when lives are demonstrably in danger. The raiders'
+true motives were not discovered until the Jackson search-warrant
+was unsealed by his lawyers, many months later. The Secret Service,
+and the Chicago Computer Fraud and Abuse Task Force,
+said absolutely nothing to Steve Jackson about any threat
+to the police 911 System. They said nothing about the Atlanta Three,
+nothing about Phrack or Knight Lightning, nothing about Terminus.
+
+Jackson was left to believe that his computers had been seized because
+he intended to publish a science fiction book that law enforcement
+considered too dangerous to see print.
+
+This misconception was repeated again and again, for months,
+to an ever-widening public audience. It was not the truth of the case;
+but as months passed, and this misconception was publicly printed again
+and again, it became one of the few publicly known "facts" about
+the mysterious Hacker Crackdown. The Secret Service had seized a computer
+to stop the publication of a cyberpunk science fiction book.
+
+The second section of this book, "The Digital Underground,"
+is almost finished now. We have become acquainted with all
+the major figures of this case who actually belong to the
+underground milieu of computer intrusion. We have some idea
+of their history, their motives, their general modus operandi.
+We now know, I hope, who they are, where they came from,
+and more or less what they want. In the next section of this book,
+"Law and Order," we will leave this milieu and directly enter the
+world of America's computer-crime police.
+
+At this point, however, I have another figure to introduce: myself.
+
+My name is Bruce Sterling. I live in Austin, Texas, where I am
+a science fiction writer by trade: specifically, a CYBERPUNK
+science fiction writer.
+
+Like my "cyberpunk" colleagues in the U.S. and Canada,
+I've never been entirely happy with this literary label--
+especially after it became a synonym for computer criminal.
+But I did once edit a book of stories by my colleagues,
+called Mirrorshades: the Cyberpunk Anthology, and I've
+long been a writer of literary-critical cyberpunk manifestos.
+I am not a "hacker" of any description, though I do have readers
+in the digital underground.
+
+When the Steve Jackson Games seizure occurred, I naturally took
+an intense interest. If "cyberpunk" books were being banned
+by federal police in my own home town, I reasonably wondered
+whether I myself might be next. Would my computer be seized
+by the Secret Service? At the time, I was in possession
+of an aging Apple IIe without so much as a hard disk.
+If I were to be raided as an author of computer-crime manuals,
+the loss of my feeble word-processor would likely provoke more
+snickers than sympathy.
+
+I'd known Steve Jackson for many years. We knew
+one another as colleagues, for we frequented
+the same local science-fiction conventions.
+I'd played Jackson games, and recognized his cleverness;
+but he certainly had never struck me as a potential mastermind
+of computer crime.
+
+I also knew a little about computer bulletin-board systems.
+In the mid-1980s I had taken an active role in an Austin board
+called "SMOF-BBS," one of the first boards dedicated to science fiction.
+I had a modem, and on occasion I'd logged on to Illuminati,
+which always looked entertainly wacky, but certainly harmless enough.
+
+At the time of the Jackson seizure, I had no experience
+whatsoever with underground boards. But I knew that no one
+on Illuminati talked about breaking into systems illegally,
+or about robbing phone companies. Illuminati didn't even
+offer pirated computer games. Steve Jackson, like many creative artists,
+was markedly touchy about theft of intellectual property.
+
+It seemed to me that Jackson was either seriously suspected
+of some crime--in which case, he would be charged soon,
+and would have his day in court--or else he was innocent,
+in which case the Secret Service would quickly return his equipment,
+and everyone would have a good laugh. I rather expected the good laugh.
+The situation was not without its comic side. The raid, known
+as the "Cyberpunk Bust" in the science fiction community,
+was winning a great deal of free national publicity both
+for Jackson himself and the "cyberpunk" science fiction
+writers generally.
+
+Besides, science fiction people are used to being misinterpreted.
+Science fiction is a colorful, disreputable, slipshod occupation,
+full of unlikely oddballs, which, of course, is why we like it.
+Weirdness can be an occupational hazard in our field. People who
+wear Halloween costumes are sometimes mistaken for monsters.
+
+Once upon a time--back in 1939, in New York City--
+science fiction and the U.S. Secret Service collided in
+a comic case of mistaken identity. This weird incident
+involved a literary group quite famous in science fiction,
+known as "the Futurians," whose membership included
+such future genre greats as Isaac Asimov, Frederik Pohl,
+and Damon Knight. The Futurians were every bit as
+offbeat and wacky as any of their spiritual descendants,
+including the cyberpunks, and were given to communal living,
+spontaneous group renditions of light opera, and midnight fencing
+exhibitions on the lawn. The Futurians didn't have bulletin
+board systems, but they did have the technological equivalent
+in 1939--mimeographs and a private printing press. These were
+in steady use, producing a stream of science-fiction fan magazines,
+literary manifestos, and weird articles, which were picked up
+in ink-sticky bundles by a succession of strange, gangly,
+spotty young men in fedoras and overcoats.
+
+The neighbors grew alarmed at the antics of the Futurians
+and reported them to the Secret Service as suspected counterfeiters.
+In the winter of 1939, a squad of USSS agents with drawn guns burst into
+"Futurian House," prepared to confiscate the forged currency and illicit
+printing presses. There they discovered a slumbering science fiction fan
+named George Hahn, a guest of the Futurian commune who had just arrived
+in New York. George Hahn managed to explain himself and his group,
+and the Secret Service agents left the Futurians in peace henceforth.
+(Alas, Hahn died in 1991, just before I had discovered this astonishing
+historical parallel, and just before I could interview him for this book.)
+
+But the Jackson case did not come to a swift and comic end.
+No quick answers came his way, or mine; no swift reassurances
+that all was right in the digital world, that matters were well
+in hand after all. Quite the opposite. In my alternate role
+as a sometime pop-science journalist, I interviewed Jackson
+and his staff for an article in a British magazine.
+The strange details of the raid left me more concerned than ever.
+Without its computers, the company had been financially
+and operationally crippled. Half the SJG workforce,
+a group of entirely innocent people, had been sorrowfully fired,
+deprived of their livelihoods by the seizure. It began to dawn on me
+that authors--American writers--might well have their computers seized,
+under sealed warrants, without any criminal charge; and that,
+as Steve Jackson had discovered, there was no immediate recourse for this.
+This was no joke; this wasn't science fiction; this was real.
+
+I determined to put science fiction aside until I had discovered
+what had happened and where this trouble had come from.
+It was time to enter the purportedly real world of electronic
+free expression and computer crime. Hence, this book.
+Hence, the world of the telcos; and the world of the digital underground;
+and next, the world of the police.
+
+
+
+PART THREE: LAW AND ORDER
+
+
+Of the various anti-hacker activities of 1990, "Operation Sundevil"
+had by far the highest public profile. The sweeping, nationwide
+computer seizures of May 8, 1990 were unprecedented in scope and highly,
+if rather selectively, publicized.
+
+Unlike the efforts of the Chicago Computer Fraud and Abuse Task Force,
+"Operation Sundevil" was not intended to combat "hacking" in the sense
+of computer intrusion or sophisticated raids on telco switching stations.
+Nor did it have anything to do with hacker misdeeds with AT&T's software,
+or with Southern Bell's proprietary documents.
+
+Instead, "Operation Sundevil" was a crackdown on those traditional scourges
+of the digital underground: credit-card theft and telephone code abuse.
+The ambitious activities out of Chicago, and the somewhat lesser-known
+but vigorous anti-hacker actions of the New York State Police in 1990,
+were never a part of "Operation Sundevil" per se, which was based in Arizona.
+
+Nevertheless, after the spectacular May 8 raids, the public, misled by
+police secrecy, hacker panic, and a puzzled national press-corps,
+conflated all aspects of the nationwide crackdown in 1990 under
+the blanket term "Operation Sundevil." "Sundevil" is still the best-known
+synonym for the crackdown of 1990. But the Arizona organizers of "Sundevil"
+did not really deserve this reputation--any more, for instance, than all
+hackers deserve a reputation as "hackers."
+
+There was some justice in this confused perception, though.
+For one thing, the confusion was abetted by the Washington office
+of the Secret Service, who responded to Freedom of Information Act
+requests on "Operation Sundevil" by referring investigators
+to the publicly known cases of Knight Lightning and the Atlanta Three.
+And "Sundevil" was certainly the largest aspect of the Crackdown,
+the most deliberate and the best-organized. As a crackdown on electronic
+fraud, "Sundevil" lacked the frantic pace of the war on the Legion of Doom;
+on the contrary, Sundevil's targets were picked out with cool deliberation
+over an elaborate investigation lasting two full years.
+
+And once again the targets were bulletin board systems.
+
+Boards can be powerful aids to organized fraud. Underground boards carry
+lively, extensive, detailed, and often quite flagrant "discussions" of
+lawbreaking techniques and lawbreaking activities. "Discussing" crime
+in the abstract, or "discussing" the particulars of criminal cases,
+is not illegal--but there are stern state and federal laws against
+coldbloodedly conspiring in groups in order to commit crimes.
+
+In the eyes of police, people who actively conspire to break the law
+are not regarded as "clubs," "debating salons," "users' groups," or
+"free speech advocates." Rather, such people tend to find themselves
+formally indicted by prosecutors as "gangs," "racketeers," "corrupt
+organizations" and "organized crime figures."
+
+What's more, the illicit data contained on outlaw boards goes well beyond
+mere acts of speech and/or possible criminal conspiracy. As we have seen,
+it was common practice in the digital underground to post purloined telephone
+codes on boards, for any phreak or hacker who cared to abuse them. Is posting
+digital booty of this sort supposed to be protected by the First Amendment?
+Hardly--though the issue, like most issues in cyberspace, is not entirely
+resolved. Some theorists argue that to merely RECITE a number publicly
+is not illegal--only its USE is illegal. But anti-hacker police point out
+that magazines and newspapers (more traditional forms of free expression)
+never publish stolen telephone codes (even though this might well
+raise their circulation).
+
+Stolen credit card numbers, being riskier and more valuable,
+were less often publicly posted on boards--but there is no question
+that some underground boards carried "carding" traffic,
+generally exchanged through private mail.
+
+Underground boards also carried handy programs for "scanning" telephone
+codes and raiding credit card companies, as well as the usual obnoxious
+galaxy of pirated software, cracked passwords, blue-box schematics,
+intrusion manuals, anarchy files, porn files, and so forth.
+
+But besides their nuisance potential for the spread of illicit knowledge,
+bulletin boards have another vitally interesting aspect for the
+professional investigator. Bulletin boards are cram-full of EVIDENCE.
+All that busy trading of electronic mail, all those hacker boasts,
+brags and struts, even the stolen codes and cards, can be neat,
+electronic, real-time recordings of criminal activity.
+As an investigator, when you seize a pirate board, you have
+scored a coup as effective as tapping phones or intercepting mail.
+However, you have not actually tapped a phone or intercepted a letter.
+The rules of evidence regarding phone-taps and mail interceptions are old,
+stern and well-understood by police, prosecutors and defense attorneys alike.
+The rules of evidence regarding boards are new, waffling, and understood
+by nobody at all.
+
+Sundevil was the largest crackdown on boards in world history.
+On May 7, 8, and 9, 1990, about forty-two computer systems were seized.
+Of those forty-two computers, about twenty-five actually were running boards.
+(The vagueness of this estimate is attributable to the vagueness of
+(a) what a "computer system" is, and (b) what it actually means to
+"run a board" with one--or with two computers, or with three.)
+
+About twenty-five boards vanished into police custody in May 1990.
+As we have seen, there are an estimated 30,000 boards in America today.
+If we assume that one board in a hundred is up to no good with codes
+and cards (which rather flatters the honesty of the board-using community),
+then that would leave 2,975 outlaw boards untouched by Sundevil.
+Sundevil seized about one tenth of one percent of all computer
+bulletin boards in America. Seen objectively, this is something less
+than a comprehensive assault. In 1990, Sundevil's organizers--
+the team at the Phoenix Secret Service office, and the Arizona
+Attorney General's office-- had a list of at least THREE HUNDRED
+boards that they considered fully deserving of search and seizure warrants.
+The twenty-five boards actually seized were merely among the most obvious
+and egregious of this much larger list of candidates. All these boards
+had been examined beforehand--either by informants, who had passed printouts
+to the Secret Service, or by Secret Service agents themselves, who not only
+come equipped with modems but know how to use them.
+
+There were a number of motives for Sundevil. First, it offered
+a chance to get ahead of the curve on wire-fraud crimes.
+Tracking back credit-card ripoffs to their perpetrators
+can be appallingly difficult. If these miscreants
+have any kind of electronic sophistication, they can snarl
+their tracks through the phone network into a mind-boggling,
+untraceable mess, while still managing to "reach out and rob someone."
+Boards, however, full of brags and boasts, codes and cards,
+offer evidence in the handy congealed form.
+
+Seizures themselves--the mere physical removal of machines--
+tends to take the pressure off. During Sundevil, a large number
+of code kids, warez d00dz, and credit card thieves would be deprived
+of those boards--their means of community and conspiracy--in one swift blow.
+As for the sysops themselves (commonly among the boldest offenders)
+they would be directly stripped of their computer equipment,
+and rendered digitally mute and blind.
+
+And this aspect of Sundevil was carried out with great success.
+Sundevil seems to have been a complete tactical surprise--
+unlike the fragmentary and continuing seizures of the war on the
+Legion of Doom, Sundevil was precisely timed and utterly overwhelming.
+At least forty "computers" were seized during May 7, 8 and 9, 1990,
+in Cincinnati, Detroit, Los Angeles, Miami, Newark, Phoenix, Tucson,
+Richmond, San Diego, San Jose, Pittsburgh and San Francisco.
+Some cities saw multiple raids, such as the five separate raids
+in the New York City environs. Plano, Texas (essentially a suburb of
+the Dallas/Fort Worth metroplex, and a hub of the telecommunications industry)
+saw four computer seizures. Chicago, ever in the forefront, saw its own
+local Sundevil raid, briskly carried out by Secret Service agents
+Timothy Foley and Barbara Golden.
+
+Many of these raids occurred, not in the cities proper,
+but in associated white-middle class suburbs--places like
+Mount Lebanon, Pennsylvania and Clark Lake, Michigan.
+There were a few raids on offices; most took place in people's homes,
+the classic hacker basements and bedrooms.
+
+The Sundevil raids were searches and seizures, not a group of mass arrests.
+There were only four arrests during Sundevil. "Tony the Trashman,"
+a longtime teenage bete noire of the Arizona Racketeering unit,
+was arrested in Tucson on May 9. "Dr. Ripco," sysop of an outlaw board
+with the misfortune to exist in Chicago itself, was also arrested--
+on illegal weapons charges. Local units also arrested a 19-year-old
+female phone phreak named "Electra" in Pennsylvania, and a male juvenile
+in California. Federal agents however were not seeking arrests, but computers.
+
+Hackers are generally not indicted (if at all) until the evidence
+in their seized computers is evaluated--a process that can take weeks,
+months--even years. When hackers are arrested on the spot, it's generally
+an arrest for other reasons. Drugs and/or illegal weapons show up in a good
+third of anti-hacker computer seizures (though not during Sundevil).
+
+That scofflaw teenage hackers (or their parents) should have marijuana
+in their homes is probably not a shocking revelation, but the surprisingly
+common presence of illegal firearms in hacker dens is a bit disquieting.
+A Personal Computer can be a great equalizer for the techno-cowboy--
+much like that more traditional American "Great Equalizer,"
+the Personal Sixgun. Maybe it's not all that surprising
+that some guy obsessed with power through illicit technology
+would also have a few illicit high-velocity-impact devices around.
+An element of the digital underground particularly dotes on those
+"anarchy philes," and this element tends to shade into the crackpot milieu
+of survivalists, gun-nuts, anarcho-leftists and the ultra-libertarian
+right-wing.
+
+This is not to say that hacker raids to date have uncovered any
+major crack-dens or illegal arsenals; but Secret Service agents
+do not regard "hackers" as "just kids." They regard hackers as
+unpredictable people, bright and slippery. It doesn't help matters
+that the hacker himself has been "hiding behind his keyboard"
+all this time. Commonly, police have no idea what he looks like.
+This makes him an unknown quantity, someone best treated with
+proper caution.
+
+To date, no hacker has come out shooting, though they do sometimes brag on
+boards that they will do just that. Threats of this sort are taken seriously.
+Secret Service hacker raids tend to be swift, comprehensive, well-manned
+(even over-manned); and agents generally burst through every door
+in the home at once, sometimes with drawn guns. Any potential resistance
+is swiftly quelled. Hacker raids are usually raids on people's homes.
+It can be a very dangerous business to raid an American home;
+people can panic when strangers invade their sanctum. Statistically speaking,
+the most dangerous thing a policeman can do is to enter someone's home.
+(The second most dangerous thing is to stop a car in traffic.)
+People have guns in their homes. More cops are hurt in homes
+than are ever hurt in biker bars or massage parlors.
+
+But in any case, no one was hurt during Sundevil,
+or indeed during any part of the Hacker Crackdown.
+
+Nor were there any allegations of any physical mistreatment of a suspect.
+Guns were pointed, interrogations were sharp and prolonged; but no one
+in 1990 claimed any act of brutality by any crackdown raider.
+
+In addition to the forty or so computers, Sundevil reaped floppy disks
+in particularly great abundance--an estimated 23,000 of them, which
+naturally included every manner of illegitimate data: pirated games,
+stolen codes, hot credit card numbers, the complete text and software
+of entire pirate bulletin-boards. These floppy disks, which remain
+in police custody today, offer a gigantic, almost embarrassingly
+rich source of possible criminal indictments. These 23,000 floppy disks
+also include a thus-far unknown quantity of legitimate computer games,
+legitimate software, purportedly "private" mail from boards,
+business records, and personal correspondence of all kinds.
+
+Standard computer-crime search warrants lay great emphasis on seizing
+written documents as well as computers--specifically including photocopies,
+computer printouts, telephone bills, address books, logs, notes,
+memoranda and correspondence. In practice, this has meant that diaries,
+gaming magazines, software documentation, nonfiction books on hacking
+and computer security, sometimes even science fiction novels, have all
+vanished out the door in police custody. A wide variety of electronic items
+have been known to vanish as well, including telephones, televisions, answering
+machines, Sony Walkmans, desktop printers, compact disks, and audiotapes.
+
+No fewer than 150 members of the Secret Service were sent into
+the field during Sundevil. They were commonly accompanied by
+squads of local and/or state police. Most of these officers--
+especially the locals--had never been on an anti-hacker raid before.
+(This was one good reason, in fact, why so many of them were invited along
+in the first place.) Also, the presence of a uniformed police officer
+assures the raidees that the people entering their homes are, in fact, police.
+Secret Service agents wear plain clothes. So do the telco security experts
+who commonly accompany the Secret Service on raids (and who make no particular
+effort to identify themselves as mere employees of telephone companies).
+
+A typical hacker raid goes something like this. First, police storm in
+rapidly, through every entrance, with overwhelming force,
+in the assumption that this tactic will keep casualties to a minimum.
+Second, possible suspects are immediately removed from the vicinity
+of any and all computer systems, so that they will have no chance
+to purge or destroy computer evidence. Suspects are herded into a room
+without computers, commonly the living room, and kept under guard--
+not ARMED guard, for the guns are swiftly holstered, but under guard
+nevertheless. They are presented with the search warrant and warned
+that anything they say may be held against them. Commonly they have
+a great deal to say, especially if they are unsuspecting parents.
+
+Somewhere in the house is the "hot spot"--a computer tied to a phone
+line (possibly several computers and several phones). Commonly it's
+a teenager's bedroom, but it can be anywhere in the house;
+there may be several such rooms. This "hot spot" is put in charge
+of a two-agent team, the "finder" and the "recorder." The "finder"
+is computer-trained, commonly the case agent who has actually obtained
+the search warrant from a judge. He or she understands what is being sought,
+and actually carries out the seizures: unplugs machines, opens drawers,
+desks, files, floppy-disk containers, etc. The "recorder" photographs
+all the equipment, just as it stands--especially the tangle of
+wired connections in the back, which can otherwise be a real nightmare
+to restore. The recorder will also commonly photograph every room
+in the house, lest some wily criminal claim that the police had robbed him
+during the search. Some recorders carry videocams or tape recorders;
+however, it's more common for the recorder to simply take written notes.
+Objects are described and numbered as the finder seizes them, generally
+on standard preprinted police inventory forms.
+
+Even Secret Service agents were not, and are not, expert computer users.
+They have not made, and do not make, judgements on the fly about potential
+threats posed by various forms of equipment. They may exercise discretion;
+they may leave Dad his computer, for instance, but they don't HAVE to.
+Standard computer-crime search warrants, which date back to the early 80s,
+use a sweeping language that targets computers, most anything attached
+to a computer, most anything used to operate a computer--most anything
+that remotely resembles a computer--plus most any and all written documents
+surrounding it. Computer-crime investigators have strongly urged agents
+to seize the works.
+
+In this sense, Operation Sundevil appears to have been a complete success.
+Boards went down all over America, and were shipped en masse to the computer
+investigation lab of the Secret Service, in Washington DC, along with the
+23,000 floppy disks and unknown quantities of printed material.
+
+But the seizure of twenty-five boards, and the multi-megabyte mountains
+of possibly useful evidence contained in these boards (and in their owners'
+other computers, also out the door), were far from the only motives for
+Operation Sundevil. An unprecedented action of great ambition and size,
+Sundevil's motives can only be described as political. It was a
+public-relations effort, meant to pass certain messages, meant to make
+certain situations clear: both in the mind of the general public,
+and in the minds of various constituencies of the electronic community.
+
+ First --and this motivation was vital--a "message" would be sent from
+law enforcement to the digital underground. This very message was recited
+in so many words by Garry M. Jenkins, the Assistant Director of the
+US Secret Service, at the Sundevil press conference in Phoenix on
+May 9, 1990, immediately after the raids. In brief, hackers were
+mistaken in their foolish belief that they could hide behind the
+"relative anonymity of their computer terminals." On the contrary,
+they should fully understand that state and federal cops were
+actively patrolling the beat in cyberspace--that they were
+on the watch everywhere, even in those sleazy and secretive
+dens of cybernetic vice, the underground boards.
+
+This is not an unusual message for police to publicly convey to crooks.
+The message is a standard message; only the context is new.
+
+In this respect, the Sundevil raids were the digital equivalent
+of the standard vice-squad crackdown on massage parlors, porno bookstores,
+head-shops, or floating crap-games. There may be few or no arrests in a raid
+of this sort; no convictions, no trials, no interrogations. In cases of this
+sort, police may well walk out the door with many pounds of sleazy magazines,
+X-rated videotapes, sex toys, gambling equipment, baggies of marijuana. . . .
+
+Of course, if something truly horrendous is discovered by the raiders,
+there will be arrests and prosecutions. Far more likely, however,
+there will simply be a brief but sharp disruption of the closed
+and secretive world of the nogoodniks. There will be "street hassle."
+"Heat." "Deterrence." And, of course, the immediate loss of the seized goods.
+It is very unlikely that any of this seized material will ever be returned.
+Whether charged or not, whether convicted or not, the perpetrators will
+almost surely lack the nerve ever to ask for this stuff to be given back.
+
+Arrests and trials--putting people in jail--may involve all kinds of
+formal legalities; but dealing with the justice system is far from the only
+task of police. Police do not simply arrest people. They don't simply
+put people in jail. That is not how the police perceive their jobs.
+Police "protect and serve." Police "keep the peace," they "keep public order."
+Like other forms of public relations, keeping public order is not an
+exact science. Keeping public order is something of an art-form.
+
+If a group of tough-looking teenage hoodlums was loitering on a street-corner,
+no one would be surprised to see a street-cop arrive and sternly order
+them to "break it up." On the contrary, the surprise would come if one
+of these ne'er-do-wells stepped briskly into a phone-booth,
+called a civil rights lawyer, and instituted a civil suit
+in defense of his Constitutional rights of free speech
+and free assembly. But something much along this line
+was one of the many anomolous outcomes of the Hacker Crackdown.
+
+Sundevil also carried useful "messages" for other constituents of
+the electronic community. These messages may not have been read
+aloud from the Phoenix podium in front of the press corps,
+but there was little mistaking their meaning. There was a message
+of reassurance for the primary victims of coding and carding:
+the telcos, and the credit companies. Sundevil was greeted with joy
+by the security officers of the electronic business community.
+After years of high-tech harassment and spiralling revenue losses,
+their complaints of rampant outlawry were being taken seriously by
+law enforcement. No more head-scratching or dismissive shrugs;
+no more feeble excuses about "lack of computer-trained officers" or
+the low priority of "victimless" white-collar telecommunication crimes.
+
+Computer-crime experts have long believed that computer-related offenses
+are drastically under-reported. They regard this as a major open scandal
+of their field. Some victims are reluctant to come forth, because they
+believe that police and prosecutors are not computer-literate,
+and can and will do nothing. Others are embarrassed by
+their vulnerabilities, and will take strong measures
+to avoid any publicity; this is especially true of banks,
+who fear a loss of investor confidence should an embezzlement-case
+or wire-fraud surface. And some victims are so helplessly confused
+by their own high technology that they never even realize that
+a crime has occurred--even when they have been fleeced to the bone.
+
+The results of this situation can be dire.
+Criminals escape apprehension and punishment.
+The computer-crime units that do exist, can't get work.
+The true scope of computer-crime: its size, its real nature,
+the scope of its threats, and the legal remedies for it--
+all remain obscured.
+
+Another problem is very little publicized, but it is a cause
+of genuine concern. Where there is persistent crime,
+but no effective police protection, then vigilantism can result.
+Telcos, banks, credit companies, the major corporations who
+maintain extensive computer networks vulnerable to hacking
+--these organizations are powerful, wealthy, and
+politically influential. They are disinclined to be
+pushed around by crooks (or by most anyone else,
+for that matter). They often maintain well-organized
+private security forces, commonly run by
+experienced veterans of military and police units,
+who have left public service for the greener pastures
+of the private sector. For police, the corporate
+security manager can be a powerful ally; but if this
+gentleman finds no allies in the police, and the
+pressure is on from his board-of-directors,
+he may quietly take certain matters into his own hands.
+
+Nor is there any lack of disposable hired-help in the
+corporate security business. Private security agencies--
+the `security business' generally--grew explosively in the 1980s.
+Today there are spooky gumshoed armies of "security consultants,"
+"rent-a- cops," "private eyes," "outside experts"--every manner
+of shady operator who retails in "results" and discretion.
+Or course, many of these gentlemen and ladies may be paragons
+of professional and moral rectitude. But as anyone
+who has read a hard-boiled detective novel knows,
+police tend to be less than fond of this sort
+of private-sector competition.
+
+Companies in search of computer-security have even been
+known to hire hackers. Police shudder at this prospect.
+
+Police treasure good relations with the business community.
+Rarely will you see a policeman so indiscreet as to allege
+publicly that some major employer in his state or city has succumbed
+to paranoia and gone off the rails. Nevertheless,
+police --and computer police in particular--are aware
+of this possibility. Computer-crime police can and do
+spend up to half of their business hours just doing
+public relations: seminars, "dog and pony shows,"
+sometimes with parents' groups or computer users,
+but generally with their core audience: the likely
+victims of hacking crimes. These, of course, are telcos,
+credit card companies and large computer-equipped corporations.
+The police strongly urge these people, as good citizens,
+to report offenses and press criminal charges;
+they pass the message that there is someone in authority who cares,
+understands, and, best of all, will take useful action
+should a computer-crime occur.
+
+But reassuring talk is cheap. Sundevil offered action.
+
+The final message of Sundevil was intended for internal consumption
+by law enforcement. Sundevil was offered as proof that the community
+of American computer-crime police had come of age. Sundevil was
+proof that enormous things like Sundevil itself could now be accomplished.
+Sundevil was proof that the Secret Service and its local law-enforcement
+allies could act like a well-oiled machine--(despite the hampering use
+of those scrambled phones). It was also proof that the Arizona Organized
+Crime and Racketeering Unit--the sparkplug of Sundevil--ranked with the best
+in the world in ambition, organization, and sheer conceptual daring.
+
+And, as a final fillip, Sundevil was a message from the Secret Service
+to their longtime rivals in the Federal Bureau of Investigation.
+By Congressional fiat, both USSS and FBI formally share jurisdiction
+over federal computer-crimebusting activities. Neither of these groups
+has ever been remotely happy with this muddled situation. It seems to
+suggest that Congress cannot make up its mind as to which of these groups
+is better qualified. And there is scarcely a G-man or a Special Agent
+anywhere without a very firm opinion on that topic.
+
+#
+
+For the neophyte, one of the most puzzling aspects of the crackdown
+on hackers is why the United States Secret Service has anything at all
+to do with this matter.
+
+The Secret Service is best known for its primary public role:
+its agents protect the President of the United States.
+They also guard the President's family, the Vice President and his family,
+former Presidents, and Presidential candidates. They sometimes guard
+foreign dignitaries who are visiting the United States, especially foreign
+heads of state, and have been known to accompany American officials
+on diplomatic missions overseas.
+
+Special Agents of the Secret Service don't wear uniforms, but the
+Secret Service also has two uniformed police agencies. There's the
+former White House Police (now known as the Secret Service Uniformed Division,
+since they currently guard foreign embassies in Washington, as well as the
+White House itself). And there's the uniformed Treasury Police Force.
+
+The Secret Service has been charged by Congress with a number
+of little-known duties. They guard the precious metals in Treasury vaults.
+They guard the most valuable historical documents of the United States:
+originals of the Constitution, the Declaration of Independence,
+Lincoln's Second Inaugural Address, an American-owned copy of
+the Magna Carta, and so forth. Once they were assigned to guard
+the Mona Lisa, on her American tour in the 1960s.
+
+The entire Secret Service is a division of the Treasury Department.
+Secret Service Special Agents (there are about 1,900 of them)
+are bodyguards for the President et al, but they all work for the Treasury.
+And the Treasury (through its divisions of the U.S. Mint and the
+Bureau of Engraving and Printing) prints the nation's money.
+
+As Treasury police, the Secret Service guards the nation's currency;
+it is the only federal law enforcement agency with direct jurisdiction
+over counterfeiting and forgery. It analyzes documents for authenticity,
+and its fight against fake cash is still quite lively (especially since
+the skilled counterfeiters of Medellin, Columbia have gotten into the act).
+Government checks, bonds, and other obligations, which exist in untold
+millions and are worth untold billions, are common targets for forgery,
+which the Secret Service also battles. It even handles forgery
+of postage stamps.
+
+But cash is fading in importance today as money has become electronic.
+As necessity beckoned, the Secret Service moved from fighting the
+counterfeiting of paper currency and the forging of checks,
+to the protection of funds transferred by wire.
+
+From wire-fraud, it was a simple skip-and-jump to what is formally
+known as "access device fraud." Congress granted the Secret Service
+the authority to investigate "access device fraud" under Title 18
+of the United States Code (U.S.C. Section 1029).
+
+The term "access device" seems intuitively simple. It's some kind
+of high-tech gizmo you use to get money with. It makes good sense
+to put this sort of thing in the charge of counterfeiting and
+wire-fraud experts.
+
+However, in Section 1029, the term "access device" is very
+generously defined. An access device is: "any card, plate,
+code, account number, or other means of account access
+that can be used, alone or in conjunction with another access device,
+to obtain money, goods, services, or any other thing of value,
+or that can be used to initiate a transfer of funds."
+
+"Access device" can therefore be construed to include credit cards
+themselves (a popular forgery item nowadays). It also includes credit card
+account NUMBERS, those standards of the digital underground. The same goes
+for telephone charge cards (an increasingly popular item with telcos,
+who are tired of being robbed of pocket change by phone-booth thieves).
+And also telephone access CODES, those OTHER standards of the digital
+underground. (Stolen telephone codes may not "obtain money," but they
+certainly do obtain valuable "services," which is specifically forbidden
+by Section 1029.)
+
+We can now see that Section 1029 already pits the United States Secret Service
+directly against the digital underground, without any mention at all of
+the word "computer."
+
+Standard phreaking devices, like "blue boxes," used to steal phone service
+from old-fashioned mechanical switches, are unquestionably "counterfeit
+access devices." Thanks to Sec.1029, it is not only illegal to USE
+counterfeit access devices, but it is even illegal to BUILD them.
+"Producing," "designing" "duplicating" or "assembling" blue boxes
+are all federal crimes today, and if you do this, the Secret Service
+has been charged by Congress to come after you.
+
+Automatic Teller Machines, which replicated all over America during the 1980s,
+are definitely "access devices," too, and an attempt to tamper with their
+punch-in codes and plastic bank cards falls directly under Sec. 1029.
+
+Section 1029 is remarkably elastic. Suppose you find a computer password
+in somebody's trash. That password might be a "code"--it's certainly a
+"means of account access." Now suppose you log on to a computer
+and copy some software for yourself. You've certainly obtained
+"service" (computer service) and a "thing of value" (the software).
+Suppose you tell a dozen friends about your swiped password,
+and let them use it, too. Now you're "trafficking in unauthorized
+access devices." And when the Prophet, a member of the Legion of Doom,
+passed a stolen telephone company document to Knight Lightning
+at Phrack magazine, they were both charged under Sec. 1029!
+
+There are two limitations on Section 1029. First, the offense must
+"affect interstate or foreign commerce" in order to become a matter
+of federal jurisdiction. The term "affecting commerce" is not well defined;
+but you may take it as a given that the Secret Service can take an interest
+if you've done most anything that happens to cross a state line.
+State and local police can be touchy about their jurisdictions,
+and can sometimes be mulish when the feds show up. But when it comes
+to computer-crime, the local police are pathetically grateful
+for federal help--in fact they complain that they can't get enough of it.
+If you're stealing long-distance service, you're almost certainly crossing
+state lines, and you're definitely "affecting the interstate commerce"
+of the telcos. And if you're abusing credit cards by ordering stuff
+out of glossy catalogs from, say, Vermont, you're in for it.
+
+The second limitation is money. As a rule, the feds don't pursue
+penny-ante offenders. Federal judges will dismiss cases that appear
+to waste their time. Federal crimes must be serious; Section 1029
+specifies a minimum loss of a thousand dollars.
+
+We now come to the very next section of Title 18, which is Section 1030,
+"Fraud and related activity in connection with computers." This statute
+gives the Secret Service direct jurisdiction over acts of computer intrusion.
+On the face of it, the Secret Service would now seem to command the field.
+Section 1030, however, is nowhere near so ductile as Section 1029.
+
+The first annoyance is Section 1030(d), which reads:
+
+"(d) The United States Secret Service shall,
+IN ADDITION TO ANY OTHER AGENCY HAVING SUCH AUTHORITY,
+have the authority to investigate offenses under this section.
+Such authority of the United States Secret Service shall be
+exercised in accordance with an agreement which shall be entered
+into by the Secretary of the Treasury AND THE ATTORNEY GENERAL."
+(Author's italics.) [Represented by capitals.]
+
+The Secretary of the Treasury is the titular head of the Secret Service,
+while the Attorney General is in charge of the FBI. In Section (d),
+Congress shrugged off responsibility for the computer-crime turf-battle
+between the Service and the Bureau, and made them fight it out all
+by themselves. The result was a rather dire one for the Secret Service,
+for the FBI ended up with exclusive jurisdiction over computer break-ins
+having to do with national security, foreign espionage, federally insured
+banks, and U.S. military bases, while retaining joint jurisdiction over
+all the other computer intrusions. Essentially, when it comes to Section 1030,
+the FBI not only gets the real glamor stuff for itself, but can peer over the
+shoulder of the Secret Service and barge in to meddle whenever it suits them.
+
+The second problem has to do with the dicey term
+"Federal interest computer." Section 1030(a)(2)
+makes it illegal to "access a computer without authorization"
+if that computer belongs to a financial institution or an issuer
+of credit cards (fraud cases, in other words). Congress was quite
+willing to give the Secret Service jurisdiction over
+money-transferring computers, but Congress balked at
+letting them investigate any and all computer intrusions.
+Instead, the USSS had to settle for the money machines
+and the "Federal interest computers." A "Federal interest computer"
+is a computer which the government itself owns, or is using.
+Large networks of interstate computers, linked over state lines,
+are also considered to be of "Federal interest." (This notion of
+"Federal interest" is legally rather foggy and has never been
+clearly defined in the courts. The Secret Service has never yet
+had its hand slapped for investigating computer break-ins that were NOT
+of "Federal interest," but conceivably someday this might happen.)
+
+So the Secret Service's authority over "unauthorized access"
+to computers covers a lot of territory, but by no means the
+whole ball of cyberspatial wax. If you are, for instance,
+a LOCAL computer retailer, or the owner of a LOCAL bulletin
+board system, then a malicious LOCAL intruder can break in,
+crash your system, trash your files and scatter viruses,
+and the U.S. Secret Service cannot do a single thing about it.
+
+At least, it can't do anything DIRECTLY. But the Secret Service
+will do plenty to help the local people who can.
+
+The FBI may have dealt itself an ace off the bottom of the deck
+when it comes to Section 1030; but that's not the whole story;
+that's not the street. What's Congress thinks is one thing,
+and Congress has been known to change its mind. The REAL
+turf-struggle is out there in the streets where it's happening.
+If you're a local street-cop with a computer problem,
+the Secret Service wants you to know where you can find
+the real expertise. While the Bureau crowd are off having
+their favorite shoes polished--(wing-tips)--and making derisive
+fun of the Service's favorite shoes--("pansy-ass tassels")--
+the tassel-toting Secret Service has a crew of ready-and-able
+hacker-trackers installed in the capital of every state in the Union.
+Need advice? They'll give you advice, or at least point you in
+the right direction. Need training? They can see to that, too.
+
+If you're a local cop and you call in the FBI, the FBI
+(as is widely and slanderously rumored) will order you around
+like a coolie, take all the credit for your busts,
+and mop up every possible scrap of reflected glory.
+The Secret Service, on the other hand, doesn't brag a lot.
+They're the quiet types. VERY quiet. Very cool. Efficient.
+High-tech. Mirrorshades, icy stares, radio ear-plugs,
+an Uzi machine-pistol tucked somewhere in that well-cut jacket.
+American samurai, sworn to give their lives to protect our President.
+"The granite agents." Trained in martial arts, absolutely fearless.
+Every single one of 'em has a top-secret security clearance.
+Something goes a little wrong, you're not gonna hear any whining
+and moaning and political buck-passing out of these guys.
+
+The facade of the granite agent is not, of course, the reality.
+Secret Service agents are human beings. And the real glory
+in Service work is not in battling computer crime--not yet,
+anyway--but in protecting the President. The real glamour
+of Secret Service work is in the White House Detail.
+If you're at the President's side, then the kids and the wife
+see you on television; you rub shoulders with the most powerful
+people in the world. That's the real heart of Service work,
+the number one priority. More than one computer investigation
+has stopped dead in the water when Service agents vanished at
+the President's need.
+
+There's romance in the work of the Service. The intimate access
+to circles of great power; the esprit-de-corps of a highly trained
+and disciplined elite; the high responsibility of defending the
+Chief Executive; the fulfillment of a patriotic duty. And as police
+work goes, the pay's not bad. But there's squalor in Service work, too.
+You may get spat upon by protesters howling abuse--and if they get violent,
+if they get too close, sometimes you have to knock one of them down--
+discreetly.
+
+The real squalor in Service work is drudgery such as "the quarterlies,"
+traipsing out four times a year, year in, year out, to interview the various
+pathetic wretches, many of them in prisons and asylums, who have seen fit
+to threaten the President's life. And then there's the grinding stress
+of searching all those faces in the endless bustling crowds, looking for
+hatred, looking for psychosis, looking for the tight, nervous face
+of an Arthur Bremer, a Squeaky Fromme, a Lee Harvey Oswald.
+It's watching all those grasping, waving hands for sudden movements,
+while your ears strain at your radio headphone for the long-rehearsed
+cry of "Gun!"
+
+It's poring, in grinding detail, over the biographies of every rotten
+loser who ever shot at a President. It's the unsung work of the
+Protective Research Section, who study scrawled, anonymous death threats
+with all the meticulous tools of anti-forgery techniques.
+
+And it's maintaining the hefty computerized files on anyone
+who ever threatened the President's life. Civil libertarians
+have become increasingly concerned at the Government's use
+of computer files to track American citizens--but the
+Secret Service file of potential Presidential assassins,
+which has upward of twenty thousand names, rarely causes
+a peep of protest. If you EVER state that you intend to
+kill the President, the Secret Service will want to know
+and record who you are, where you are, what you are,
+and what you're up to. If you're a serious threat--
+if you're officially considered "of protective interest"--
+then the Secret Service may well keep tabs on you
+for the rest of your natural life.
+
+Protecting the President has first call on all the Service's resources.
+But there's a lot more to the Service's traditions and history than
+standing guard outside the Oval Office.
+
+The Secret Service is the nation's oldest general federal
+law-enforcement agency. Compared to the Secret Service,
+the FBI are new-hires and the CIA are temps. The Secret Service
+was founded 'way back in 1865, at the suggestion of Hugh McCulloch,
+Abraham Lincoln's Secretary of the Treasury. McCulloch wanted
+a specialized Treasury police to combat counterfeiting.
+Abraham Lincoln agreed that this seemed a good idea, and,
+with a terrible irony, Abraham Lincoln was shot that
+very night by John Wilkes Booth.
+
+The Secret Service originally had nothing to do with protecting Presidents.
+They didn't take this on as a regular assignment until after the Garfield
+assassination in 1881. And they didn't get any Congressional money for it
+until President McKinley was shot in 1901. The Service was originally
+designed for one purpose: destroying counterfeiters.
+
+#
+
+There are interesting parallels between the Service's
+nineteenth-century entry into counterfeiting,
+and America's twentieth-century entry into computer-crime.
+
+In 1865, America's paper currency was a terrible muddle.
+Security was drastically bad. Currency was printed on the spot
+by local banks in literally hundreds of different designs.
+No one really knew what the heck a dollar bill was supposed to look like.
+Bogus bills passed easily. If some joker told you that a one-dollar bill
+from the Railroad Bank of Lowell, Massachusetts had a woman leaning on
+a shield, with a locomotive, a cornucopia, a compass, various agricultural
+implements, a railroad bridge, and some factories, then you pretty much had
+to take his word for it. (And in fact he was telling the truth!)
+
+SIXTEEN HUNDRED local American banks designed and printed their own
+paper currency, and there were no general standards for security.
+Like a badly guarded node in a computer network, badly designed bills
+were easy to fake, and posed a security hazard for the entire monetary system.
+
+No one knew the exact extent of the threat to the currency.
+There were panicked estimates that as much as a third of
+the entire national currency was faked. Counterfeiters--
+known as "boodlers" in the underground slang of the time--
+were mostly technically skilled printers who had gone to the bad.
+Many had once worked printing legitimate currency.
+Boodlers operated in rings and gangs. Technical experts
+engraved the bogus plates--commonly in basements in New York City.
+Smooth confidence men passed large wads of high-quality,
+high-denomination fakes, including the really sophisticated stuff--
+government bonds, stock certificates, and railway shares.
+Cheaper, botched fakes were sold or sharewared to low-level
+gangs of boodler wannabes. (The really cheesy lowlife boodlers
+merely upgraded real bills by altering face values,
+changing ones to fives, tens to hundreds, and so on.)
+
+The techniques of boodling were little-known and regarded
+with a certain awe by the mid- nineteenth-century public.
+The ability to manipulate the system for rip-off seemed
+diabolically clever. As the skill and daring of the
+boodlers increased, the situation became intolerable.
+The federal government stepped in, and began offering
+its own federal currency, which was printed in fancy green ink,
+but only on the back--the original "greenbacks." And at first,
+the improved security of the well-designed, well-printed
+federal greenbacks seemed to solve the problem; but then
+the counterfeiters caught on. Within a few years things were
+worse than ever: a CENTRALIZED system where ALL security was bad!
+
+The local police were helpless. The Government tried offering
+blood money to potential informants, but this met with little success.
+Banks, plagued by boodling, gave up hope of police help and hired
+private security men instead. Merchants and bankers queued up
+by the thousands to buy privately-printed manuals on currency security,
+slim little books like Laban Heath's INFALLIBLE GOVERNMENT
+COUNTERFEIT DETECTOR. The back of the book offered Laban Heath's
+patent microscope for five bucks.
+
+Then the Secret Service entered the picture. The first agents
+were a rough and ready crew. Their chief was one William P. Wood,
+a former guerilla in the Mexican War who'd won a reputation busting
+contractor fraudsters for the War Department during the Civil War.
+Wood, who was also Keeper of the Capital Prison, had a sideline
+as a counterfeiting expert, bagging boodlers for the federal bounty money.
+
+Wood was named Chief of the new Secret Service in July 1865.
+There were only ten Secret Service agents in all: Wood himself,
+a handful who'd worked for him in the War Department, and a few
+former private investigators--counterfeiting experts--whom Wood
+had won over to public service. (The Secret Service of 1865 was
+much the size of the Chicago Computer Fraud Task Force or the
+Arizona Racketeering Unit of 1990.) These ten "Operatives"
+had an additional twenty or so "Assistant Operatives" and "Informants."
+Besides salary and per diem, each Secret Service employee received
+a whopping twenty-five dollars for each boodler he captured.
+
+Wood himself publicly estimated that at least HALF of America's currency
+was counterfeit, a perhaps pardonable perception. Within a year the
+Secret Service had arrested over 200 counterfeiters. They busted about
+two hundred boodlers a year for four years straight.
+
+Wood attributed his success to travelling fast and light, hitting the
+bad-guys hard, and avoiding bureaucratic baggage. "Because my raids
+were made without military escort and I did not ask the assistance
+of state officers, I surprised the professional counterfeiter."
+
+Wood's social message to the once-impudent boodlers bore an eerie ring
+of Sundevil: "It was also my purpose to convince such characters that
+it would no longer be healthy for them to ply their vocation without
+being handled roughly, a fact they soon discovered."
+
+William P. Wood, the Secret Service's guerilla pioneer,
+did not end well. He succumbed to the lure of aiming for
+the really big score. The notorious Brockway Gang of New York City,
+headed by William E. Brockway, the "King of the Counterfeiters,"
+had forged a number of government bonds. They'd passed these
+brilliant fakes on the prestigious Wall Street investment
+firm of Jay Cooke and Company. The Cooke firm were frantic
+and offered a huge reward for the forgers' plates.
+
+Laboring diligently, Wood confiscated the plates
+(though not Mr. Brockway) and claimed the reward.
+But the Cooke company treacherously reneged.
+Wood got involved in a down-and-dirty lawsuit
+with the Cooke capitalists. Wood's boss,
+Secretary of the Treasury McCulloch, felt that
+Wood's demands for money and glory were unseemly,
+and even when the reward money finally came through,
+McCulloch refused to pay Wood anything.
+Wood found himself mired in a seemingly endless
+round of federal suits and Congressional lobbying.
+
+Wood never got his money. And he lost his job to boot.
+He resigned in 1869.
+
+Wood's agents suffered, too. On May 12, 1869, the second Chief
+of the Secret Service took over, and almost immediately fired
+most of Wood's pioneer Secret Service agents: Operatives,
+Assistants and Informants alike. The practice of receiving $25
+per crook was abolished. And the Secret Service began the long,
+uncertain process of thorough professionalization.
+
+Wood ended badly. He must have felt stabbed in the back.
+In fact his entire organization was mangled.
+
+On the other hand, William P. Wood WAS the first head of the Secret Service.
+William Wood was the pioneer. People still honor his name. Who remembers
+the name of the SECOND head of the Secret Service?
+
+As for William Brockway (also known as "Colonel Spencer"),
+he was finally arrested by the Secret Service in 1880.
+He did five years in prison, got out, and was still boodling
+at the age of seventy-four.
+
+#
+
+Anyone with an interest in Operation Sundevil--
+or in American computer-crime generally--
+could scarcely miss the presence of Gail Thackeray,
+Assistant Attorney General of the State of Arizona.
+Computer-crime training manuals often cited
+Thackeray's group and her work; she was the
+highest-ranking state official to specialize
+in computer-related offenses. Her name had been
+on the Sundevil press release (though modestly ranked
+well after the local federal prosecuting attorney and
+the head of the Phoenix Secret Service office).
+
+As public commentary, and controversy, began to mount
+about the Hacker Crackdown, this Arizonan state official
+began to take a higher and higher public profile.
+Though uttering almost nothing specific about
+the Sundevil operation itself, she coined some
+of the most striking soundbites of the growing propaganda war:
+"Agents are operating in good faith, and I don't think
+you can say that for the hacker community," was one.
+Another was the memorable "I am not a mad dog prosecutor"
+(Houston Chronicle, Sept 2, 1990.) In the meantime,
+the Secret Service maintained its usual extreme discretion;
+the Chicago Unit, smarting from the backlash
+of the Steve Jackson scandal, had gone completely to earth.
+
+As I collated my growing pile of newspaper clippings,
+Gail Thackeray ranked as a comparative fount of public
+knowledge on police operations.
+
+I decided that I had to get to know Gail Thackeray.
+I wrote to her at the Arizona Attorney General's Office.
+Not only did she kindly reply to me, but, to my astonishment,
+she knew very well what "cyberpunk" science fiction was.
+
+Shortly after this, Gail Thackeray lost her job.
+And I temporarily misplaced my own career as
+a science-fiction writer, to become a full-time
+computer-crime journalist. In early March, 1991,
+I flew to Phoenix, Arizona, to interview Gail Thackeray
+for my book on the hacker crackdown.
+
+#
+
+"Credit cards didn't used to cost anything to get,"
+says Gail Thackeray. "Now they cost forty bucks--
+and that's all just to cover the costs from RIP-OFF ARTISTS."
+
+Electronic nuisance criminals are parasites.
+One by one they're not much harm, no big deal.
+But they never come just one by one. They come in swarms,
+heaps, legions, sometimes whole subcultures. And they bite.
+Every time we buy a credit card today, we lose a little financial
+vitality to a particular species of bloodsucker.
+
+What, in her expert opinion, are the worst forms of electronic crime,
+I ask, consulting my notes. Is it--credit card fraud? Breaking into
+ATM bank machines? Phone-phreaking? Computer intrusions?
+Software viruses? Access-code theft? Records tampering?
+Software piracy? Pornographic bulletin boards?
+Satellite TV piracy? Theft of cable service?
+It's a long list. By the time I reach the end
+of it I feel rather depressed.
+
+"Oh no," says Gail Thackeray, leaning forward over the table,
+her whole body gone stiff with energetic indignation,
+"the biggest damage is telephone fraud. Fake sweepstakes,
+fake charities. Boiler-room con operations. You could pay off
+the national debt with what these guys steal. . . .
+They target old people, they get hold of credit ratings
+and demographics, they rip off the old and the weak."
+The words come tumbling out of her.
+
+It's low-tech stuff, your everyday boiler-room fraud.
+Grifters, conning people out of money over the phone,
+have been around for decades. This is where the word "phony" came from!
+
+It's just that it's so much EASIER now, horribly facilitated by advances
+in technology and the byzantine structure of the modern phone system.
+The same professional fraudsters do it over and over, Thackeray tells me,
+they hide behind dense onion-shells of fake companies. . . fake holding
+corporations nine or ten layers deep, registered all over the map.
+They get a phone installed under a false name in an empty safe-house.
+And then they call-forward everything out of that phone to yet
+another phone, a phone that may even be in another STATE.
+And they don't even pay the charges on their phones;
+after a month or so, they just split; set up somewhere else
+in another Podunkville with the same seedy crew of veteran phone-crooks.
+They buy or steal commercial credit card reports, slap them on the PC,
+have a program pick out people over sixty-five who pay a lot to charities.
+A whole subculture living off this, merciless folks on the con.
+
+"The `light-bulbs for the blind' people," Thackeray muses,
+with a special loathing. "There's just no end to them."
+
+We're sitting in a downtown diner in Phoenix, Arizona.
+It's a tough town, Phoenix. A state capital seeing some hard times.
+Even to a Texan like myself, Arizona state politics seem rather baroque.
+There was, and remains, endless trouble over the Martin Luther King holiday,
+the sort of stiff-necked, foot-shooting incident for which Arizona politics
+seem famous. There was Evan Mecham, the eccentric Republican millionaire
+governor who was impeached, after reducing state government to a
+ludicrous shambles. Then there was the national Keating scandal,
+involving Arizona savings and loans, in which both of Arizona's
+U.S. senators, DeConcini and McCain, played sadly prominent roles.
+
+And the very latest is the bizarre AzScam case,
+in which state legislators were videotaped,
+eagerly taking cash from an informant of the Phoenix city
+police department, who was posing as a Vegas mobster.
+
+"Oh," says Thackeray cheerfully. "These people are amateurs here,
+they thought they were finally getting to play with the big boys.
+They don't have the least idea how to take a bribe!
+It's not institutional corruption. It's not like back in Philly."
+
+Gail Thackeray was a former prosecutor in Philadelphia.
+Now she's a former assistant attorney general of the State of Arizona.
+Since moving to Arizona in 1986, she had worked under the aegis
+of Steve Twist, her boss in the Attorney General's office.
+Steve Twist wrote Arizona's pioneering computer crime laws
+and naturally took an interest in seeing them enforced.
+It was a snug niche, and Thackeray's Organized Crime and
+Racketeering Unit won a national reputation for ambition
+and technical knowledgeability. . . . Until the latest
+election in Arizona. Thackeray's boss ran for the top
+job, and lost. The victor, the new Attorney General,
+apparently went to some pains to eliminate the bureaucratic
+traces of his rival, including his pet group--Thackeray's group.
+Twelve people got their walking papers.
+
+Now Thackeray's painstakingly assembled computer lab
+sits gathering dust somewhere in the glass-and-concrete
+Attorney General's HQ on 1275 Washington Street.
+Her computer-crime books, her painstakingly garnered
+back issues of phreak and hacker zines, all bought
+at her own expense--are piled in boxes somewhere.
+The State of Arizona is simply not particularly
+interested in electronic racketeering at the moment.
+
+At the moment of our interview, Gail Thackeray,
+officially unemployed, is working out of the county
+sheriff's office, living on her savings, and prosecuting
+several cases--working 60-hour weeks, just as always--
+for no pay at all. "I'm trying to train people,"
+she mutters.
+
+Half her life seems to be spent training people--merely pointing out,
+to the naive and incredulous (such as myself) that this stuff
+is ACTUALLY GOING ON OUT THERE. It's a small world, computer crime.
+A young world. Gail Thackeray, a trim blonde Baby-Boomer who favors
+Grand Canyon white-water rafting to kill some slow time,
+is one of the world's most senior, most veteran "hacker-trackers."
+Her mentor was Donn Parker, the California think-tank theorist
+who got it all started `way back in the mid-70s, the "grandfather
+of the field," "the great bald eagle of computer crime."
+
+And what she has learned, Gail Thackeray teaches. Endlessly.
+Tirelessly. To anybody. To Secret Service agents and state police,
+at the Glynco, Georgia federal training center. To local police,
+on "roadshows" with her slide projector and notebook.
+To corporate security personnel. To journalists. To parents.
+
+Even CROOKS look to Gail Thackeray for advice.
+Phone-phreaks call her at the office. They know very
+well who she is. They pump her for information
+on what the cops are up to, how much they know.
+Sometimes whole CROWDS of phone phreaks,
+hanging out on illegal conference calls, will call Gail
+Thackeray up. They taunt her. And, as always,
+they boast. Phone-phreaks, real stone phone-phreaks,
+simply CANNOT SHUT UP. They natter on for hours.
+
+Left to themselves, they mostly talk about the intricacies
+of ripping-off phones; it's about as interesting as listening
+to hot-rodders talk about suspension and distributor-caps.
+They also gossip cruelly about each other. And when talking
+to Gail Thackeray, they incriminate themselves. "I have tapes,"
+Thackeray says coolly.
+
+Phone phreaks just talk like crazy. "Dial-Tone" out in Alabama
+has been known to spend half-an-hour simply reading stolen
+phone-codes aloud into voice-mail answering machines.
+Hundreds, thousands of numbers, recited in a monotone,
+without a break--an eerie phenomenon. When arrested,
+it's a rare phone phreak who doesn't inform at endless length
+on everybody he knows.
+
+Hackers are no better. What other group of criminals,
+she asks rhetorically, publishes newsletters and holds conventions?
+She seems deeply nettled by the sheer brazenness of this behavior,
+though to an outsider, this activity might make one wonder
+whether hackers should be considered "criminals" at all.
+Skateboarders have magazines, and they trespass a lot.
+Hot rod people have magazines and they break speed limits
+and sometimes kill people. . . .
+
+I ask her whether it would be any loss to society if phone phreaking
+and computer hacking, as hobbies, simply dried up and blew away,
+so that nobody ever did it again.
+
+She seems surprised. "No," she says swiftly. "Maybe a little. . .
+in the old days. . .the MIT stuff. . . . But there's a lot of wonderful,
+legal stuff you can do with computers now, you don't have to break into
+somebody else's just to learn. You don't have that excuse.
+You can learn all you like."
+
+Did you ever hack into a system? I ask.
+
+The trainees do it at Glynco. Just to demonstrate system vulnerabilities.
+She's cool to the notion. Genuinely indifferent.
+
+"What kind of computer do you have?"
+
+"A Compaq 286LE," she mutters.
+
+"What kind do you WISH you had?"
+
+At this question, the unmistakable light of true hackerdom flares in
+Gail Thackeray's eyes. She becomes tense, animated, the words pour out:
+"An Amiga 2000 with an IBM card and Mac emulation! The most common hacker
+machines are Amigas and Commodores. And Apples." If she had the Amiga,
+she enthuses, she could run a whole galaxy of seized computer-evidence disks
+on one convenient multifunctional machine. A cheap one, too. Not like the
+old Attorney General lab, where they had an ancient CP/M machine,
+assorted Amiga flavors and Apple flavors, a couple IBMS, all the
+utility software. . .but no Commodores. The workstations down
+at the Attorney General's are Wang dedicated word-processors.
+Lame machines tied in to an office net--though at least they get
+on- line to the Lexis and Westlaw legal data services.
+
+I don't say anything. I recognize the syndrome, though.
+This computer-fever has been running through segments of
+our society for years now. It's a strange kind of lust:
+K-hunger, Meg-hunger; but it's a shared disease;
+it can kill parties dead, as conversation spirals into
+the deepest and most deviant recesses of software releases
+and expensive peripherals. . . . The mark of the hacker beast.
+I have it too. The whole "electronic community," whatever the hell
+that is, has it. Gail Thackeray has it. Gail Thackeray is a hacker cop.
+My immediate reaction is a strong rush of indignant pity:
+WHY DOESN'T SOMEBODY BUY THIS WOMAN HER AMIGA?!
+It's not like she's asking for a Cray X-MP
+supercomputer mainframe; an Amiga's a sweet little
+cookie-box thing. We're losing zillions in organized fraud;
+prosecuting and defending a single hacker case in court can cost
+a hundred grand easy. How come nobody can come up with four lousy grand
+so this woman can do her job? For a hundred grand we could buy every
+computer cop in America an Amiga. There aren't that many of 'em.
+
+Computers. The lust, the hunger, for computers.
+The loyalty they inspire, the intense sense of possessiveness.
+The culture they have bred. I myself am sitting in downtown Phoenix,
+Arizona because it suddenly occurred to me that the police might--
+just MIGHT--come and take away my computer. The prospect of this,
+the mere IMPLIED THREAT, was unbearable. It literally changed my life.
+It was changing the lives of many others. Eventually it would change
+everybody's life.
+
+Gail Thackeray was one of the top computer-crime people in America.
+And I was just some novelist, and yet I had a better computer than hers.
+PRACTICALLY EVERYBODY I KNEW had a better computer than Gail Thackeray
+and her feeble laptop 286. It was like sending the sheriff in to clean
+up Dodge City and arming her with a slingshot cut from an old rubber tire.
+
+But then again, you don't need a howitzer to enforce the law.
+You can do a lot just with a badge. With a badge alone,
+you can basically wreak havoc, take a terrible vengeance on wrongdoers.
+Ninety percent of "computer crime investigation" is just "crime investigation:"
+names, places, dossiers, modus operandi, search warrants, victims,
+complainants, informants. . . .
+
+What will computer crime look like in ten years? Will it get better?
+Did "Sundevil" send 'em reeling back in confusion?
+
+It'll be like it is now, only worse, she tells me with perfect conviction.
+Still there in the background, ticking along, changing with the times:
+the criminal underworld. It'll be like drugs are. Like our problems
+with alcohol. All the cops and laws in the world never solved our problems
+with alcohol. If there's something people want, a certain percentage
+of them are just going to take it. Fifteen percent of the populace
+will never steal. Fifteen percent will steal most anything not nailed down.
+The battle is for the hearts and minds of the remaining seventy percent.
+
+And criminals catch on fast. If there's not "too steep a learning curve"--
+if it doesn't require a baffling amount of expertise and practice--
+then criminals are often some of the first through the gate of a
+new technology. Especially if it helps them to hide.
+They have tons of cash, criminals. The new communications tech--
+like pagers, cellular phones, faxes, Federal Express--were pioneered
+by rich corporate people, and by criminals. In the early years
+of pagers and beepers, dope dealers were so enthralled this technology
+that owing a beeper was practically prima facie evidence of cocaine dealing.
+CB radio exploded when the speed limit hit 55 and breaking the highway law
+became a national pastime. Dope dealers send cash by Federal Express,
+despite, or perhaps BECAUSE OF, the warnings in FedEx offices that tell you
+never to try this. Fed Ex uses X-rays and dogs on their mail,
+to stop drug shipments. That doesn't work very well.
+
+Drug dealers went wild over cellular phones.
+There are simple methods of faking ID on cellular phones,
+making the location of the call mobile, free of charge,
+and effectively untraceable. Now victimized cellular
+companies routinely bring in vast toll-lists of calls
+to Colombia and Pakistan.
+
+Judge Greene's fragmentation of the phone company
+is driving law enforcement nuts. Four thousand
+telecommunications companies. Fraud skyrocketing.
+Every temptation in the world available with a phone
+and a credit card number. Criminals untraceable.
+A galaxy of "new neat rotten things to do."
+
+If there were one thing Thackeray would like to have,
+it would be an effective legal end-run through this new
+fragmentation minefield.
+
+It would be a new form of electronic search warrant,
+an "electronic letter of marque" to be issued by a judge.
+It would create a new category of "electronic emergency."
+Like a wiretap, its use would be rare, but it would cut
+across state lines and force swift cooperation from all concerned.
+Cellular, phone, laser, computer network, PBXes, AT&T, Baby Bells,
+long-distance entrepreneurs, packet radio. Some document,
+some mighty court-order, that could slice through four thousand
+separate forms of corporate red-tape, and get her at once to
+the source of calls, the source of email threats and viruses,
+the sources of bomb threats, kidnapping threats. "From now on,"
+she says, "the Lindbergh baby will always die."
+
+Something that would make the Net sit still, if only for a moment.
+Something that would get her up to speed. Seven league boots.
+That's what she really needs. "Those guys move in nanoseconds
+and I'm on the Pony Express."
+
+And then, too, there's the coming international angle.
+Electronic crime has never been easy to localize,
+to tie to a physical jurisdiction. And phone-phreaks
+and hackers loathe boundaries, they jump them whenever they can.
+The English. The Dutch. And the Germans, especially the ubiquitous
+Chaos Computer Club. The Australians. They've all learned phone-phreaking
+from America. It's a growth mischief industry. The multinational
+networks are global, but governments and the police simply aren't.
+Neither are the laws. Or the legal frameworks for citizen protection.
+
+One language is global, though--English. Phone phreaks speak English;
+it's their native tongue even if they're Germans. English may have started
+in England but now it's the Net language; it might as well be called "CNNese."
+
+Asians just aren't much into phone phreaking. They're the world masters
+at organized software piracy. The French aren't into phone-phreaking either.
+The French are into computerized industrial espionage.
+
+In the old days of the MIT righteous hackerdom, crashing systems
+didn't hurt anybody. Not all that much, anyway. Not permanently.
+Now the players are more venal. Now the consequences are worse.
+Hacking will begin killing people soon. Already there are methods
+of stacking calls onto 911 systems, annoying the police, and possibly
+causing the death of some poor soul calling in with a genuine emergency.
+Hackers in Amtrak computers, or air-traffic control computers, will kill
+somebody someday. Maybe a lot of people. Gail Thackeray expects it.
+
+And the viruses are getting nastier. The "Scud" virus is the latest one out.
+It wipes hard-disks.
+
+According to Thackeray, the idea that phone-phreaks are Robin Hoods is a fraud.
+They don't deserve this repute. Basically, they pick on the weak. AT&T now
+protects itself with the fearsome ANI (Automatic Number Identification)
+trace capability. When AT&T wised up and tightened security generally,
+the phreaks drifted into the Baby Bells. The Baby Bells lashed out in 1989
+and 1990, so the phreaks switched to smaller long-distance entrepreneurs.
+Today, they are moving into locally owned PBXes and voice-mail systems,
+which are full of security holes, dreadfully easy to hack. These victims
+aren't the moneybags Sheriff of Nottingham or Bad King John, but small groups
+of innocent people who find it hard to protect themselves, and who really
+suffer from these depredations. Phone phreaks pick on the weak. They do it
+for power. If it were legal, they wouldn't do it. They don't want service,
+or knowledge, they want the thrill of power-tripping. There's plenty of
+knowledge or service around if you're willing to pay. Phone phreaks don't pay,
+they steal. It's because it is illegal that it feels like power,
+that it gratifies their vanity.
+
+I leave Gail Thackeray with a handshake at the door of her office building--
+a vast International-Style office building downtown. The Sheriff's office
+is renting part of it. I get the vague impression that quite a lot of the
+building is empty--real estate crash.
+
+In a Phoenix sports apparel store, in a downtown mall, I meet
+the "Sun Devil" himself. He is the cartoon mascot of
+Arizona State University, whose football stadium, "Sundevil,"
+is near the local Secret Service HQ--hence the name Operation Sundevil.
+The Sun Devil himself is named "Sparky." Sparky the Sun Devil is maroon
+and bright yellow, the school colors. Sparky brandishes a three-tined
+yellow pitchfork. He has a small mustache, pointed ears, a barbed tail,
+and is dashing forward jabbing the air with the pitchfork,
+with an expression of devilish glee.
+
+Phoenix was the home of Operation Sundevil. The Legion of Doom
+ran a hacker bulletin board called "The Phoenix Project."
+An Australian hacker named "Phoenix" once burrowed through
+the Internet to attack Cliff Stoll, then bragged and boasted
+about it to The New York Times. This net of coincidence
+is both odd and meaningless.
+
+The headquarters of the Arizona Attorney General, Gail Thackeray's
+former workplace, is on 1275 Washington Avenue. Many of the downtown
+streets in Phoenix are named after prominent American presidents:
+Washington, Jefferson, Madison. . . .
+
+After dark, all the employees go home to their suburbs.
+Washington, Jefferson and Madison--what would be the
+Phoenix inner city, if there were an inner city in this
+sprawling automobile-bred town--become the haunts
+of transients and derelicts. The homeless. The sidewalks
+along Washington are lined with orange trees.
+Ripe fallen fruit lies scattered like croquet balls
+on the sidewalks and gutters. No one seems to be eating them.
+I try a fresh one. It tastes unbearably bitter.
+
+The Attorney General's office, built in 1981 during the
+Babbitt administration, is a long low two-story building
+of white cement and wall-sized sheets of curtain-glass.
+Behind each glass wall is a lawyer's office, quite open
+and visible to anyone strolling by. Across the street
+is a dour government building labelled simply ECONOMIC SECURITY,
+something that has not been in great supply in the American
+Southwest lately.
+
+The offices are about twelve feet square. They feature
+tall wooden cases full of red-spined lawbooks;
+Wang computer monitors; telephones; Post-it notes galore.
+Also framed law diplomas and a general excess of bad
+Western landscape art. Ansel Adams photos are a big favorite,
+perhaps to compensate for the dismal specter of the parking lot,
+two acres of striped black asphalt, which features gravel landscaping
+and some sickly-looking barrel cacti.
+
+It has grown dark. Gail Thackeray has told me that the people
+who work late here, are afraid of muggings in the parking lot.
+It seems cruelly ironic that a woman tracing electronic racketeers
+across the interstate labyrinth of Cyberspace should fear an assault
+by a homeless derelict in the parking lot of her own workplace.
+
+Perhaps this is less than coincidence. Perhaps these two seemingly
+disparate worlds are somehow generating one another. The poor and
+disenfranchised take to the streets, while the rich and computer-equipped,
+safe in their bedrooms, chatter over their modems. Quite often the derelicts
+kick the glass out and break in to the lawyers' offices, if they see something
+they need or want badly enough.
+
+I cross the parking lot to the street behind the Attorney General's office.
+A pair of young tramps are bedding down on flattened sheets of cardboard,
+under an alcove stretching over the sidewalk. One tramp wears a
+glitter-covered T-shirt reading "CALIFORNIA" in Coca-Cola cursive.
+His nose and cheeks look chafed and swollen; they glisten with
+what seems to be Vaseline. The other tramp has a ragged long-sleeved
+shirt and lank brown hair parted in the middle. They both wear blue jeans
+coated in grime. They are both drunk.
+
+"You guys crash here a lot?" I ask them.
+
+They look at me warily. I am wearing black jeans, a black pinstriped
+suit jacket and a black silk tie. I have odd shoes and a funny haircut.
+
+"It's our first time here," says the red-nosed tramp unconvincingly.
+There is a lot of cardboard stacked here. More than any two people could use.
+
+"We usually stay at the Vinnie's down the street," says the brown-haired tramp,
+puffing a Marlboro with a meditative air, as he sprawls with his head on
+a blue nylon backpack. "The Saint Vincent's."
+
+"You know who works in that building over there?" I ask, pointing.
+
+The brown-haired tramp shrugs. "Some kind of attorneys, it says."
+
+We urge one another to take it easy. I give them five bucks.
+
+A block down the street I meet a vigorous workman who is wheeling along
+some kind of industrial trolley; it has what appears to be a tank of
+propane on it.
+
+We make eye contact. We nod politely. I walk past him. "Hey!
+Excuse me sir!" he says.
+
+"Yes?" I say, stopping and turning.
+
+"Have you seen," the guy says rapidly, "a black guy, about 6'7",
+scars on both his cheeks like this--" he gestures-- "wears a
+black baseball cap on backwards, wandering around here anyplace?"
+
+"Sounds like I don't much WANT to meet him," I say.
+
+"He took my wallet," says my new acquaintance.
+"Took it this morning. Y'know, some people would be
+SCARED of a guy like that. But I'm not scared.
+I'm from Chicago. I'm gonna hunt him down.
+We do things like that in Chicago."
+
+"Yeah?"
+
+"I went to the cops and now he's got an APB out on his ass,"
+he says with satisfaction. "You run into him, you let me know."
+
+"Okay," I say. "What is your name, sir?"
+
+"Stanley. . . ."
+
+"And how can I reach you?"
+
+"Oh," Stanley says, in the same rapid voice,
+"you don't have to reach, uh, me.
+You can just call the cops. Go straight to the cops."
+He reaches into a pocket and pulls out a greasy piece of pasteboard.
+"See, here's my report on him."
+
+I look. The "report," the size of an index card, is labelled PRO-ACT:
+Phoenix Residents Opposing Active Crime Threat. . . . or is it
+Organized Against Crime Threat? In the darkening street it's hard
+to read. Some kind of vigilante group? Neighborhood watch?
+I feel very puzzled.
+
+"Are you a police officer, sir?"
+
+He smiles, seems very pleased by the question.
+
+"No," he says.
+
+"But you are a `Phoenix Resident?'"
+
+"Would you believe a homeless person," Stanley says.
+
+"Really? But what's with the. . . ." For the first time I take a close look
+at Stanley's trolley. It's a rubber-wheeled thing of industrial metal,
+but the device I had mistaken for a tank of propane is in fact a water-cooler.
+Stanley also has an Army duffel-bag, stuffed tight as a sausage with clothing
+or perhaps a tent, and, at the base of his trolley, a cardboard box and a
+battered leather briefcase.
+
+"I see," I say, quite at a loss. For the first time I notice that Stanley
+has a wallet. He has not lost his wallet at all. It is in his back pocket
+and chained to his belt. It's not a new wallet. It seems to have seen
+a lot of wear.
+
+"Well, you know how it is, brother," says Stanley.
+Now that I know that he is homeless--A POSSIBLE
+THREAT--my entire perception of him has changed
+in an instant. His speech, which once seemed just
+bright and enthusiastic, now seems to have a
+dangerous tang of mania. "I have to do this!"
+he assures me. "Track this guy down. . . .
+It's a thing I do. . . you know. . .to keep myself together!"
+He smiles, nods, lifts his trolley by its decaying rubber handgrips.
+
+"Gotta work together, y'know," Stanley booms, his face alight
+with cheerfulness, "the police can't do everything!"
+The gentlemen I met in my stroll in downtown Phoenix
+are the only computer illiterates in this book.
+To regard them as irrelevant, however, would be a grave mistake.
+
+As computerization spreads across society, the populace at large
+is subjected to wave after wave of future shock. But, as a
+necessary converse, the "computer community" itself is subjected
+to wave after wave of incoming computer illiterates.
+How will those currently enjoying America's digital bounty regard,
+and treat, all this teeming refuse yearning to breathe free?
+Will the electronic frontier be another Land of Opportunity--
+or an armed and monitored enclave, where the disenfranchised
+snuggle on their cardboard at the locked doors of our houses of justice?
+
+Some people just don't get along with computers. They can't read.
+They can't type. They just don't have it in their heads to master
+arcane instructions in wirebound manuals. Somewhere, the process
+of computerization of the populace will reach a limit. Some people--
+quite decent people maybe, who might have thrived in any other situation--
+will be left irretrievably outside the bounds. What's to be done with
+these people, in the bright new shiny electroworld? How will they
+be regarded, by the mouse-whizzing masters of cyberspace? With contempt?
+Indifference? Fear?
+
+In retrospect, it astonishes me to realize how quickly poor Stanley
+became a perceived threat. Surprise and fear are closely allied feelings.
+And the world of computing is full of surprises.
+
+I met one character in the streets of Phoenix whose role in this book
+is supremely and directly relevant. That personage was Stanley's giant
+thieving scarred phantom. This phantasm is everywhere in this book.
+He is the specter haunting cyberspace.
+
+Sometimes he's a maniac vandal ready to smash the phone system
+for no sane reason at all. Sometimes he's a fascist fed,
+coldly programming his mighty mainframes to destroy our Bill of Rights.
+Sometimes he's a telco bureaucrat, covertly conspiring to register all modems
+in the service of an Orwellian surveillance regime. Mostly, though,
+this fearsome phantom is a "hacker." He's strange, he doesn't belong,
+he's not authorized, he doesn't smell right, he's not keeping his proper place,
+he's not one of us. The focus of fear is the hacker, for much the same
+reasons that Stanley's fancied assailant is black.
+
+Stanley's demon can't go away, because he doesn't exist.
+Despite singleminded and tremendous effort, he can't be arrested,
+sued, jailed, or fired. The only constructive way to do ANYTHING
+about him is to learn more about Stanley himself. This learning process
+may be repellent, it may be ugly, it may involve grave elements of paranoiac
+confusion, but it's necessary. Knowing Stanley requires something more
+than class-crossing condescension. It requires more than steely
+legal objectivity. It requires human compassion and sympathy.
+
+To know Stanley is to know his demon. If you know the other guy's demon,
+then maybe you'll come to know some of your own. You'll be able to
+separate reality from illusion. And then you won't do your cause,
+and yourself, more harm than good. Like poor damned Stanley from Chicago did.
+
+#
+
+The Federal Computer Investigations Committee (FCIC) is the most important
+and influential organization in the realm of American computer-crime.
+Since the police of other countries have largely taken their computer-crime
+cues from American methods, the FCIC might well be called the most important
+computer crime group in the world.
+
+It is also, by federal standards, an organization of great unorthodoxy.
+State and local investigators mix with federal agents. Lawyers,
+financial auditors and computer-security programmers trade notes
+with street cops. Industry vendors and telco security people show up
+to explain their gadgetry and plead for protection and justice.
+Private investigators, think-tank experts and industry pundits throw in
+their two cents' worth. The FCIC is the antithesis of a formal bureaucracy.
+
+Members of the FCIC are obscurely proud of this fact; they recognize their
+group as aberrant, but are entirely convinced that this, for them,
+outright WEIRD behavior is nevertheless ABSOLUTELY NECESSARY
+to get their jobs done.
+
+FCIC regulars --from the Secret Service, the FBI, the IRS,
+the Department of Labor, the offices of federal attorneys,
+state police, the Air Force, from military intelligence--
+often attend meetings, held hither and thither across the country,
+at their own expense. The FCIC doesn't get grants. It doesn't
+charge membership fees. It doesn't have a boss. It has no headquarters--
+just a mail drop in Washington DC, at the Fraud Division of the Secret Service.
+It doesn't have a budget. It doesn't have schedules. It meets three times
+a year--sort of. Sometimes it issues publications, but the FCIC
+has no regular publisher, no treasurer, not even a secretary.
+There are no minutes of FCIC meetings. Non-federal people are considered
+"non-voting members," but there's not much in the way of elections.
+There are no badges, lapel pins or certificates of membership.
+Everyone is on a first-name basis. There are about forty of them.
+Nobody knows how many, exactly. People come, people go--
+sometimes people "go" formally but still hang around anyway.
+Nobody has ever exactly figured out what "membership" of this
+"Committee" actually entails.
+
+Strange as this may seem to some, to anyone familiar with the social world
+of computing, the "organization" of the FCIC is very recognizable.
+
+For years now, economists and management theorists have speculated
+that the tidal wave of the information revolution would destroy rigid,
+pyramidal bureaucracies, where everything is top-down and
+centrally controlled. Highly trained "employees" would take on
+much greater autonomy, being self-starting, and self-motivating,
+moving from place to place, task to task, with great speed and fluidity.
+"Ad-hocracy" would rule, with groups of people spontaneously knitting
+together across organizational lines, tackling the problem at hand,
+applying intense computer-aided expertise to it, and then vanishing
+whence they came.
+
+This is more or less what has actually happened in the world of
+federal computer investigation. With the conspicuous exception
+of the phone companies, which are after all over a hundred years old,
+practically EVERY organization that plays any important role in this book
+functions just like the FCIC. The Chicago Task Force, the Arizona
+Racketeering Unit, the Legion of Doom, the Phrack crowd, the
+Electronic Frontier Foundation--they ALL look and act like "tiger teams"
+or "user's groups." They are all electronic ad-hocracies leaping up
+spontaneously to attempt to meet a need.
+
+Some are police. Some are, by strict definition, criminals.
+Some are political interest-groups. But every single group
+has that same quality of apparent spontaneity--"Hey, gang!
+My uncle's got a barn--let's put on a show!"
+
+Every one of these groups is embarrassed by this "amateurism,"
+and, for the sake of their public image in a world of non-computer people,
+they all attempt to look as stern and formal and impressive as possible.
+These electronic frontier-dwellers resemble groups of nineteenth-century
+pioneers hankering after the respectability of statehood.
+There are however, two crucial differences in the historical experience
+of these "pioneers" of the nineteeth and twenty-first centuries.
+
+First, powerful information technology DOES play into the hands of small,
+fluid, loosely organized groups. There have always been "pioneers,"
+"hobbyists," "amateurs," "dilettantes," "volunteers," "movements,"
+"users' groups" and "blue-ribbon panels of experts" around.
+But a group of this kind--when technically equipped to ship
+huge amounts of specialized information, at lightning speed,
+to its members, to government, and to the press--is simply
+a different kind of animal. It's like the difference between
+an eel and an electric eel.
+
+The second crucial change is that American society is currently
+in a state approaching permanent technological revolution.
+In the world of computers particularly, it is practically impossible
+to EVER stop being a "pioneer," unless you either drop dead or
+deliberately jump off the bus. The scene has never slowed down
+enough to become well-institutionalized. And after twenty, thirty,
+forty years the "computer revolution" continues to spread,
+to permeate new corners of society. Anything that really works
+is already obsolete.
+
+If you spend your entire working life as a "pioneer," the word "pioneer"
+begins to lose its meaning. Your way of life looks less and less like
+an introduction to something else" more stable and organized,
+and more and more like JUST THE WAY THINGS ARE. A "permanent revolution"
+is really a contradiction in terms. If "turmoil" lasts long enough,
+it simply becomes A NEW KIND OF SOCIETY--still the same game of history,
+but new players, new rules.
+
+Apply this to the world of late twentieth-century law enforcement,
+and the implications are novel and puzzling indeed. Any bureaucratic
+rulebook you write about computer-crime will be flawed when you write it,
+and almost an antique by the time it sees print. The fluidity and fast
+reactions of the FCIC give them a great advantage in this regard,
+which explains their success. Even with the best will in the world
+(which it does not, in fact, possess) it is impossible for an organization
+the size of the U.S. Federal Bureau of Investigation to get up to speed
+on the theory and practice of computer crime. If they tried to train all
+their agents to do this, it would be SUICIDAL, as they would NEVER BE ABLE
+TO DO ANYTHING ELSE.
+
+The FBI does try to train its agents in the basics of electronic crime,
+at their base in Quantico, Virginia. And the Secret Service, along with
+many other law enforcement groups, runs quite successful and well-attended
+training courses on wire fraud, business crime, and computer intrusion
+at the Federal Law Enforcement Training Center (FLETC, pronounced "fletsy")
+in Glynco, Georgia. But the best efforts of these bureaucracies does not
+remove the absolute need for a "cutting-edge mess" like the FCIC.
+
+For you see--the members of FCIC ARE the trainers of the rest
+of law enforcement. Practically and literally speaking,
+they are the Glynco computer-crime faculty by another name.
+If the FCIC went over a cliff on a bus, the U.S. law enforcement
+community would be rendered deaf dumb and blind in the world
+of computer crime, and would swiftly feel a desperate need
+to reinvent them. And this is no time to go starting from scratch.
+
+On June 11, 1991, I once again arrived in Phoenix, Arizona,
+for the latest meeting of the Federal Computer Investigations Committee.
+This was more or less the twentieth meeting of this stellar group.
+The count was uncertain, since nobody could figure out whether to
+include the meetings of "the Colluquy," which is what the FCIC
+was called in the mid-1980s before it had even managed to obtain
+the dignity of its own acronym.
+
+Since my last visit to Arizona, in May, the local AzScam bribery scandal
+had resolved itself in a general muddle of humiliation. The Phoenix chief
+of police, whose agents had videotaped nine state legislators up to no good,
+had resigned his office in a tussle with the Phoenix city council over
+the propriety of his undercover operations.
+
+The Phoenix Chief could now join Gail Thackeray and eleven of her closest
+associates in the shared experience of politically motivated unemployment.
+As of June, resignations were still continuing at the Arizona Attorney
+General's office, which could be interpreted as either a New Broom
+Sweeping Clean or a Night of the Long Knives Part II, depending on
+your point of view.
+
+The meeting of FCIC was held at the Scottsdale Hilton Resort.
+Scottsdale is a wealthy suburb of Phoenix, known as "Scottsdull"
+to scoffing local trendies, but well-equipped with posh shopping-malls
+and manicured lawns, while conspicuously undersupplied with homeless derelicts.
+The Scottsdale Hilton Resort was a sprawling hotel in postmodern
+crypto-Southwestern style. It featured a "mission bell tower"
+plated in turquoise tile and vaguely resembling a Saudi minaret.
+
+Inside it was all barbarically striped Santa Fe Style decor.
+There was a health spa downstairs and a large oddly-shaped
+pool in the patio. A poolside umbrella-stand offered Ben and Jerry's
+politically correct Peace Pops.
+
+I registered as a member of FCIC, attaining a handy discount rate,
+then went in search of the Feds. Sure enough, at the back of the
+hotel grounds came the unmistakable sound of Gail Thackeray
+holding forth.
+
+Since I had also attended the Computers Freedom and Privacy conference
+(about which more later), this was the second time I had seen Thackeray
+in a group of her law enforcement colleagues. Once again I was struck
+by how simply pleased they seemed to see her. It was natural that she'd
+get SOME attention, as Gail was one of two women in a group of some thirty men;
+but there was a lot more to it than that.
+
+Gail Thackeray personifies the social glue of the FCIC. They could give
+a damn about her losing her job with the Attorney General. They were sorry
+about it, of course, but hell, they'd all lost jobs. If they were the kind
+of guys who liked steady boring jobs, they would never have gotten into
+computer work in the first place.
+
+I wandered into her circle and was immediately introduced to five strangers.
+The conditions of my visit at FCIC were reviewed. I would not quote
+anyone directly. I would not tie opinions expressed to the agencies
+of the attendees. I would not (a purely hypothetical example)
+report the conversation of a guy from the Secret Service talking
+quite civilly to a guy from the FBI, as these two agencies NEVER
+talk to each other, and the IRS (also present, also hypothetical)
+NEVER TALKS TO ANYBODY.
+
+Worse yet, I was forbidden to attend the first conference. And I didn't.
+I have no idea what the FCIC was up to behind closed doors that afternoon.
+I rather suspect that they were engaging in a frank and thorough confession
+of their errors, goof-ups and blunders, as this has been a feature of every
+FCIC meeting since their legendary Memphis beer-bust of 1986. Perhaps the
+single greatest attraction of FCIC is that it is a place where you can go,
+let your hair down, and completely level with people who actually comprehend
+what you are talking about. Not only do they understand you, but they
+REALLY PAY ATTENTION, they are GRATEFUL FOR YOUR INSIGHTS, and they
+FORGIVE YOU, which in nine cases out of ten is something even your
+boss can't do, because as soon as you start talking "ROM," "BBS,"
+or "T-1 trunk," his eyes glaze over.
+
+I had nothing much to do that afternoon. The FCIC were beavering away
+in their conference room. Doors were firmly closed, windows too dark
+to peer through. I wondered what a real hacker, a computer intruder,
+would do at a meeting like this.
+
+The answer came at once. He would "trash" the place. Not reduce the place
+to trash in some orgy of vandalism; that's not the use of the term in the
+hacker milieu. No, he would quietly EMPTY THE TRASH BASKETS and silently
+raid any valuable data indiscreetly thrown away.
+
+Journalists have been known to do this. (Journalists hunting information
+have been known to do almost every single unethical thing that hackers
+have ever done. They also throw in a few awful techniques all their own.)
+The legality of `trashing' is somewhat dubious but it is not in fact
+flagrantly illegal. It was, however, absurd to contemplate trashing the FCIC.
+These people knew all about trashing. I wouldn't last fifteen seconds.
+
+The idea sounded interesting, though. I'd been hearing a lot about
+the practice lately. On the spur of the moment, I decided I would try
+trashing the office ACROSS THE HALL from the FCIC, an area which had
+nothing to do with the investigators.
+
+The office was tiny; six chairs, a table. . . . Nevertheless, it was open,
+so I dug around in its plastic trash can.
+
+To my utter astonishment, I came up with the torn scraps of a SPRINT
+long-distance phone bill. More digging produced a bank statement
+and the scraps of a hand-written letter, along with gum, cigarette ashes,
+candy wrappers and a day-old-issue of USA TODAY.
+
+The trash went back in its receptacle while the scraps of data went into
+my travel bag. I detoured through the hotel souvenir shop for some
+Scotch tape and went up to my room.
+
+Coincidence or not, it was quite true. Some poor soul had, in fact,
+thrown a SPRINT bill into the hotel's trash. Date May 1991,
+total amount due: $252.36. Not a business phone, either,
+but a residential bill, in the name of someone called Evelyn
+(not her real name). Evelyn's records showed a ## PAST DUE BILL ##!
+Here was her nine-digit account ID. Here was a stern computer-printed warning:
+
+"TREAT YOUR FONCARD AS YOU WOULD ANY CREDIT CARD. TO SECURE AGAINST FRAUD,
+NEVER GIVE YOUR FONCARD NUMBER OVER THE PHONE UNLESS YOU INITIATED THE CALL.
+IF YOU RECEIVE SUSPICIOUS CALLS PLEASE NOTIFY CUSTOMER SERVICE IMMEDIATELY!"
+
+I examined my watch. Still plenty of time left for the FCIC to carry on.
+I sorted out the scraps of Evelyn's SPRINT bill and re-assembled them with
+fresh Scotch tape. Here was her ten-digit FONCARD number. Didn't seem
+to have the ID number necessary to cause real fraud trouble.
+
+I did, however, have Evelyn's home phone number. And the phone numbers
+for a whole crowd of Evelyn's long-distance friends and acquaintances.
+In San Diego, Folsom, Redondo, Las Vegas, La Jolla, Topeka, and Northampton
+Massachusetts. Even somebody in Australia!
+
+I examined other documents. Here was a bank statement. It was Evelyn's
+IRA account down at a bank in San Mateo California (total balance $1877.20).
+Here was a charge-card bill for $382.64. She was paying it off bit by bit.
+
+Driven by motives that were completely unethical and prurient,
+I now examined the handwritten notes. They had been torn fairly
+thoroughly, so much so that it took me almost an entire five minutes
+to reassemble them.
+
+They were drafts of a love letter. They had been written on
+the lined stationery of Evelyn's employer, a biomedical company.
+Probably written at work when she should have been doing something else.
+
+"Dear Bob," (not his real name) "I guess in everyone's life there comes
+a time when hard decisions have to be made, and this is a difficult one
+for me--very upsetting. Since you haven't called me, and I don't understand
+why, I can only surmise it's because you don't want to. I thought I would
+have heard from you Friday. I did have a few unusual problems with my phone
+and possibly you tried, I hope so.
+
+"Robert, you asked me to `let go'. . . ."
+
+The first note ended. UNUSUAL PROBLEMS WITH HER PHONE?
+I looked swiftly at the next note.
+
+"Bob, not hearing from you for the whole weekend has left me very perplexed. . . ."
+
+Next draft.
+
+"Dear Bob, there is so much I don't understand right now, and I wish I did.
+I wish I could talk to you, but for some unknown reason you have elected not
+to call--this is so difficult for me to understand. . . ."
+
+She tried again.
+
+"Bob, Since I have always held you in such high esteem, I had every hope that
+we could remain good friends, but now one essential ingredient is missing--
+respect. Your ability to discard people when their purpose is served is
+appalling to me. The kindest thing you could do for me now is to leave me
+alone. You are no longer welcome in my heart or home. . . ."
+
+Try again.
+
+"Bob, I wrote a very factual note to you to say how much respect I had lost
+for you, by the way you treat people, me in particular, so uncaring and cold.
+The kindest thing you can do for me is to leave me alone entirely,
+as you are no longer welcome in my heart or home. I would appreciate it
+if you could retire your debt to me as soon as possible--I wish no link
+to you in any way. Sincerely, Evelyn."
+
+Good heavens, I thought, the bastard actually owes her money!
+I turned to the next page.
+
+"Bob: very simple. GOODBYE! No more mind games--no more fascination--
+no more coldness--no more respect for you! It's over--Finis. Evie"
+
+There were two versions of the final brushoff letter, but they read about
+the same. Maybe she hadn't sent it. The final item in my illicit and
+shameful booty was an envelope addressed to "Bob" at his home address,
+but it had no stamp on it and it hadn't been mailed.
+
+Maybe she'd just been blowing off steam because her rascal boyfriend
+had neglected to call her one weekend. Big deal. Maybe they'd kissed
+and made up, maybe she and Bob were down at Pop's Chocolate Shop now,
+sharing a malted. Sure.
+
+Easy to find out. All I had to do was call Evelyn up. With a half-clever
+story and enough brass-plated gall I could probably trick the truth out of her.
+Phone-phreaks and hackers deceive people over the phone all the time.
+It's called "social engineering." Social engineering is a very common practice
+in the underground, and almost magically effective. Human beings are almost
+always the weakest link in computer security. The simplest way to learn
+Things You Are Not Meant To Know is simply to call up and exploit the
+knowledgeable people. With social engineering, you use the bits of specialized
+knowledge you already have as a key, to manipulate people into believing
+that you are legitimate. You can then coax, flatter, or frighten them into
+revealing almost anything you want to know. Deceiving people (especially
+over the phone) is easy and fun. Exploiting their gullibility is very
+gratifying; it makes you feel very superior to them.
+
+If I'd been a malicious hacker on a trashing raid, I would now have Evelyn
+very much in my power. Given all this inside data, it wouldn't take much
+effort at all to invent a convincing lie. If I were ruthless enough,
+and jaded enough, and clever enough, this momentary indiscretion of hers--
+maybe committed in tears, who knows--could cause her a whole world of
+confusion and grief.
+
+I didn't even have to have a MALICIOUS motive. Maybe I'd be "on her side,"
+and call up Bob instead, and anonymously threaten to break both his kneecaps
+if he didn't take Evelyn out for a steak dinner pronto. It was still
+profoundly NONE OF MY BUSINESS. To have gotten this knowledge at all
+was a sordid act and to use it would be to inflict a sordid injury.
+
+To do all these awful things would require exactly zero high-tech expertise.
+All it would take was the willingness to do it and a certain amount
+of bent imagination.
+
+I went back downstairs. The hard-working FCIC, who had labored forty-five
+minutes over their schedule, were through for the day, and adjourned to the
+hotel bar. We all had a beer.
+
+I had a chat with a guy about "Isis," or rather IACIS,
+the International Association of Computer Investigation Specialists.
+They're into "computer forensics," the techniques of picking computer-
+systems apart without destroying vital evidence. IACIS, currently run
+out of Oregon, is comprised of investigators in the U.S., Canada, Taiwan
+and Ireland. "Taiwan and Ireland?" I said. Are TAIWAN and IRELAND
+really in the forefront of this stuff? Well not exactly, my informant
+admitted. They just happen to have been the first ones to have caught
+on by word of mouth. Still, the international angle counts, because this
+is obviously an international problem. Phone-lines go everywhere.
+
+There was a Mountie here from the Royal Canadian Mounted Police.
+He seemed to be having quite a good time. Nobody had flung this
+Canadian out because he might pose a foreign security risk.
+These are cyberspace cops. They still worry a lot about "jurisdictions,"
+but mere geography is the least of their troubles.
+
+NASA had failed to show. NASA suffers a lot from computer intrusions,
+in particular from Australian raiders and a well-trumpeted Chaos
+Computer Club case, and in 1990 there was a brief press flurry
+when it was revealed that one of NASA's Houston branch-exchanges
+had been systematically ripped off by a gang of phone-phreaks.
+But the NASA guys had had their funding cut. They were stripping everything.
+
+Air Force OSI, its Office of Special Investigations, is the ONLY federal
+entity dedicated full-time to computer security. They'd been expected
+to show up in force, but some of them had cancelled--a Pentagon budget pinch.
+
+As the empties piled up, the guys began joshing around and telling war-stories.
+"These are cops," Thackeray said tolerantly. "If they're not talking shop
+they talk about women and beer."
+
+I heard the story about the guy who, asked for "a copy" of a computer disk,
+PHOTOCOPIED THE LABEL ON IT. He put the floppy disk onto the glass plate
+of a photocopier. The blast of static when the copier worked completely
+erased all the real information on the disk.
+
+Some other poor souls threw a whole bag of confiscated diskettes
+into the squad-car trunk next to the police radio. The powerful radio
+signal blasted them, too.
+
+We heard a bit about Dave Geneson, the first computer prosecutor,
+a mainframe-runner in Dade County, turned lawyer. Dave Geneson
+was one guy who had hit the ground running, a signal virtue
+in making the transition to computer-crime. It was generally
+agreed that it was easier to learn the world of computers first,
+then police or prosecutorial work. You could take certain computer
+people and train 'em to successful police work--but of course they
+had to have the COP MENTALITY. They had to have street smarts.
+Patience. Persistence. And discretion. You've got to make sure
+they're not hot-shots, show-offs, "cowboys."
+
+Most of the folks in the bar had backgrounds in military intelligence,
+or drugs, or homicide. It was rudely opined that "military intelligence"
+was a contradiction in terms, while even the grisly world of homicide
+was considered cleaner than drug enforcement. One guy had been 'way
+undercover doing dope-work in Europe for four years straight.
+"I'm almost recovered now," he said deadpan, with the acid black humor
+that is pure cop. "Hey, now I can say FUCKER without putting MOTHER
+in front of it."
+
+"In the cop world," another guy said earnestly, "everything is good and bad,
+black and white. In the computer world everything is gray."
+
+One guy--a founder of the FCIC, who'd been with the group
+since it was just the Colluquy--described his own introduction
+to the field. He'd been a Washington DC homicide guy called in
+on a "hacker" case. From the word "hacker," he naturally assumed
+he was on the trail of a knife-wielding marauder, and went to the
+computer center expecting blood and a body. When he finally figured
+out what was happening there (after loudly demanding, in vain,
+that the programmers "speak English"), he called headquarters
+and told them he was clueless about computers. They told him nobody
+else knew diddly either, and to get the hell back to work.
+
+So, he said, he had proceeded by comparisons. By analogy. By metaphor.
+"Somebody broke in to your computer, huh?" Breaking and entering;
+I can understand that. How'd he get in? "Over the phone-lines."
+Harassing phone-calls, I can understand that! What we need here
+is a tap and a trace!
+
+It worked. It was better than nothing. And it worked a lot faster
+when he got hold of another cop who'd done something similar.
+And then the two of them got another, and another, and pretty soon
+the Colluquy was a happening thing. It helped a lot that everybody
+seemed to know Carlton Fitzpatrick, the data-processing trainer in Glynco.
+
+The ice broke big-time in Memphis in '86. The Colluquy had attracted
+a bunch of new guys--Secret Service, FBI, military, other feds, heavy guys.
+Nobody wanted to tell anybody anything. They suspected that if word got back
+to the home office they'd all be fired. They passed an uncomfortably
+guarded afternoon.
+
+The formalities got them nowhere. But after the formal session was over,
+the organizers brought in a case of beer. As soon as the participants
+knocked it off with the bureaucratic ranks and turf-fighting, everything
+changed. "I bared my soul," one veteran reminisced proudly. By nightfall
+they were building pyramids of empty beer-cans and doing everything
+but composing a team fight song.
+
+FCIC were not the only computer-crime people around. There was DATTA
+(District Attorneys' Technology Theft Association), though they mostly
+specialized in chip theft, intellectual property, and black-market cases.
+There was HTCIA (High Tech Computer Investigators Association),
+also out in Silicon Valley, a year older than FCIC and featuring
+brilliant people like Donald Ingraham. There was LEETAC
+(Law Enforcement Electronic Technology Assistance Committee)
+in Florida, and computer-crime units in Illinois and Maryland
+and Texas and Ohio and Colorado and Pennsylvania. But these were
+local groups. FCIC were the first to really network nationally
+and on a federal level.
+
+FCIC people live on the phone lines. Not on bulletin board systems--
+they know very well what boards are, and they know that boards aren't secure.
+Everyone in the FCIC has a voice-phone bill like you wouldn't believe.
+FCIC people have been tight with the telco people for a long time.
+Telephone cyberspace is their native habitat.
+
+FCIC has three basic sub-tribes: the trainers, the security people,
+and the investigators. That's why it's called an "Investigations
+Committee" with no mention of the term "computer-crime"--the dreaded
+"C-word." FCIC, officially, is "an association of agencies rather
+than individuals;" unofficially, this field is small enough that
+the influence of individuals and individual expertise is paramount.
+Attendance is by invitation only, and most everyone in FCIC considers
+himself a prophet without honor in his own house.
+
+Again and again I heard this, with different terms but identical
+sentiments. "I'd been sitting in the wilderness talking to myself."
+"I was totally isolated." "I was desperate." "FCIC is the best
+thing there is about computer crime in America." "FCIC is what
+really works." "This is where you hear real people telling you
+what's really happening out there, not just lawyers picking nits."
+"We taught each other everything we knew."
+
+The sincerity of these statements convinces me that this is true.
+FCIC is the real thing and it is invaluable. It's also very sharply
+at odds with the rest of the traditions and power structure
+in American law enforcement. There probably hasn't been anything
+around as loose and go-getting as the FCIC since the start of the
+U.S. Secret Service in the 1860s. FCIC people are living like
+twenty-first-century people in a twentieth-century environment,
+and while there's a great deal to be said for that, there's also
+a great deal to be said against it, and those against it happen
+to control the budgets.
+
+I listened to two FCIC guys from Jersey compare life histories.
+One of them had been a biker in a fairly heavy-duty gang in the 1960s.
+"Oh, did you know so-and-so?" said the other guy from Jersey.
+"Big guy, heavyset?"
+
+"Yeah, I knew him."
+
+"Yeah, he was one of ours. He was our plant in the gang."
+
+"Really? Wow! Yeah, I knew him. Helluva guy."
+
+Thackeray reminisced at length about being tear-gassed blind
+in the November 1969 antiwar protests in Washington Circle,
+covering them for her college paper. "Oh yeah, I was there,"
+said another cop. "Glad to hear that tear gas hit somethin'.
+Haw haw haw." He'd been so blind himself, he confessed,
+that later that day he'd arrested a small tree.
+
+FCIC are an odd group, sifted out by coincidence and necessity,
+and turned into a new kind of cop. There are a lot of specialized
+cops in the world--your bunco guys, your drug guys, your tax guys,
+but the only group that matches FCIC for sheer isolation are probably
+the child-pornography people. Because they both deal with conspirators
+who are desperate to exchange forbidden data and also desperate to hide;
+and because nobody else in law enforcement even wants to hear about it.
+
+FCIC people tend to change jobs a lot. They tend not to get the equipment
+and training they want and need. And they tend to get sued quite often.
+
+As the night wore on and a band set up in the bar, the talk grew darker.
+Nothing ever gets done in government, someone opined, until there's
+a DISASTER. Computing disasters are awful, but there's no denying
+that they greatly help the credibility of FCIC people. The Internet Worm,
+for instance. "For years we'd been warning about that--but it's nothing
+compared to what's coming." They expect horrors, these people.
+They know that nothing will really get done until there is a horror.
+
+#
+
+Next day we heard an extensive briefing from a guy who'd been a computer cop,
+gotten into hot water with an Arizona city council, and now installed
+computer networks for a living (at a considerable rise in pay).
+He talked about pulling fiber-optic networks apart.
+
+Even a single computer, with enough peripherals, is a literal
+"network"--a bunch of machines all cabled together, generally
+with a complexity that puts stereo units to shame. FCIC people
+invent and publicize methods of seizing computers and maintaining
+their evidence. Simple things, sometimes, but vital rules of thumb
+for street cops, who nowadays often stumble across a busy computer
+in the midst of a drug investigation or a white-collar bust.
+For instance: Photograph the system before you touch it.
+Label the ends of all the cables before you detach anything.
+"Park" the heads on the disk drives before you move them.
+Get the diskettes. Don't put the diskettes in magnetic fields.
+Don't write on diskettes with ballpoint pens. Get the manuals.
+Get the printouts. Get the handwritten notes. Copy data before
+you look at it, and then examine the copy instead of the original.
+
+Now our lecturer distributed copied diagrams of a typical LAN
+or "Local Area Network", which happened to be out of Connecticut.
+ONE HUNDRED AND FIFTY-NINE desktop computers, each with its own
+peripherals. Three "file servers." Five "star couplers"
+each with thirty-two ports. One sixteen-port coupler
+off in the corner office. All these machines talking to each other,
+distributing electronic mail, distributing software, distributing,
+quite possibly, criminal evidence. All linked by high-capacity
+fiber-optic cable. A bad guy--cops talk a about "bad guys"
+--might be lurking on PC #47 lot or #123 and distributing
+his ill doings onto some dupe's "personal" machine in
+another office--or another floor--or, quite possibly,
+two or three miles away! Or, conceivably, the evidence might
+be "data-striped"--split up into meaningless slivers stored,
+one by one, on a whole crowd of different disk drives.
+
+The lecturer challenged us for solutions. I for one was utterly clueless.
+As far as I could figure, the Cossacks were at the gate; there were probably
+more disks in this single building than were seized during the entirety
+of Operation Sundevil.
+
+"Inside informant," somebody said. Right. There's always the human angle,
+something easy to forget when contemplating the arcane recesses of high
+technology. Cops are skilled at getting people to talk, and computer people,
+given a chair and some sustained attention, will talk about their computers
+till their throats go raw. There's a case on record of a single question--
+"How'd you do it?"--eliciting a forty-five-minute videotaped confession
+from a computer criminal who not only completely incriminated himself
+but drew helpful diagrams.
+
+Computer people talk. Hackers BRAG. Phone-phreaks
+talk PATHOLOGICALLY--why else are they stealing phone-codes,
+if not to natter for ten hours straight to their friends
+on an opposite seaboard? Computer-literate people do
+in fact possess an arsenal of nifty gadgets and techniques
+that would allow them to conceal all kinds of exotic skullduggery,
+and if they could only SHUT UP about it, they could probably
+get away with all manner of amazing information-crimes.
+But that's just not how it works--or at least,
+that's not how it's worked SO FAR.
+
+Most every phone-phreak ever busted has swiftly implicated his mentors,
+his disciples, and his friends. Most every white-collar computer-criminal,
+smugly convinced that his clever scheme is bulletproof, swiftly learns
+otherwise when, for the first time in his life, an actual no-kidding
+policeman leans over, grabs the front of his shirt, looks him right
+in the eye and says: "All right, ASSHOLE--you and me are going downtown!"
+All the hardware in the world will not insulate your nerves from
+these actual real-life sensations of terror and guilt.
+
+Cops know ways to get from point A to point Z without thumbing
+through every letter in some smart-ass bad-guy's alphabet.
+Cops know how to cut to the chase. Cops know a lot of things
+other people don't know.
+
+Hackers know a lot of things other people don't know, too.
+Hackers know, for instance, how to sneak into your computer
+through the phone-lines. But cops can show up RIGHT ON YOUR DOORSTEP
+and carry off YOU and your computer in separate steel boxes.
+A cop interested in hackers can grab them and grill them.
+A hacker interested in cops has to depend on hearsay,
+underground legends, and what cops are willing to publicly reveal.
+And the Secret Service didn't get named "the SECRET Service"
+because they blab a lot.
+
+Some people, our lecturer informed us, were under the mistaken
+impression that it was "impossible" to tap a fiber-optic line.
+Well, he announced, he and his son had just whipped up a
+fiber-optic tap in his workshop at home. He passed it around
+the audience, along with a circuit-covered LAN plug-in card
+so we'd all recognize one if we saw it on a case. We all had a look.
+
+The tap was a classic "Goofy Prototype"--a thumb-length rounded
+metal cylinder with a pair of plastic brackets on it.
+From one end dangled three thin black cables, each of which ended
+in a tiny black plastic cap. When you plucked the safety-cap
+off the end of a cable, you could see the glass fiber--
+no thicker than a pinhole.
+
+Our lecturer informed us that the metal cylinder was a
+"wavelength division multiplexer." Apparently, what one did
+was to cut the fiber-optic cable, insert two of the legs into
+the cut to complete the network again, and then read any passing data
+on the line by hooking up the third leg to some kind of monitor.
+Sounded simple enough. I wondered why nobody had thought of it before.
+I also wondered whether this guy's son back at the workshop had any
+teenage friends.
+
+We had a break. The guy sitting next to me was wearing a giveaway
+baseball cap advertising the Uzi submachine gun. We had a desultory chat
+about the merits of Uzis. Long a favorite of the Secret Service,
+it seems Uzis went out of fashion with the advent of the Persian Gulf War,
+our Arab allies taking some offense at Americans toting Israeli weapons.
+Besides, I was informed by another expert, Uzis jam. The equivalent weapon
+of choice today is the Heckler & Koch, manufactured in Germany.
+
+The guy with the Uzi cap was a forensic photographer. He also did a lot
+of photographic surveillance work in computer crime cases. He used to,
+that is, until the firings in Phoenix. He was now a private investigator and,
+with his wife, ran a photography salon specializing in weddings and portrait
+photos. At--one must repeat--a considerable rise in income.
+
+He was still FCIC. If you were FCIC, and you needed to talk
+to an expert about forensic photography, well, there he was,
+willing and able. If he hadn't shown up, people would have missed him.
+
+Our lecturer had raised the point that preliminary investigation
+of a computer system is vital before any seizure is undertaken.
+It's vital to understand how many machines are in there, what kinds
+there are, what kind of operating system they use, how many people
+use them, where the actual data itself is stored. To simply barge into
+an office demanding "all the computers" is a recipe for swift disaster.
+
+This entails some discreet inquiries beforehand. In fact, what it
+entails is basically undercover work. An intelligence operation.
+SPYING, not to put too fine a point on it.
+
+In a chat after the lecture, I asked an attendee whether "trashing" might work.
+
+I received a swift briefing on the theory and practice of "trash covers."
+Police "trash covers," like "mail covers" or like wiretaps, require the
+agreement of a judge. This obtained, the "trashing" work of cops is just
+like that of hackers, only more so and much better organized. So much so,
+I was informed, that mobsters in Phoenix make extensive use of locked
+garbage cans picked up by a specialty high-security trash company.
+
+In one case, a tiger team of Arizona cops had trashed a local residence
+for four months. Every week they showed up on the municipal garbage truck,
+disguised as garbagemen, and carried the contents of the suspect cans off
+to a shade tree, where they combed through the garbage--a messy task,
+especially considering that one of the occupants was undergoing
+kidney dialysis. All useful documents were cleaned, dried and examined.
+A discarded typewriter-ribbon was an especially valuable source of data,
+as its long one-strike ribbon of film contained the contents of every
+letter mailed out of the house. The letters were neatly retyped by
+a police secretary equipped with a large desk-mounted magnifying glass.
+
+There is something weirdly disquieting about the whole subject of
+"trashing"-- an unsuspected and indeed rather disgusting mode of
+deep personal vulnerability. Things that we pass by every day,
+that we take utterly for granted, can be exploited with so little work.
+Once discovered, the knowledge of these vulnerabilities tend to spread.
+
+Take the lowly subject of MANHOLE COVERS. The humble manhole cover
+reproduces many of the dilemmas of computer-security in miniature.
+Manhole covers are, of course, technological artifacts, access-points
+to our buried urban infrastructure. To the vast majority of us,
+manhole covers are invisible. They are also vulnerable. For many years now,
+the Secret Service has made a point of caulking manhole covers along all routes
+of the Presidential motorcade. This is, of course, to deter terrorists from
+leaping out of underground ambush or, more likely, planting remote-control
+car-smashing bombs beneath the street.
+
+Lately, manhole covers have seen more and more criminal exploitation,
+especially in New York City. Recently, a telco in New York City
+discovered that a cable television service had been sneaking into
+telco manholes and installing cable service alongside the phone-lines--
+WITHOUT PAYING ROYALTIES. New York companies have also suffered a
+general plague of (a) underground copper cable theft; (b) dumping of garbage,
+including toxic waste, and (c) hasty dumping of murder victims.
+
+Industry complaints reached the ears of an innovative New England
+industrial-security company, and the result was a new product known
+as "the Intimidator," a thick titanium-steel bolt with a precisely machined
+head that requires a special device to unscrew. All these "keys" have registered
+serial numbers kept on file with the manufacturer. There are now some
+thousands of these "Intimidator" bolts being sunk into American pavements
+wherever our President passes, like some macabre parody of strewn roses.
+They are also spreading as fast as steel dandelions around US military bases
+and many centers of private industry.
+
+Quite likely it has never occurred to you to peer under a manhole cover,
+perhaps climb down and walk around down there with a flashlight, just to see
+what it's like. Formally speaking, this might be trespassing, but if you
+didn't hurt anything, and didn't make an absolute habit of it, nobody would
+really care. The freedom to sneak under manholes was likely a freedom
+you never intended to exercise.
+
+You now are rather less likely to have that freedom at all.
+You may never even have missed it until you read about it here,
+but if you're in New York City it's gone, and elsewhere it's likely going.
+This is one of the things that crime, and the reaction to
+crime, does to us.
+
+The tenor of the meeting now changed as the Electronic Frontier Foundation
+arrived. The EFF, whose personnel and history will be examined in detail
+in the next chapter, are a pioneering civil liberties group who arose in
+direct response to the Hacker Crackdown of 1990.
+
+Now Mitchell Kapor, the Foundation's president, and Michael Godwin,
+its chief attorney, were confronting federal law enforcement MANO A MANO
+for the first time ever. Ever alert to the manifold uses of publicity,
+Mitch Kapor and Mike Godwin had brought their own journalist in tow:
+Robert Draper, from Austin, whose recent well-received book about
+ROLLING STONE magazine was still on the stands. Draper was on assignment
+for TEXAS MONTHLY.
+
+The Steve Jackson/EFF civil lawsuit against the Chicago Computer Fraud
+and Abuse Task Force was a matter of considerable regional interest in Texas.
+There were now two Austinite journalists here on the case. In fact,
+counting Godwin (a former Austinite and former journalist) there were
+three of us. Lunch was like Old Home Week.
+
+Later, I took Draper up to my hotel room. We had a long frank talk
+about the case, networking earnestly like a miniature freelance-journo
+version of the FCIC: privately confessing the numerous blunders
+of journalists covering the story, and trying hard to figure out
+who was who and what the hell was really going on out there.
+I showed Draper everything I had dug out of the Hilton trashcan.
+We pondered the ethics of "trashing" for a while, and agreed
+that they were dismal. We also agreed that finding a SPRINT
+bill on your first time out was a heck of a coincidence.
+
+First I'd "trashed"--and now, mere hours later, I'd bragged to someone else.
+Having entered the lifestyle of hackerdom, I was now, unsurprisingly,
+following its logic. Having discovered something remarkable through
+a surreptitious action, I of course HAD to "brag," and to drag the passing
+Draper into my iniquities. I felt I needed a witness. Otherwise nobody
+would have believed what I'd discovered. . . .
+
+Back at the meeting, Thackeray cordially, if rather tentatively,
+introduced Kapor and Godwin to her colleagues. Papers were distributed.
+Kapor took center stage. The brilliant Bostonian high-tech entrepreneur,
+normally the hawk in his own administration and quite an effective
+public speaker, seemed visibly nervous, and frankly admitted as much.
+He began by saying he consided computer-intrusion to be morally wrong,
+and that the EFF was not a "hacker defense fund," despite what had appeared
+in print. Kapor chatted a bit about the basic motivations of his group,
+emphasizing their good faith and willingness to listen and seek common ground
+with law enforcement--when, er, possible.
+
+Then, at Godwin's urging, Kapor suddenly remarked that EFF's own Internet
+machine had been "hacked" recently, and that EFF did not consider
+this incident amusing.
+
+After this surprising confession, things began to loosen up
+quite rapidly. Soon Kapor was fielding questions, parrying objections,
+challenging definitions, and juggling paradigms with something akin
+to his usual gusto.
+
+Kapor seemed to score quite an effect with his shrewd and skeptical analysis
+of the merits of telco "Caller-ID" services. (On this topic, FCIC and EFF
+have never been at loggerheads, and have no particular established earthworks
+to defend.) Caller-ID has generally been promoted as a privacy service
+for consumers, a presentation Kapor described as a "smokescreen,"
+the real point of Caller-ID being to ALLOW CORPORATE CUSTOMERS TO BUILD
+EXTENSIVE COMMERCIAL DATABASES ON EVERYBODY WHO PHONES OR FAXES THEM.
+Clearly, few people in the room had considered this possibility,
+except perhaps for two late-arrivals from US WEST RBOC security,
+who chuckled nervously.
+
+Mike Godwin then made an extensive presentation on
+"Civil Liberties Implications of Computer Searches and Seizures."
+Now, at last, we were getting to the real nitty-gritty here,
+real political horse-trading. The audience listened with close
+attention, angry mutters rising occasionally: "He's trying to
+teach us our jobs!" "We've been thinking about this for years!
+We think about these issues every day!" "If I didn't seize the works,
+I'd be sued by the guy's victims!" "I'm violating the law if I leave
+ten thousand disks full of illegal PIRATED SOFTWARE and STOLEN CODES!"
+"It's our job to make sure people don't trash the Constitution--
+we're the DEFENDERS of the Constitution!" "We seize stuff when
+we know it will be forfeited anyway as restitution for the victim!"
+
+"If it's forfeitable, then don't get a search warrant, get a
+forfeiture warrant," Godwin suggested coolly. He further remarked
+that most suspects in computer crime don't WANT to see their computers
+vanish out the door, headed God knew where, for who knows how long.
+They might not mind a search, even an extensive search, but they want
+their machines searched on-site.
+
+"Are they gonna feed us?" somebody asked sourly.
+
+"How about if you take copies of the data?" Godwin parried.
+
+"That'll never stand up in court."
+
+"Okay, you make copies, give THEM the copies, and take the originals."
+
+Hmmm.
+
+Godwin championed bulletin-board systems as repositories of First Amendment
+protected free speech. He complained that federal computer-crime training
+manuals gave boards a bad press, suggesting that they are hotbeds of crime
+haunted by pedophiles and crooks, whereas the vast majority of the nation's
+thousands of boards are completely innocuous, and nowhere near so
+romantically suspicious.
+
+People who run boards violently resent it when their systems are seized,
+and their dozens (or hundreds) of users look on in abject horror.
+Their rights of free expression are cut short. Their right to associate
+with other people is infringed. And their privacy is violated as their
+private electronic mail becomes police property.
+
+Not a soul spoke up to defend the practice of seizing boards.
+The issue passed in chastened silence. Legal principles aside--
+(and those principles cannot be settled without laws passed or
+court precedents)--seizing bulletin boards has become public-relations
+poison for American computer police.
+
+And anyway, it's not entirely necessary. If you're a cop, you can get 'most
+everything you need from a pirate board, just by using an inside informant.
+Plenty of vigilantes--well, CONCERNED CITIZENS--will inform police the moment
+they see a pirate board hit their area (and will tell the police all about it,
+in such technical detail, actually, that you kinda wish they'd shut up).
+They will happily supply police with extensive downloads or printouts.
+It's IMPOSSIBLE to keep this fluid electronic information out of the
+hands of police.
+
+Some people in the electronic community become enraged at the prospect
+of cops "monitoring" bulletin boards. This does have touchy aspects,
+as Secret Service people in particular examine bulletin boards with
+some regularity. But to expect electronic police to be deaf dumb
+and blind in regard to this particular medium rather flies in the face
+of common sense. Police watch television, listen to radio, read newspapers
+and magazines; why should the new medium of boards be different?
+Cops can exercise the same access to electronic information
+as everybody else. As we have seen, quite a few computer
+police maintain THEIR OWN bulletin boards, including anti-hacker
+"sting" boards, which have generally proven quite effective.
+
+As a final clincher, their Mountie friends in Canada (and colleagues
+in Ireland and Taiwan) don't have First Amendment or American
+constitutional restrictions, but they do have phone lines,
+and can call any bulletin board in America whenever they please.
+The same technological determinants that play into the hands of hackers,
+phone phreaks and software pirates can play into the hands of police.
+"Technological determinants" don't have ANY human allegiances.
+They're not black or white, or Establishment or Underground,
+or pro-or-anti anything.
+
+Godwin complained at length about what he called "the Clever Hobbyist
+hypothesis" --the assumption that the "hacker" you're busting is clearly
+a technical genius, and must therefore by searched with extreme thoroughness.
+So: from the law's point of view, why risk missing anything? Take the works.
+Take the guy's computer. Take his books. Take his notebooks.
+Take the electronic drafts of his love letters. Take his Walkman.
+Take his wife's computer. Take his dad's computer. Take his kid
+sister's computer. Take his employer's computer. Take his compact disks--
+they MIGHT be CD-ROM disks, cunningly disguised as pop music.
+Take his laser printer--he might have hidden something vital in the
+printer's 5meg of memory. Take his software manuals and hardware
+documentation. Take his science-fiction novels and his simulation-
+gaming books. Take his Nintendo Game-Boy and his Pac-Man arcade game.
+Take his answering machine, take his telephone out of the wall.
+Take anything remotely suspicious.
+
+Godwin pointed out that most "hackers" are not, in fact, clever
+genius hobbyists. Quite a few are crooks and grifters who don't
+have much in the way of technical sophistication; just some rule-of-thumb
+rip-off techniques. The same goes for most fifteen-year-olds who've
+downloaded a code-scanning program from a pirate board. There's no
+real need to seize everything in sight. It doesn't require an entire
+computer system and ten thousand disks to prove a case in court.
+
+What if the computer is the instrumentality of a crime? someone demanded.
+
+Godwin admitted quietly that the doctrine of seizing the instrumentality
+of a crime was pretty well established in the American legal system.
+
+The meeting broke up. Godwin and Kapor had to leave. Kapor was testifying
+next morning before the Massachusetts Department Of Public Utility,
+about ISDN narrowband wide-area networking.
+
+As soon as they were gone, Thackeray seemed elated.
+She had taken a great risk with this. Her colleagues had not,
+in fact, torn Kapor and Godwin's heads off. She was very proud of them,
+and told them so.
+
+"Did you hear what Godwin said about INSTRUMENTALITY OF A CRIME?"
+she exulted, to nobody in particular. "Wow, that means
+MITCH ISN'T GOING TO SUE ME."
+
+#
+
+America's computer police are an interesting group.
+As a social phenomenon they are far more interesting,
+and far more important, than teenage phone phreaks
+and computer hackers. First, they're older and wiser;
+not dizzy hobbyists with leaky morals, but seasoned adult
+professionals with all the responsibilities of public service.
+And, unlike hackers, they possess not merely TECHNICAL
+power alone, but heavy-duty legal and social authority.
+
+And, very interestingly, they are just as much at
+sea in cyberspace as everyone else. They are not
+happy about this. Police are authoritarian by nature,
+and prefer to obey rules and precedents. (Even those police
+who secretly enjoy a fast ride in rough territory will soberly
+disclaim any "cowboy" attitude.) But in cyberspace there ARE
+no rules and precedents. They are groundbreaking pioneers,
+Cyberspace Rangers, whether they like it or not.
+
+In my opinion, any teenager enthralled by computers,
+fascinated by the ins and outs of computer security,
+and attracted by the lure of specialized forms of knowledge and power,
+would do well to forget all about "hacking" and set his (or her)
+sights on becoming a fed. Feds can trump hackers at almost every
+single thing hackers do, including gathering intelligence,
+undercover disguise, trashing, phone-tapping, building dossiers,
+networking, and infiltrating computer systems--CRIMINAL computer systems.
+Secret Service agents know more about phreaking, coding and carding
+than most phreaks can find out in years, and when it comes to viruses,
+break-ins, software bombs and trojan horses, Feds have direct access to red-hot
+confidential information that is only vague rumor in the underground.
+
+And if it's an impressive public rep you're after, there are few people
+in the world who can be so chillingly impressive as a well-trained,
+well-armed United States Secret Service agent.
+
+Of course, a few personal sacrifices are necessary in order to obtain
+that power and knowledge. First, you'll have the galling discipline
+of belonging to a large organization; but the world of computer crime
+is still so small, and so amazingly fast-moving, that it will remain
+spectacularly fluid for years to come. The second sacrifice is that
+you'll have to give up ripping people off. This is not a great loss.
+Abstaining from the use of illegal drugs, also necessary, will be a boon
+to your health.
+
+A career in computer security is not a bad choice for a young man
+or woman today. The field will almost certainly expand drastically
+in years to come. If you are a teenager today, by the time you
+become a professional, the pioneers you have read about in this book
+will be the grand old men and women of the field, swamped by their many
+disciples and successors. Of course, some of them, like William P. Wood
+of the 1865 Secret Service, may well be mangled in the whirring machinery
+of legal controversy; but by the time you enter the computer-crime field,
+it may have stabilized somewhat, while remaining entertainingly challenging.
+
+But you can't just have a badge. You have to win it. First, there's the
+federal law enforcement training. And it's hard--it's a challenge.
+A real challenge--not for wimps and rodents.
+
+Every Secret Service agent must complete gruelling courses at the
+Federal Law Enforcement Training Center. (In fact, Secret Service
+agents are periodically re-trained during their entire careers.)
+
+In order to get a glimpse of what this might be like,
+I myself travelled to FLETC.
+
+#
+
+The Federal Law Enforcement Training Center is a 1500-acre facility
+on Georgia's Atlantic coast. It's a milieu of marshgrass, seabirds,
+damp, clinging sea-breezes, palmettos, mosquitos, and bats.
+Until 1974, it was a Navy Air Base, and still features a working runway,
+and some WWII vintage blockhouses and officers' quarters.
+The Center has since benefitted by a forty-million-dollar retrofit,
+but there's still enough forest and swamp on the facility for the
+Border Patrol to put in tracking practice.
+
+As a town, "Glynco" scarcely exists. The nearest real town is Brunswick,
+a few miles down Highway 17, where I stayed at the aptly named Marshview
+Holiday Inn. I had Sunday dinner at a seafood restaurant called "Jinright's,"
+where I feasted on deep-fried alligator tail. This local favorite was
+a heaped basket of bite-sized chunks of white, tender, almost fluffy
+reptile meat, steaming in a peppered batter crust. Alligator makes
+a culinary experience that's hard to forget, especially when liberally
+basted with homemade cocktail sauce from a Jinright squeeze-bottle.
+
+The crowded clientele were tourists, fishermen, local black folks
+in their Sunday best, and white Georgian locals who all seemed
+to bear an uncanny resemblance to Georgia humorist Lewis Grizzard.
+
+The 2,400 students from 75 federal agencies who make up the FLETC
+population scarcely seem to make a dent in the low-key local scene.
+The students look like tourists, and the teachers seem to have taken
+on much of the relaxed air of the Deep South. My host was Mr. Carlton
+Fitzpatrick, the Program Coordinator of the Financial Fraud Institute.
+Carlton Fitzpatrick is a mustached, sinewy, well-tanned Alabama native
+somewhere near his late forties, with a fondness for chewing tobacco,
+powerful computers, and salty, down-home homilies. We'd met before,
+at FCIC in Arizona.
+
+The Financial Fraud Institute is one of the nine divisions at FLETC.
+Besides Financial Fraud, there's Driver & Marine, Firearms,
+and Physical Training. These are specialized pursuits.
+There are also five general training divisions: Basic Training,
+Operations, Enforcement Techniques, Legal Division, and Behavioral Science.
+
+Somewhere in this curriculum is everything necessary to turn green college
+graduates into federal agents. First they're given ID cards. Then they get
+the rather miserable-looking blue coveralls known as "smurf suits."
+The trainees are assigned a barracks and a cafeteria, and immediately
+set on FLETC's bone-grinding physical training routine. Besides the
+obligatory daily jogging--(the trainers run up danger flags beside
+the track when the humidity rises high enough to threaten heat stroke)--
+here's the Nautilus machines, the martial arts, the survival skills. . . .
+
+The eighteen federal agencies who maintain on-site academies at FLETC
+employ a wide variety of specialized law enforcement units, some of them
+rather arcane. There's Border Patrol, IRS Criminal Investigation Division,
+Park Service, Fish and Wildlife, Customs, Immigration, Secret Service and
+the Treasury's uniformed subdivisions. . . . If you're a federal cop
+and you don't work for the FBI, you train at FLETC. This includes people
+as apparently obscure as the agents of the Railroad Retirement Board
+Inspector General. Or the Tennessee Valley Authority Police,
+who are in fact federal police officers, and can and do arrest criminals
+on the federal property of the Tennessee Valley Authority.
+
+And then there are the computer-crime people. All sorts, all backgrounds.
+Mr. Fitzpatrick is not jealous of his specialized knowledge. Cops all over,
+in every branch of service, may feel a need to learn what he can teach.
+Backgrounds don't matter much. Fitzpatrick himself was originally a
+Border Patrol veteran, then became a Border Patrol instructor at FLETC.
+His Spanish is still fluent--but he found himself strangely fascinated
+when the first computers showed up at the Training Center. Fitzpatrick
+did have a background in electrical engineering, and though he never
+considered himself a computer hacker, he somehow found himself writing
+useful little programs for this new and promising gizmo.
+
+He began looking into the general subject of computers and crime,
+reading Donn Parker's books and articles, keeping an ear cocked
+for war stories, useful insights from the field, the up-and-coming
+people of the local computer-crime and high-technology units. . . .
+Soon he got a reputation around FLETC as the resident "computer expert,"
+and that reputation alone brought him more exposure, more experience--
+until one day he looked around, and sure enough he WAS a federal
+computer-crime expert.
+
+In fact, this unassuming, genial man may be THE federal computer-crime expert.
+There are plenty of very good computer people, and plenty of very good
+federal investigators, but the area where these worlds of expertise overlap
+is very slim. And Carlton Fitzpatrick has been right at the center of that
+since 1985, the first year of the Colluquy, a group which owes much to
+his influence.
+
+He seems quite at home in his modest, acoustic-tiled office,
+with its Ansel Adams-style Western photographic art, a gold-framed
+Senior Instructor Certificate, and a towering bookcase crammed with
+three-ring binders with ominous titles such as Datapro Reports on
+Information Security and CFCA Telecom Security '90.
+
+The phone rings every ten minutes; colleagues show up at the door
+to chat about new developments in locksmithing or to shake their heads
+over the latest dismal developments in the BCCI global banking scandal.
+
+Carlton Fitzpatrick is a fount of computer-crime war-stories,
+related in an acerbic drawl. He tells me the colorful tale
+of a hacker caught in California some years back. He'd been
+raiding systems, typing code without a detectable break,
+for twenty, twenty-four, thirty-six hours straight. Not just
+logged on--TYPING. Investigators were baffled. Nobody
+could do that. Didn't he have to go to the bathroom?
+Was it some kind of automatic keyboard-whacking device
+that could actually type code?
+
+A raid on the suspect's home revealed a situation of astonishing squalor.
+The hacker turned out to be a Pakistani computer-science student who had
+flunked out of a California university. He'd gone completely underground
+as an illegal electronic immigrant, and was selling stolen phone-service
+to stay alive. The place was not merely messy and dirty, but in a state
+of psychotic disorder. Powered by some weird mix of culture shock,
+computer addiction, and amphetamines, the suspect had in fact been sitting
+in front of his computer for a day and a half straight, with snacks and
+drugs at hand on the edge of his desk and a chamber-pot under his chair.
+
+Word about stuff like this gets around in the hacker-tracker community.
+
+Carlton Fitzpatrick takes me for a guided tour by car around the
+FLETC grounds. One of our first sights is the biggest indoor
+firing range in the world. There are federal trainees in there,
+Fitzpatrick assures me politely, blasting away with a wide variety
+of automatic weapons: Uzis, Glocks, AK-47s. . . . He's willing to
+take me inside. I tell him I'm sure that's really interesting,
+but I'd rather see his computers. Carlton Fitzpatrick seems quite
+surprised and pleased. I'm apparently the first journalist he's ever
+seen who has turned down the shooting gallery in favor of microchips.
+
+Our next stop is a favorite with touring Congressmen: the three-mile
+long FLETC driving range. Here trainees of the Driver & Marine Division
+are taught high-speed pursuit skills, setting and breaking road-blocks,
+diplomatic security driving for VIP limousines. . . . A favorite FLETC
+pastime is to strap a passing Senator into the passenger seat beside a
+Driver & Marine trainer, hit a hundred miles an hour, then take it right into
+"the skid-pan," a section of greased track where two tons of Detroit iron
+can whip and spin like a hockey puck.
+
+Cars don't fare well at FLETC. First they're rifled again and again
+for search practice. Then they do 25,000 miles of high-speed
+pursuit training; they get about seventy miles per set
+of steel-belted radials. Then it's off to the skid pan,
+where sometimes they roll and tumble headlong in the grease.
+When they're sufficiently grease-stained, dented, and creaky,
+they're sent to the roadblock unit, where they're battered without pity.
+And finally then they're sacrificed to the Bureau of Alcohol,
+Tobacco and Firearms, whose trainees learn the ins and outs
+of car-bomb work by blowing them into smoking wreckage.
+
+There's a railroad box-car on the FLETC grounds, and a large
+grounded boat, and a propless plane; all training-grounds for searches.
+The plane sits forlornly on a patch of weedy tarmac next to an eerie
+blockhouse known as the "ninja compound," where anti-terrorism specialists
+practice hostage rescues. As I gaze on this creepy paragon of modern
+low-intensity warfare, my nerves are jangled by a sudden staccato outburst
+of automatic weapons fire, somewhere in the woods to my right.
+"Nine-millimeter," Fitzpatrick judges calmly.
+
+Even the eldritch ninja compound pales somewhat compared
+to the truly surreal area known as "the raid-houses."
+This is a street lined on both sides with nondescript
+concrete-block houses with flat pebbled roofs.
+They were once officers' quarters. Now they are training grounds.
+The first one to our left, Fitzpatrick tells me, has been specially
+adapted for computer search-and-seizure practice. Inside it has been
+wired for video from top to bottom, with eighteen pan-and-tilt
+remotely controlled videocams mounted on walls and in corners.
+Every movement of the trainee agent is recorded live by teachers,
+for later taped analysis. Wasted movements, hesitations, possibly lethal
+tactical mistakes--all are gone over in detail.
+
+Perhaps the weirdest single aspect of this building is its front door,
+scarred and scuffed all along the bottom, from the repeated impact,
+day after day, of federal shoe-leather.
+
+Down at the far end of the row of raid-houses some people are practicing
+a murder. We drive by slowly as some very young and rather nervous-looking
+federal trainees interview a heavyset bald man on the raid-house lawn.
+Dealing with murder takes a lot of practice; first you have to learn
+to control your own instinctive disgust and panic, then you have to learn
+to control the reactions of a nerve-shredded crowd of civilians,
+some of whom may have just lost a loved one, some of whom may be murderers--
+quite possibly both at once.
+
+A dummy plays the corpse. The roles of the bereaved, the morbidly curious,
+and the homicidal are played, for pay, by local Georgians: waitresses,
+musicians, most anybody who needs to moonlight and can learn a script.
+These people, some of whom are FLETC regulars year after year,
+must surely have one of the strangest jobs in the world.
+
+Something about the scene: "normal" people in a weird situation,
+standing around talking in bright Georgia sunshine, unsuccessfully
+pretending that something dreadful has gone on, while a dummy lies
+inside on faked bloodstains. . . . While behind this weird masquerade,
+like a nested set of Russian dolls, are grim future realities of real death,
+real violence, real murders of real people, that these young agents
+will really investigate, many times during their careers. . . .
+Over and over. . . . Will those anticipated murders look like this,
+feel like this--not as "real" as these amateur actors are trying to
+make it seem, but both as "real," and as numbingly unreal, as watching
+fake people standing around on a fake lawn? Something about this scene
+unhinges me. It seems nightmarish to me, Kafkaesque. I simply don't
+know how to take it; my head is turned around; I don't know whether to laugh,
+cry, or just shudder.
+
+When the tour is over, Carlton Fitzpatrick and I talk about computers.
+For the first time cyberspace seems like quite a comfortable place.
+It seems very real to me suddenly, a place where I know what I'm talking about,
+a place I'm used to. It's real. "Real." Whatever.
+
+Carlton Fitzpatrick is the only person I've met in cyberspace circles
+who is happy with his present equipment. He's got a 5 Meg RAM PC with
+a 112 meg hard disk; a 660 meg's on the way. He's got a Compaq 386 desktop,
+and a Zenith 386 laptop with 120 meg. Down the hall is a NEC Multi-Sync 2A
+with a CD-ROM drive and a 9600 baud modem with four com-lines.
+There's a training minicomputer, and a 10-meg local mini just for the Center,
+and a lab-full of student PC clones and half-a-dozen Macs or so.
+There's a Data General MV 2500 with 8 meg on board and a 370 meg disk.
+
+Fitzpatrick plans to run a UNIX board on the Data General when he's
+finished beta-testing the software for it, which he wrote himself.
+It'll have E-mail features, massive files on all manner of computer-crime
+and investigation procedures, and will follow the computer-security
+specifics of the Department of Defense "Orange Book." He thinks
+it will be the biggest BBS in the federal government.
+
+Will it have Phrack on it? I ask wryly.
+
+Sure, he tells me. Phrack, TAP, Computer Underground Digest,
+all that stuff. With proper disclaimers, of course.
+
+I ask him if he plans to be the sysop. Running a system that size is very
+time-consuming, and Fitzpatrick teaches two three-hour courses every day.
+
+No, he says seriously, FLETC has to get its money worth out of the instructors.
+He thinks he can get a local volunteer to do it, a high-school student.
+
+He says a bit more, something I think about an Eagle Scout law-enforcement
+liaison program, but my mind has rocketed off in disbelief.
+
+"You're going to put a TEENAGER in charge of a federal security BBS?"
+I'm speechless. It hasn't escaped my notice that the FLETC Financial
+Fraud Institute is the ULTIMATE hacker-trashing target; there is stuff in here,
+stuff of such utter and consummate cool by every standard of the
+digital underground. . . .
+
+I imagine the hackers of my acquaintance, fainting dead-away from
+forbidden-knowledge greed-fits, at the mere prospect of cracking
+the superultra top-secret computers used to train the Secret Service
+in computer-crime. . . .
+
+"Uhm, Carlton," I babble, "I'm sure he's a really nice kid and all,
+but that's a terrible temptation to set in front of somebody who's,
+you know, into computers and just starting out. . . ."
+
+"Yeah," he says, "that did occur to me." For the first time I begin
+to suspect that he's pulling my leg.
+
+He seems proudest when he shows me an ongoing project called JICC,
+Joint Intelligence Control Council. It's based on the services provided
+by EPIC, the El Paso Intelligence Center, which supplies data and intelligence
+to the Drug Enforcement Administration, the Customs Service, the Coast Guard,
+and the state police of the four southern border states. Certain EPIC files
+can now be accessed by drug-enforcement police of Central America,
+South America and the Caribbean, who can also trade information
+among themselves. Using a telecom program called "White Hat,"
+written by two brothers named Lopez from the Dominican Republic,
+police can now network internationally on inexpensive PCs.
+Carlton Fitzpatrick is teaching a class of drug-war agents
+from the Third World, and he's very proud of their progress.
+Perhaps soon the sophisticated smuggling networks of the
+Medellin Cartel will be matched by a sophisticated computer
+network of the Medellin Cartel's sworn enemies. They'll track boats,
+track contraband, track the international drug-lords who now leap over
+borders with great ease, defeating the police through the clever use
+of fragmented national jurisdictions.
+
+JICC and EPIC must remain beyond the scope of this book.
+They seem to me to be very large topics fraught with complications
+that I am not fit to judge. I do know, however, that the international,
+computer-assisted networking of police, across national boundaries,
+is something that Carlton Fitzpatrick considers very important,
+a harbinger of a desirable future. I also know that networks
+by their nature ignore physical boundaries. And I also know
+that where you put communications you put a community,
+and that when those communities become self-aware
+they will fight to preserve themselves and to expand their influence.
+I make no judgements whether this is good or bad.
+It's just cyberspace; it's just the way things are.
+
+I asked Carlton Fitzpatrick what advice he would have for
+a twenty-year-old who wanted to shine someday in the world
+of electronic law enforcement.
+
+He told me that the number one rule was simply not to be
+scared of computers. You don't need to be an obsessive
+"computer weenie," but you mustn't be buffaloed just because
+some machine looks fancy. The advantages computers give
+smart crooks are matched by the advantages they give smart cops.
+Cops in the future will have to enforce the law "with their heads,
+not their holsters." Today you can make good cases without ever
+leaving your office. In the future, cops who resist the computer
+revolution will never get far beyond walking a beat.
+
+I asked Carlton Fitzpatrick if he had some single message for the public;
+some single thing that he would most like the American public to know
+about his work.
+
+He thought about it while. "Yes," he said finally. "TELL me the rules,
+and I'll TEACH those rules!" He looked me straight in the eye.
+"I do the best that I can."
+
+
+
+PART FOUR: THE CIVIL LIBERTARIANS
+
+
+The story of the Hacker Crackdown, as we have followed it thus far,
+has been technological, subcultural, criminal and legal.
+The story of the Civil Libertarians, though it partakes
+of all those other aspects, is profoundly and thoroughly POLITICAL.
+
+In 1990, the obscure, long-simmering struggle over the ownership
+and nature of cyberspace became loudly and irretrievably public.
+People from some of the oddest corners of American society suddenly
+found themselves public figures. Some of these people found this
+situation much more than they had ever bargained for. They backpedalled,
+and tried to retreat back to the mandarin obscurity of their cozy
+subcultural niches. This was generally to prove a mistake.
+
+But the civil libertarians seized the day in 1990. They found themselves
+organizing, propagandizing, podium-pounding, persuading, touring,
+negotiating, posing for publicity photos, submitting to interviews,
+squinting in the limelight as they tried a tentative, but growingly
+sophisticated, buck-and-wing upon the public stage.
+
+It's not hard to see why the civil libertarians should have
+this competitive advantage.
+
+The hackers of the digital underground are an hermetic elite.
+They find it hard to make any remotely convincing case for
+their actions in front of the general public. Actually,
+hackers roundly despise the "ignorant" public, and have never
+trusted the judgement of "the system." Hackers do propagandize,
+but only among themselves, mostly in giddy, badly spelled manifestos
+of class warfare, youth rebellion or naive techie utopianism.
+Hackers must strut and boast in order to establish and preserve
+their underground reputations. But if they speak out too loudly
+and publicly, they will break the fragile surface-tension of the underground,
+and they will be harrassed or arrested. Over the longer term,
+most hackers stumble, get busted, get betrayed, or simply give up.
+As a political force, the digital underground is hamstrung.
+
+The telcos, for their part, are an ivory tower under protracted seige.
+They have plenty of money with which to push their calculated public image,
+but they waste much energy and goodwill attacking one another with
+slanderous and demeaning ad campaigns. The telcos have suffered
+at the hands of politicians, and, like hackers, they don't trust
+the public's judgement. And this distrust may be well-founded.
+Should the general public of the high-tech 1990s come to understand
+its own best interests in telecommunications, that might well pose
+a grave threat to the specialized technical power and authority
+that the telcos have relished for over a century. The telcos do
+have strong advantages: loyal employees, specialized expertise,
+influence in the halls of power, tactical allies in law enforcement,
+and unbelievably vast amounts of money. But politically speaking, they lack
+genuine grassroots support; they simply don't seem to have many friends.
+
+Cops know a lot of things other people don't know.
+But cops willingly reveal only those aspects of their
+knowledge that they feel will meet their institutional
+purposes and further public order. Cops have respect,
+they have responsibilities, they have power in the streets
+and even power in the home, but cops don't do particularly
+well in limelight. When pressed, they will step out in the
+public gaze to threaten bad-guys, or to cajole prominent citizens,
+or perhaps to sternly lecture the naive and misguided.
+But then they go back within their time-honored fortress
+of the station-house, the courtroom and the rule-book.
+
+The electronic civil libertarians, however, have proven to be
+born political animals. They seemed to grasp very early on
+the postmodern truism that communication is power. Publicity is power.
+Soundbites are power. The ability to shove one's issue onto the public
+agenda--and KEEP IT THERE--is power. Fame is power. Simple personal
+fluency and eloquence can be power, if you can somehow catch the
+public's eye and ear.
+
+The civil libertarians had no monopoly on "technical power"--
+though they all owned computers, most were not particularly
+advanced computer experts. They had a good deal of money,
+but nowhere near the earthshaking wealth and the galaxy
+of resources possessed by telcos or federal agencies.
+They had no ability to arrest people. They carried
+out no phreak and hacker covert dirty-tricks.
+
+But they really knew how to network.
+
+Unlike the other groups in this book, the civil libertarians
+have operated very much in the open, more or less right
+in the public hurly-burly. They have lectured audiences galore
+and talked to countless journalists, and have learned to
+refine their spiels. They've kept the cameras clicking,
+kept those faxes humming, swapped that email,
+run those photocopiers on overtime, licked envelopes
+and spent small fortunes on airfare and long-distance.
+In an information society, this open, overt, obvious activity
+has proven to be a profound advantage.
+
+In 1990, the civil libertarians of cyberspace assembled
+out of nowhere in particular, at warp speed. This "group"
+(actually, a networking gaggle of interested parties
+which scarcely deserves even that loose term) has almost nothing
+in the way of formal organization. Those formal civil libertarian
+organizations which did take an interest in cyberspace issues,
+mainly the Computer Professionals for Social Responsibility
+and the American Civil Liberties Union, were carried along
+by events in 1990, and acted mostly as adjuncts,
+underwriters or launching-pads.
+
+The civil libertarians nevertheless enjoyed the greatest success
+of any of the groups in the Crackdown of 1990. At this writing,
+their future looks rosy and the political initiative is firmly in their hands.
+This should be kept in mind as we study the highly unlikely lives
+and lifestyles of the people who actually made this happen.
+
+#
+
+In June 1989, Apple Computer, Inc., of Cupertino,
+California, had a problem. Someone had illicitly copied
+a small piece of Apple's proprietary software, software
+which controlled an internal chip driving the Macintosh
+screen display. This Color QuickDraw source code was
+a closely guarded piece of Apple's intellectual property.
+Only trusted Apple insiders were supposed to possess it.
+
+But the "NuPrometheus League" wanted things otherwise.
+This person (or persons) made several illicit copies
+of this source code, perhaps as many as two dozen.
+He (or she, or they) then put those illicit floppy disks
+into envelopes and mailed them to people all over America:
+people in the computer industry who were associated with,
+but not directly employed by, Apple Computer.
+
+The NuPrometheus caper was a complex, highly ideological,
+and very hacker-like crime. Prometheus, it will be recalled,
+stole the fire of the Gods and gave this potent gift to the
+general ranks of downtrodden mankind. A similar god-in-the-manger
+attitude was implied for the corporate elite of Apple Computer,
+while the "Nu" Prometheus had himself cast in the role of rebel demigod.
+The illicitly copied data was given away for free.
+
+The new Prometheus, whoever he was, escaped the
+fate of the ancient Greek Prometheus, who was chained
+to a rock for centuries by the vengeful gods while an eagle
+tore and ate his liver. On the other hand, NuPrometheus
+chickened out somewhat by comparison with his role model.
+The small chunk of Color QuickDraw code he had filched
+and replicated was more or less useless to Apple's
+industrial rivals (or, in fact, to anyone else).
+Instead of giving fire to mankind, it was more as if
+NuPrometheus had photocopied the schematics for part of a Bic lighter.
+The act was not a genuine work of industrial espionage.
+It was best interpreted as a symbolic, deliberate slap
+in the face for the Apple corporate heirarchy.
+
+Apple's internal struggles were well-known in the industry. Apple's founders,
+Jobs and Wozniak, had both taken their leave long since. Their raucous core
+of senior employees had been a barnstorming crew of 1960s Californians,
+many of them markedly less than happy with the new button-down multimillion
+dollar regime at Apple. Many of the programmers and developers who had
+invented the Macintosh model in the early 1980s had also taken their leave of
+the company. It was they, not the current masters of Apple's corporate fate,
+who had invented the stolen Color QuickDraw code. The NuPrometheus stunt
+was well-calculated to wound company morale.
+
+Apple called the FBI. The Bureau takes an interest in high-profile
+intellectual-property theft cases, industrial espionage and theft
+of trade secrets. These were likely the right people to call,
+and rumor has it that the entities responsible were in fact discovered
+by the FBI, and then quietly squelched by Apple management. NuPrometheus
+was never publicly charged with a crime, or prosecuted, or jailed.
+But there were no further illicit releases of Macintosh internal software.
+Eventually the painful issue of NuPrometheus was allowed to fade.
+
+In the meantime, however, a large number of puzzled bystanders
+found themselves entertaining surprise guests from the FBI.
+
+One of these people was John Perry Barlow. Barlow is a most unusual man,
+difficult to describe in conventional terms. He is perhaps best known as
+a songwriter for the Grateful Dead, for he composed lyrics for
+"Hell in a Bucket," "Picasso Moon," "Mexicali Blues," "I Need a Miracle,"
+and many more; he has been writing for the band since 1970.
+
+Before we tackle the vexing question as to why a rock lyricist
+should be interviewed by the FBI in a computer-crime case,
+it might be well to say a word or two about the Grateful Dead.
+The Grateful Dead are perhaps the most successful and long-lasting
+of the numerous cultural emanations from the Haight-Ashbury district
+of San Francisco, in the glory days of Movement politics and
+lysergic transcendance. The Grateful Dead are a nexus, a veritable
+whirlwind, of applique decals, psychedelic vans, tie-dyed T-shirts,
+earth-color denim, frenzied dancing and open and unashamed drug use.
+The symbols, and the realities, of Californian freak power surround
+the Grateful Dead like knotted macrame.
+
+The Grateful Dead and their thousands of Deadhead devotees
+are radical Bohemians. This much is widely understood.
+Exactly what this implies in the 1990s is rather more problematic.
+
+The Grateful Dead are among the world's most popular
+and wealthy entertainers: number 20, according to Forbes magazine,
+right between M.C. Hammer and Sean Connery. In 1990, this jeans-clad
+group of purported raffish outcasts earned seventeen million dollars.
+They have been earning sums much along this line for quite some time now.
+
+And while the Dead are not investment bankers or three-piece-suit
+tax specialists--they are, in point of fact, hippie musicians--
+this money has not been squandered in senseless Bohemian excess.
+The Dead have been quietly active for many years, funding various
+worthy activities in their extensive and widespread cultural community.
+
+The Grateful Dead are not conventional players in the American
+power establishment. They nevertheless are something of a force
+to be reckoned with. They have a lot of money and a lot of friends
+in many places, both likely and unlikely.
+
+The Dead may be known for back-to-the-earth environmentalist rhetoric,
+but this hardly makes them anti-technological Luddites. On the contrary,
+like most rock musicians, the Grateful Dead have spent their entire adult
+lives in the company of complex electronic equipment. They have funds to burn
+on any sophisticated tool and toy that might happen to catch their fancy.
+And their fancy is quite extensive.
+
+The Deadhead community boasts any number of recording engineers,
+lighting experts, rock video mavens, electronic technicians
+of all descriptions. And the drift goes both ways. Steve Wozniak,
+Apple's co-founder, used to throw rock festivals. Silicon Valley rocks out.
+
+These are the 1990s, not the 1960s. Today, for a surprising number of people
+all over America, the supposed dividing line between Bohemian and technician
+simply no longer exists. People of this sort may have a set of windchimes
+and a dog with a knotted kerchief 'round its neck, but they're also quite
+likely to own a multimegabyte Macintosh running MIDI synthesizer software
+and trippy fractal simulations. These days, even Timothy Leary himself,
+prophet of LSD, does virtual-reality computer-graphics demos in
+his lecture tours.
+
+John Perry Barlow is not a member of the Grateful Dead. He is, however,
+a ranking Deadhead.
+
+Barlow describes himself as a "techno-crank." A vague term like
+"social activist" might not be far from the mark, either.
+But Barlow might be better described as a "poet"--if one keeps in mind
+Percy Shelley's archaic definition of poets as "unacknowledged legislators
+of the world."
+
+Barlow once made a stab at acknowledged legislator status. In 1987,
+he narrowly missed the Republican nomination for a seat in the
+Wyoming State Senate. Barlow is a Wyoming native, the third-generation
+scion of a well-to-do cattle-ranching family. He is in his early forties,
+married and the father of three daughters.
+
+Barlow is not much troubled by other people's narrow notions of consistency.
+In the late 1980s, this Republican rock lyricist cattle rancher sold his ranch
+and became a computer telecommunications devotee.
+
+The free-spirited Barlow made this transition with ease. He genuinely
+enjoyed computers. With a beep of his modem, he leapt from small-town
+Pinedale, Wyoming, into electronic contact with a large and lively crowd
+of bright, inventive, technological sophisticates from all over the world.
+Barlow found the social milieu of computing attractive: its fast-lane pace,
+its blue-sky rhetoric, its open-endedness. Barlow began dabbling in
+computer journalism, with marked success, as he was a quick study,
+and both shrewd and eloquent. He frequently travelled to San Francisco
+to network with Deadhead friends. There Barlow made extensive contacts
+throughout the Californian computer community, including friendships
+among the wilder spirits at Apple.
+
+In May 1990, Barlow received a visit from a local Wyoming agent of the FBI.
+The NuPrometheus case had reached Wyoming.
+
+Barlow was troubled to find himself under investigation in an
+area of his interests once quite free of federal attention.
+He had to struggle to explain the very nature of computer-crime
+to a headscratching local FBI man who specialized in cattle-rustling.
+Barlow, chatting helpfully and demonstrating the wonders of his modem
+to the puzzled fed, was alarmed to find all "hackers" generally under
+FBI suspicion as an evil influence in the electronic community.
+The FBI, in pursuit of a hacker called "NuPrometheus," were tracing
+attendees of a suspect group called the Hackers Conference.
+
+The Hackers Conference, which had been started in 1984, was a
+yearly Californian meeting of digital pioneers and enthusiasts.
+The hackers of the Hackers Conference had little if anything to do
+with the hackers of the digital underground. On the contrary,
+the hackers of this conference were mostly well-to-do Californian
+high-tech CEOs, consultants, journalists and entrepreneurs.
+(This group of hackers were the exact sort of "hackers"
+most likely to react with militant fury at any criminal
+degradation of the term "hacker.")
+
+Barlow, though he was not arrested or accused of a crime,
+and though his computer had certainly not gone out the door,
+was very troubled by this anomaly. He carried the word to the Well.
+
+Like the Hackers Conference, "the Well" was an emanation of the
+Point Foundation. Point Foundation, the inspiration of a wealthy
+Californian 60s radical named Stewart Brand, was to be a major
+launch-pad of the civil libertarian effort.
+
+Point Foundation's cultural efforts, like those of their fellow Bay Area
+Californians the Grateful Dead, were multifaceted and multitudinous.
+Rigid ideological consistency had never been a strong suit of the
+Whole Earth Catalog. This Point publication had enjoyed a strong
+vogue during the late 60s and early 70s, when it offered hundreds
+of practical (and not so practical) tips on communitarian living,
+environmentalism, and getting back-to-the-land. The Whole Earth Catalog,
+and its sequels, sold two and half million copies and won a
+National Book Award.
+
+With the slow collapse of American radical dissent, the Whole Earth Catalog
+had slipped to a more modest corner of the cultural radar; but in its
+magazine incarnation, CoEvolution Quarterly, the Point Foundation
+continued to offer a magpie potpourri of "access to tools and ideas."
+
+CoEvolution Quarterly, which started in 1974, was never a widely
+popular magazine. Despite periodic outbreaks of millenarian fervor,
+CoEvolution Quarterly failed to revolutionize Western civilization
+and replace leaden centuries of history with bright new Californian paradigms.
+Instead, this propaganda arm of Point Foundation cakewalked a fine line between
+impressive brilliance and New Age flakiness. CoEvolution Quarterly carried
+no advertising, cost a lot, and came out on cheap newsprint with modest
+black-and-white graphics. It was poorly distributed, and spread mostly
+by subscription and word of mouth.
+
+It could not seem to grow beyond 30,000 subscribers.
+And yet--it never seemed to shrink much, either.
+Year in, year out, decade in, decade out, some strange
+demographic minority accreted to support the magazine.
+The enthusiastic readership did not seem to have much
+in the way of coherent politics or ideals. It was sometimes
+hard to understand what held them together (if the often bitter
+debate in the letter-columns could be described as "togetherness").
+
+But if the magazine did not flourish, it was resilient; it got by.
+Then, in 1984, the birth-year of the Macintosh computer,
+CoEvolution Quarterly suddenly hit the rapids. Point Foundation
+had discovered the computer revolution. Out came the Whole Earth
+Software Catalog of 1984, arousing headscratching doubts among
+the tie-dyed faithful, and rabid enthusiasm among the nascent
+"cyberpunk" milieu, present company included. Point Foundation
+started its yearly Hackers Conference, and began to take an
+extensive interest in the strange new possibilities of
+digital counterculture. CoEvolution Quarterlyfolded its teepee,
+replaced by Whole Earth Software Review and eventually by Whole Earth
+Review (the magazine's present incarnation, currently under
+the editorship of virtual-reality maven Howard Rheingold).
+
+1985 saw the birth of the "WELL"--the "Whole Earth 'Lectronic Link."
+The Well was Point Foundation's bulletin board system.
+
+As boards went, the Well was an anomaly from the beginning,
+and remained one. It was local to San Francisco.
+It was huge, with multiple phonelines and enormous files
+of commentary. Its complex UNIX-based software might be
+most charitably described as "user-opaque." It was run on
+a mainframe out of the rambling offices of a non-profit
+cultural foundation in Sausalito. And it was crammed with
+fans of the Grateful Dead.
+
+Though the Well was peopled by chattering hipsters of the Bay Area
+counterculture, it was by no means a "digital underground" board.
+Teenagers were fairly scarce; most Well users (known as "Wellbeings")
+were thirty- and forty-something Baby Boomers. They tended to work
+in the information industry: hardware, software, telecommunications,
+media, entertainment. Librarians, academics, and journalists were
+especially common on the Well, attracted by Point Foundation's
+open-handed distribution of "tools and ideas."
+
+There were no anarchy files on the Well, scarcely a
+dropped hint about access codes or credit-card theft.
+No one used handles. Vicious "flame-wars" were held to
+a comparatively civilized rumble. Debates were sometimes sharp,
+but no Wellbeing ever claimed that a rival had disconnected his phone,
+trashed his house, or posted his credit card numbers.
+
+The Well grew slowly as the 1980s advanced. It charged a modest sum
+for access and storage, and lost money for years--but not enough to hamper
+the Point Foundation, which was nonprofit anyway. By 1990, the Well
+had about five thousand users. These users wandered about a gigantic
+cyberspace smorgasbord of "Conferences", each conference itself consisting
+of a welter of "topics," each topic containing dozens, sometimes hundreds
+of comments, in a tumbling, multiperson debate that could last for months
+or years on end.
+
+
+In 1991, the Well's list of conferences looked like this:
+
+
+CONFERENCES ON THE WELL
+
+WELL "Screenzine" Digest (g zine)
+
+Best of the WELL - vintage material - (g best)
+
+Index listing of new topics in all conferences - (g newtops)
+
+Business - Education
+----------------------
+
+Apple Library Users Group(g alug) Agriculture (g agri)
+Brainstorming (g brain) Classifieds (g cla)
+Computer Journalism (g cj) Consultants (g consult)
+Consumers (g cons) Design (g design)
+Desktop Publishing (g desk) Disability (g disability)
+Education (g ed) Energy (g energy91)
+Entrepreneurs (g entre) Homeowners (g home)
+Indexing (g indexing) Investments (g invest)
+Kids91 (g kids) Legal (g legal)
+One Person Business (g one)
+Periodical/newsletter (g per)
+Telecomm Law (g tcl) The Future (g fut)
+Translators (g trans) Travel (g tra)
+Work (g work)
+
+Electronic Frontier Foundation (g eff)
+Computers, Freedom & Privacy (g cfp)
+Computer Professionals for Social Responsibility (g cpsr)
+
+Social - Political - Humanities
+---------------------------------
+
+Aging (g gray) AIDS (g aids)
+Amnesty International (g amnesty) Archives (g arc)
+Berkeley (g berk) Buddhist (g wonderland)
+Christian (g cross) Couples (g couples)
+Current Events (g curr) Dreams (g dream)
+Drugs (g dru) East Coast (g east)
+Emotional Health@@@@ (g private) Erotica (g eros)
+Environment (g env) Firearms (g firearms)
+First Amendment (g first) Fringes of Reason (g fringes)
+Gay (g gay) Gay (Private)# (g gaypriv)
+Geography (g geo) German (g german)
+Gulf War (g gulf) Hawaii (g aloha)
+Health (g heal) History (g hist)
+Holistic (g holi) Interview (g inter)
+Italian (g ital) Jewish (g jew)
+Liberty (g liberty) Mind (g mind)
+Miscellaneous (g misc) Men on the WELL@@ (g mow)
+Network Integration (g origin) Nonprofits (g non)
+North Bay (g north) Northwest (g nw)
+Pacific Rim (g pacrim) Parenting (g par)
+Peace (g pea) Peninsula (g pen)
+Poetry (g poetry) Philosophy (g phi)
+Politics (g pol) Psychology (g psy)
+Psychotherapy (g therapy) Recovery## (g recovery)
+San Francisco (g sanfran) Scams (g scam)
+Sexuality (g sex) Singles (g singles)
+Southern (g south) Spanish (g spanish)
+Spirituality (g spirit) Tibet (g tibet)
+Transportation (g transport) True Confessions (g tru)
+Unclear (g unclear) WELL Writer's Workshop@@@(g www)
+Whole Earth (g we) Women on the WELL@(g wow)
+Words (g words) Writers (g wri)
+
+@@@@Private Conference - mail wooly for entry
+@@@Private conference - mail sonia for entry
+@@Private conference - mail flash for entry
+@ Private conference - mail reva for entry
+# Private Conference - mail hudu for entry
+## Private Conference - mail dhawk for entry
+
+Arts - Recreation - Entertainment
+-----------------------------------
+ArtCom Electronic Net (g acen)
+Audio-Videophilia (g aud)
+Bicycles (g bike) Bay Area Tonight@@(g bat)
+Boating (g wet) Books (g books)
+CD's (g cd) Comics (g comics)
+Cooking (g cook) Flying (g flying)
+Fun (g fun) Games (g games)
+Gardening (g gard) Kids (g kids)
+Nightowls@ (g owl) Jokes (g jokes)
+MIDI (g midi) Movies (g movies)
+Motorcycling (g ride) Motoring (g car)
+Music (g mus) On Stage (g onstage)
+Pets (g pets) Radio (g rad)
+Restaurant (g rest) Science Fiction (g sf)
+Sports (g spo) Star Trek (g trek)
+Television (g tv) Theater (g theater)
+Weird (g weird) Zines/Factsheet Five(g f5)
+@Open from midnight to 6am
+@@Updated daily
+
+Grateful Dead
+-------------
+Grateful Dead (g gd) Deadplan@ (g dp)
+Deadlit (g deadlit) Feedback (g feedback)
+GD Hour (g gdh) Tapes (g tapes)
+Tickets (g tix) Tours (g tours)
+
+@Private conference - mail tnf for entry
+
+Computers
+-----------
+AI/Forth/Realtime (g realtime) Amiga (g amiga)
+Apple (g app) Computer Books (g cbook)
+Art & Graphics (g gra) Hacking (g hack)
+HyperCard (g hype) IBM PC (g ibm)
+LANs (g lan) Laptop (g lap)
+Macintosh (g mac) Mactech (g mactech)
+Microtimes (g microx) Muchomedia (g mucho)
+NeXt (g next) OS/2 (g os2)
+Printers (g print) Programmer's Net (g net)
+Siggraph (g siggraph) Software Design (g sdc)
+Software/Programming (g software)
+Software Support (g ssc)
+Unix (g unix) Windows (g windows)
+Word Processing (g word)
+
+Technical - Communications
+----------------------------
+Bioinfo (g bioinfo) Info (g boing)
+Media (g media) NAPLPS (g naplps)
+Netweaver (g netweaver) Networld (g networld)
+Packet Radio (g packet) Photography (g pho)
+Radio (g rad) Science (g science)
+Technical Writers (g tec) Telecommunications(g tele)
+Usenet (g usenet) Video (g vid)
+Virtual Reality (g vr)
+
+The WELL Itself
+---------------
+Deeper (g deeper) Entry (g ent)
+General (g gentech) Help (g help)
+Hosts (g hosts) Policy (g policy)
+System News (g news) Test (g test)
+
+The list itself is dazzling, bringing to the untutored eye
+a dizzying impression of a bizarre milieu of mountain-climbing
+Hawaiian holistic photographers trading true-life confessions
+with bisexual word-processing Tibetans.
+
+But this confusion is more apparent than real. Each of these conferences
+was a little cyberspace world in itself, comprising dozens and perhaps
+hundreds of sub-topics. Each conference was commonly frequented by
+a fairly small, fairly like-minded community of perhaps a few dozen people.
+It was humanly impossible to encompass the entire Well (especially since
+access to the Well's mainframe computer was billed by the hour).
+Most long-time users contented themselves with a few favorite
+topical neighborhoods, with the occasional foray elsewhere
+for a taste of exotica. But especially important news items,
+and hot topical debates, could catch the attention of the entire
+Well community.
+
+Like any community, the Well had its celebrities, and John Perry Barlow,
+the silver-tongued and silver-modemed lyricist of the Grateful Dead,
+ranked prominently among them. It was here on the Well that Barlow
+posted his true-life tale of computer-crime encounter with the FBI.
+
+The story, as might be expected, created a great stir. The Well was
+already primed for hacker controversy. In December 1989, Harper's magazine
+had hosted a debate on the Well about the ethics of illicit computer intrusion.
+While over forty various computer-mavens took part, Barlow proved a star
+in the debate. So did "Acid Phreak" and "Phiber Optik," a pair of young
+New York hacker-phreaks whose skills at telco switching-station intrusion
+were matched only by their apparently limitless hunger for fame.
+The advent of these two boldly swaggering outlaws in the precincts
+of the Well created a sensation akin to that of Black Panthers
+at a cocktail party for the radically chic.
+
+Phiber Optik in particular was to seize the day in 1990.
+A devotee of the 2600 circle and stalwart of the New York
+hackers' group "Masters of Deception," Phiber Optik was
+a splendid exemplar of the computer intruder as committed dissident.
+The eighteen-year-old Optik, a high-school dropout and part-time
+computer repairman, was young, smart, and ruthlessly obsessive,
+a sharp-dressing, sharp-talking digital dude who was utterly
+and airily contemptuous of anyone's rules but his own.
+By late 1991, Phiber Optik had appeared in Harper's,
+Esquire, The New York Times, in countless public debates
+and conventions, even on a television show hosted by Geraldo Rivera.
+
+Treated with gingerly respect by Barlow and other Well mavens,
+Phiber Optik swiftly became a Well celebrity. Strangely, despite
+his thorny attitude and utter single-mindedness, Phiber Optik seemed
+to arouse strong protective instincts in most of the people who met him.
+He was great copy for journalists, always fearlessly ready to swagger,
+and, better yet, to actually DEMONSTRATE some off-the-wall digital stunt.
+He was a born media darling.
+
+Even cops seemed to recognize that there was something peculiarly unworldly
+and uncriminal about this particular troublemaker. He was so bold,
+so flagrant, so young, and so obviously doomed, that even those
+who strongly disapproved of his actions grew anxious for his welfare,
+and began to flutter about him as if he were an endangered seal pup.
+
+In January 24, 1990 (nine days after the Martin Luther King Day Crash),
+Phiber Optik, Acid Phreak, and a third NYC scofflaw named Scorpion were
+raided by the Secret Service. Their computers went out the door,
+along with the usual blizzard of papers, notebooks, compact disks,
+answering machines, Sony Walkmans, etc. Both Acid Phreak and
+Phiber Optik were accused of having caused the Crash.
+
+The mills of justice ground slowly. The case eventually fell into
+the hands of the New York State Police. Phiber had lost his machinery
+in the raid, but there were no charges filed against him for over a year.
+His predicament was extensively publicized on the Well, where it caused
+much resentment for police tactics. It's one thing to merely hear about
+a hacker raided or busted; it's another to see the police attacking someone
+you've come to know personally, and who has explained his motives at length.
+Through the Harper's debate on the Well, it had become clear to the
+Wellbeings that Phiber Optik was not in fact going to "hurt anything."
+In their own salad days, many Wellbeings had tasted tear-gas in pitched
+street-battles with police. They were inclined to indulgence for
+acts of civil disobedience.
+
+Wellbeings were also startled to learn of the draconian thoroughness
+of a typical hacker search-and-seizure. It took no great stretch of
+imagination for them to envision themselves suffering much the same treatment.
+
+As early as January 1990, sentiment on the Well had already begun to sour,
+and people had begun to grumble that "hackers" were getting a raw deal
+from the ham-handed powers-that-be. The resultant issue of Harper's
+magazine posed the question as to whether computer-intrusion was a "crime"
+at all. As Barlow put it later: "I've begun to wonder if we wouldn't
+also regard spelunkers as desperate criminals if AT&T owned all the caves."
+
+In February 1991, more than a year after the raid on his home,
+Phiber Optik was finally arrested, and was charged with first-degree
+Computer Tampering and Computer Trespass, New York state offenses.
+He was also charged with a theft-of-service misdemeanor, involving a complex
+free-call scam to a 900 number. Phiber Optik pled guilty to the misdemeanor
+charge, and was sentenced to 35 hours of community service.
+
+This passing harassment from the unfathomable world of straight people
+seemed to bother Optik himself little if at all. Deprived of his computer
+by the January search-and-seizure, he simply bought himself a portable
+computer so the cops could no longer monitor the phone where he lived
+with his Mom, and he went right on with his depredations, sometimes on
+live radio or in front of television cameras.
+
+The crackdown raid may have done little to dissuade Phiber Optik,
+but its galling affect on the Wellbeings was profound. As 1990 rolled on,
+the slings and arrows mounted: the Knight Lightning raid,
+the Steve Jackson raid, the nation-spanning Operation Sundevil.
+The rhetoric of law enforcement made it clear that there was,
+in fact, a concerted crackdown on hackers in progress.
+
+The hackers of the Hackers Conference, the Wellbeings, and their ilk,
+did not really mind the occasional public misapprehension of "hacking;"
+if anything, this membrane of differentiation from straight society
+made the "computer community" feel different, smarter, better.
+They had never before been confronted, however, by a concerted
+vilification campaign.
+
+Barlow's central role in the counter-struggle was one of the major
+anomalies of 1990. Journalists investigating the controversy
+often stumbled over the truth about Barlow, but they commonly
+dusted themselves off and hurried on as if nothing had happened.
+It was as if it were TOO MUCH TO BELIEVE that a 1960s freak
+from the Grateful Dead had taken on a federal law enforcement operation
+head-to-head and ACTUALLY SEEMED TO BE WINNING!
+
+Barlow had no easily detectable power-base for a political struggle
+of this kind. He had no formal legal or technical credentials.
+Barlow was, however, a computer networker of truly stellar brilliance.
+He had a poet's gift of concise, colorful phrasing. He also had a
+journalist's shrewdness, an off-the-wall, self-deprecating wit,
+and a phenomenal wealth of simple personal charm.
+
+The kind of influence Barlow possessed is fairly common currency
+in literary, artistic, or musical circles. A gifted critic can
+wield great artistic influence simply through defining
+the temper of the times, by coining the catch-phrases
+and the terms of debate that become the common currency of the period.
+(And as it happened, Barlow WAS a part-time art critic,
+with a special fondness for the Western art of Frederic Remington.)
+
+Barlow was the first commentator to adopt William Gibson's
+striking science-fictional term "cyberspace" as a synonym
+for the present-day nexus of computer and telecommunications networks.
+Barlow was insistent that cyberspace should be regarded as
+a qualitatively new world, a "frontier." According to Barlow,
+the world of electronic communications, now made visible through
+the computer screen, could no longer be usefully regarded
+as just a tangle of high-tech wiring. Instead, it had become
+a PLACE, cyberspace, which demanded a new set of metaphors,
+a new set of rules and behaviors. The term, as Barlow employed it,
+struck a useful chord, and this concept of cyberspace was picked up
+by Time, Scientific American, computer police, hackers, and even
+Constitutional scholars. "Cyberspace" now seems likely to become
+a permanent fixture of the language.
+
+Barlow was very striking in person: a tall, craggy-faced, bearded,
+deep-voiced Wyomingan in a dashing Western ensemble of jeans, jacket,
+cowboy boots, a knotted throat-kerchief and an ever-present Grateful Dead
+cloisonne lapel pin.
+
+Armed with a modem, however, Barlow was truly in his element.
+Formal hierarchies were not Barlow's strong suit; he rarely missed
+a chance to belittle the "large organizations and their drones,"
+with their uptight, institutional mindset. Barlow was very much
+of the free-spirit persuasion, deeply unimpressed by brass-hats
+and jacks-in-office. But when it came to the digital grapevine,
+Barlow was a cyberspace ad-hocrat par excellence.
+
+There was not a mighty army of Barlows. There was only one Barlow,
+and he was a fairly anomolous individual. However, the situation only
+seemed to REQUIRE a single Barlow. In fact, after 1990, many people
+must have concluded that a single Barlow was far more than
+they'd ever bargained for.
+
+Barlow's querulous mini-essay about his encounter with the FBI
+struck a strong chord on the Well. A number of other free spirits
+on the fringes of Apple Computing had come under suspicion,
+and they liked it not one whit better than he did.
+
+One of these was Mitchell Kapor, the co-inventor of the spreadsheet
+program "Lotus 1-2-3" and the founder of Lotus Development Corporation.
+Kapor had written-off the passing indignity of being fingerprinted
+down at his own local Boston FBI headquarters, but Barlow's post
+made the full national scope of the FBI's dragnet clear to Kapor.
+The issue now had Kapor's full attention. As the Secret Service
+swung into anti-hacker operation nationwide in 1990, Kapor watched
+every move with deep skepticism and growing alarm.
+
+As it happened, Kapor had already met Barlow, who had interviewed Kapor
+for a California computer journal. Like most people who met Barlow,
+Kapor had been very taken with him. Now Kapor took it upon himself
+to drop in on Barlow for a heart-to-heart talk about the situation.
+
+Kapor was a regular on the Well. Kapor had been a devotee of the
+Whole Earth Catalogsince the beginning, and treasured a complete run
+of the magazine. And Kapor not only had a modem, but a private jet.
+In pursuit of the scattered high-tech investments of Kapor Enterprises Inc.,
+his personal, multi-million dollar holding company, Kapor commonly crossed
+state lines with about as much thought as one might give to faxing a letter.
+
+The Kapor-Barlow council of June 1990, in Pinedale, Wyoming, was the start
+of the Electronic Frontier Foundation. Barlow swiftly wrote a manifesto,
+"Crime and Puzzlement," which announced his, and Kapor's, intention
+to form a political organization to "raise and disburse funds for education,
+lobbying, and litigation in the areas relating to digital speech and the
+extension of the Constitution into Cyberspace."
+
+Furthermore, proclaimed the manifesto, the foundation would
+"fund, conduct, and support legal efforts to demonstrate
+that the Secret Service has exercised prior restraint on publications,
+limited free speech, conducted improper seizure of equipment and data,
+used undue force, and generally conducted itself in a fashion which
+is arbitrary, oppressive, and unconstitutional."
+
+"Crime and Puzzlement" was distributed far and wide through computer
+networking channels, and also printed in the Whole Earth Review.
+The sudden declaration of a coherent, politicized counter-strike
+from the ranks of hackerdom electrified the community. Steve Wozniak
+(perhaps a bit stung by the NuPrometheus scandal) swiftly offered
+to match any funds Kapor offered the Foundation.
+
+John Gilmore, one of the pioneers of Sun Microsystems, immediately offered
+his own extensive financial and personal support. Gilmore, an ardent
+libertarian, was to prove an eloquent advocate of electronic privacy issues,
+especially freedom from governmental and corporate computer-assisted
+surveillance of private citizens.
+
+A second meeting in San Francisco rounded up further allies:
+Stewart Brand of the Point Foundation, virtual-reality pioneers
+Jaron Lanier and Chuck Blanchard, network entrepreneur and venture
+capitalist Nat Goldhaber. At this dinner meeting, the activists settled on
+a formal title: the Electronic Frontier Foundation, Incorporated.
+Kapor became its president. A new EFF Conference was opened on
+the Point Foundation's Well, and the Well was declared
+"the home of the Electronic Frontier Foundation."
+
+Press coverage was immediate and intense. Like their
+nineteenth-century spiritual ancestors, Alexander Graham Bell
+and Thomas Watson, the high-tech computer entrepreneurs
+of the 1970s and 1980s--people such as Wozniak, Jobs, Kapor,
+Gates, and H. Ross Perot, who had raised themselves by their bootstraps
+to dominate a glittering new industry--had always made very good copy.
+
+But while the Wellbeings rejoiced, the press in general seemed
+nonplussed by the self-declared "civilizers of cyberspace."
+EFF's insistence that the war against "hackers" involved grave
+Constitutional civil liberties issues seemed somewhat farfetched,
+especially since none of EFF's organizers were lawyers
+or established politicians. The business press in particular
+found it easier to seize on the apparent core of the story--
+that high-tech entrepreneur Mitchell Kapor had established
+a "defense fund for hackers." Was EFF a genuinely important
+political development--or merely a clique of wealthy eccentrics,
+dabbling in matters better left to the proper authorities?
+The jury was still out.
+
+But the stage was now set for open confrontation.
+And the first and the most critical battle was the
+hacker show-trial of "Knight Lightning."
+
+#
+
+It has been my practice throughout this book to refer to hackers
+only by their "handles." There is little to gain by giving
+the real names of these people, many of whom are juveniles,
+many of whom have never been convicted of any crime, and many
+of whom had unsuspecting parents who have already suffered enough.
+
+But the trial of Knight Lightning on July 24-27, 1990,
+made this particular "hacker" a nationally known public figure.
+It can do no particular harm to himself or his family if I repeat
+the long-established fact that his name is Craig Neidorf (pronounced NYE-dorf).
+
+Neidorf's jury trial took place in the United States District Court,
+Northern District of Illinois, Eastern Division, with the
+Honorable Nicholas J. Bua presiding. The United States of America
+was the plaintiff, the defendant Mr. Neidorf. The defendant's attorney
+was Sheldon T. Zenner of the Chicago firm of Katten, Muchin and Zavis.
+
+The prosecution was led by the stalwarts of the Chicago Computer Fraud
+and Abuse Task Force: William J. Cook, Colleen D. Coughlin, and
+David A. Glockner, all Assistant United States Attorneys.
+The Secret Service Case Agent was Timothy M. Foley.
+
+It will be recalled that Neidorf was the co-editor of an underground hacker
+"magazine" called Phrack. Phrack was an entirely electronic publication,
+distributed through bulletin boards and over electronic networks.
+It was amateur publication given away for free. Neidorf had never made
+any money for his work in Phrack. Neither had his unindicted co-editor
+"Taran King" or any of the numerous Phrack contributors.
+
+The Chicago Computer Fraud and Abuse Task Force, however,
+had decided to prosecute Neidorf as a fraudster.
+To formally admit that Phrack was a "magazine"
+and Neidorf a "publisher" was to open a prosecutorial
+Pandora's Box of First Amendment issues. To do this
+was to play into the hands of Zenner and his EFF advisers,
+which now included a phalanx of prominent New York civil rights
+lawyers as well as the formidable legal staff of Katten, Muchin and Zavis.
+Instead, the prosecution relied heavily on the issue of access device fraud:
+Section 1029 of Title 18, the section from which the Secret Service drew
+its most direct jurisdiction over computer crime.
+
+Neidorf's alleged crimes centered around the E911 Document.
+He was accused of having entered into a fraudulent scheme with the Prophet,
+who, it will be recalled, was the Atlanta LoD member who had illicitly
+copied the E911 Document from the BellSouth AIMSX system.
+
+The Prophet himself was also a co-defendant in the Neidorf case,
+part-and-parcel of the alleged "fraud scheme" to "steal" BellSouth's
+E911 Document (and to pass the Document across state lines,
+which helped establish the Neidorf trial as a federal case).
+The Prophet, in the spirit of full co-operation, had agreed
+to testify against Neidorf.
+
+In fact, all three of the Atlanta crew stood ready to testify against Neidorf.
+Their own federal prosecutors in Atlanta had charged the Atlanta Three with:
+(a) conspiracy, (b) computer fraud, (c) wire fraud, (d) access device fraud,
+and (e) interstate transportation of stolen property (Title 18, Sections 371,
+1030, 1343, 1029, and 2314).
+
+Faced with this blizzard of trouble, Prophet and Leftist had ducked
+any public trial and had pled guilty to reduced charges--one conspiracy
+count apiece. Urvile had pled guilty to that odd bit of Section 1029
+which makes it illegal to possess "fifteen or more" illegal access devices
+(in his case, computer passwords). And their sentences were scheduled
+for September 14, 1990--well after the Neidorf trial. As witnesses,
+they could presumably be relied upon to behave.
+
+Neidorf, however, was pleading innocent. Most everyone else caught up
+in the crackdown had "cooperated fully" and pled guilty in hope
+of reduced sentences. (Steve Jackson was a notable exception,
+of course, and had strongly protested his innocence from the
+very beginning. But Steve Jackson could not get a day in court--
+Steve Jackson had never been charged with any crime in the first place.)
+
+Neidorf had been urged to plead guilty. But Neidorf was a political science
+major and was disinclined to go to jail for "fraud" when he had not made
+any money, had not broken into any computer, and had been publishing
+a magazine that he considered protected under the First Amendment.
+
+Neidorf's trial was the ONLY legal action of the entire Crackdown
+that actually involved bringing the issues at hand out for a public test
+in front of a jury of American citizens.
+
+Neidorf, too, had cooperated with investigators. He had voluntarily
+handed over much of the evidence that had led to his own indictment.
+He had already admitted in writing that he knew that the E911 Document
+had been stolen before he had "published" it in Phrack--or, from the
+prosecution's point of view, illegally transported stolen property by wire
+in something purporting to be a "publication."
+
+But even if the "publication" of the E911 Document was not held to be a crime,
+that wouldn't let Neidorf off the hook. Neidorf had still received
+the E911 Document when Prophet had transferred it to him from Rich Andrews'
+Jolnet node. On that occasion, it certainly hadn't been "published"--
+it was hacker booty, pure and simple, transported across state lines.
+
+The Chicago Task Force led a Chicago grand jury to indict Neidorf
+on a set of charges that could have put him in jail for thirty years.
+When some of these charges were successfully challenged before Neidorf
+actually went to trial, the Chicago Task Force rearranged his
+indictment so that he faced a possible jail term of over sixty years!
+As a first offender, it was very unlikely that Neidorf would in fact
+receive a sentence so drastic; but the Chicago Task Force clearly
+intended to see Neidorf put in prison, and his conspiratorial "magazine"
+put permanently out of commission. This was a federal case, and Neidorf
+was charged with the fraudulent theft of property worth almost
+eighty thousand dollars.
+
+William Cook was a strong believer in high-profile prosecutions
+with symbolic overtones. He often published articles on his work
+in the security trade press, arguing that "a clear message had
+to be sent to the public at large and the computer community
+in particular that unauthorized attacks on computers and the theft
+of computerized information would not be tolerated by the courts."
+
+The issues were complex, the prosecution's tactics somewhat unorthodox,
+but the Chicago Task Force had proved sure-footed to date. "Shadowhawk"
+had been bagged on the wing in 1989 by the Task Force, and sentenced
+to nine months in prison, and a $10,000 fine. The Shadowhawk case involved
+charges under Section 1030, the "federal interest computer" section.
+
+Shadowhawk had not in fact been a devotee of "federal-interest" computers
+per se. On the contrary, Shadowhawk, who owned an AT&T home computer,
+seemed to cherish a special aggression toward AT&T. He had bragged on
+the underground boards "Phreak Klass 2600" and "Dr. Ripco" of his skills
+at raiding AT&T, and of his intention to crash AT&T's national phone system.
+Shadowhawk's brags were noticed by Henry Kluepfel of Bellcore Security,
+scourge of the outlaw boards, whose relations with the Chicago Task Force
+were long and intimate.
+
+The Task Force successfully established that Section 1030 applied to
+the teenage Shadowhawk, despite the objections of his defense attorney.
+Shadowhawk had entered a computer "owned" by U.S. Missile Command
+and merely "managed" by AT&T. He had also entered an AT&T computer
+located at Robbins Air Force Base in Georgia. Attacking AT&T was
+of "federal interest" whether Shadowhawk had intended it or not.
+
+The Task Force also convinced the court that a piece of AT&T
+software that Shadowhawk had illicitly copied from Bell Labs,
+the "Artificial Intelligence C5 Expert System," was worth a cool
+one million dollars. Shadowhawk's attorney had argued that
+Shadowhawk had not sold the program and had made no profit from
+the illicit copying. And in point of fact, the C5 Expert System
+was experimental software, and had no established market value
+because it had never been on the market in the first place.
+AT&T's own assessment of a "one million dollar" figure for its
+own intangible property was accepted without challenge
+by the court, however. And the court concurred with
+the government prosecutors that Shadowhawk showed clear
+"intent to defraud" whether he'd gotten any money or not.
+Shadowhawk went to jail.
+
+The Task Force's other best-known triumph had been the conviction
+and jailing of "Kyrie." Kyrie, a true denizen of the digital
+criminal underground, was a 36-year-old Canadian woman,
+convicted and jailed for telecommunications fraud in Canada.
+After her release from prison, she had fled the wrath of Canada Bell
+and the Royal Canadian Mounted Police, and eventually settled,
+very unwisely, in Chicago.
+
+"Kyrie," who also called herself "Long Distance Information,"
+specialized in voice-mail abuse. She assembled large numbers
+of hot long-distance codes, then read them aloud into a series
+of corporate voice-mail systems. Kyrie and her friends were
+electronic squatters in corporate voice-mail systems,
+using them much as if they were pirate bulletin boards,
+then moving on when their vocal chatter clogged the system
+and the owners necessarily wised up. Kyrie's camp followers
+were a loose tribe of some hundred and fifty phone-phreaks,
+who followed her trail of piracy from machine to machine,
+ardently begging for her services and expertise.
+
+Kyrie's disciples passed her stolen credit-card numbers,
+in exchange for her stolen "long distance information."
+Some of Kyrie's clients paid her off in cash, by scamming
+credit-card cash advances from Western Union.
+
+Kyrie travelled incessantly, mostly through airline tickets
+and hotel rooms that she scammed through stolen credit cards.
+Tiring of this, she found refuge with a fellow female phone
+phreak in Chicago. Kyrie's hostess, like a surprising number
+of phone phreaks, was blind. She was also physically disabled.
+Kyrie allegedly made the best of her new situation by applying for,
+and receiving, state welfare funds under a false identity as
+a qualified caretaker for the handicapped.
+
+Sadly, Kyrie's two children by a former marriage had also vanished
+underground with her; these pre-teen digital refugees had no legal
+American identity, and had never spent a day in school.
+
+Kyrie was addicted to technical mastery and enthralled by her own
+cleverness and the ardent worship of her teenage followers.
+This foolishly led her to phone up Gail Thackeray in Arizona,
+to boast, brag, strut, and offer to play informant.
+Thackeray, however, had already learned far more
+than enough about Kyrie, whom she roundly despised
+as an adult criminal corrupting minors, a "female Fagin."
+Thackeray passed her tapes of Kyrie's boasts to the Secret Service.
+
+Kyrie was raided and arrested in Chicago in May 1989.
+She confessed at great length and pled guilty.
+
+In August 1990, Cook and his Task Force colleague Colleen Coughlin
+sent Kyrie to jail for 27 months, for computer and telecommunications fraud.
+This was a markedly severe sentence by the usual wrist-slapping standards
+of "hacker" busts. Seven of Kyrie's foremost teenage disciples were also
+indicted and convicted. The Kyrie "high-tech street gang," as Cook
+described it, had been crushed. Cook and his colleagues had been
+the first ever to put someone in prison for voice-mail abuse.
+Their pioneering efforts had won them attention and kudos.
+
+In his article on Kyrie, Cook drove the message home to the readers
+of Security Management magazine, a trade journal for corporate
+security professionals. The case, Cook said, and Kyrie's stiff sentence,
+"reflect a new reality for hackers and computer crime victims in the
+'90s. . . . Individuals and corporations who report computer
+and telecommunications crimes can now expect that their cooperation
+with federal law enforcement will result in meaningful punishment.
+Companies and the public at large must report computer-enhanced
+crimes if they want prosecutors and the course to protect their rights
+to the tangible and intangible property developed and stored on computers."
+
+Cook had made it his business to construct this "new reality for hackers."
+He'd also made it his business to police corporate property rights
+to the intangible.
+
+Had the Electronic Frontier Foundation been a "hacker defense fund"
+as that term was generally understood, they presumably would have stood up
+for Kyrie. Her 1990 sentence did indeed send a "message" that federal heat
+was coming down on "hackers." But Kyrie found no defenders at EFF,
+or anywhere else, for that matter. EFF was not a bail-out fund
+for electronic crooks.
+
+The Neidorf case paralleled the Shadowhawk case in certain ways.
+The victim once again was allowed to set the value of the "stolen" property.
+Once again Kluepfel was both investigator and technical advisor.
+Once again no money had changed hands, but the "intent to defraud" was central.
+
+The prosecution's case showed signs of weakness early on. The Task Force
+had originally hoped to prove Neidorf the center of a nationwide
+Legion of Doom criminal conspiracy. The Phrack editors threw physical
+get-togethers every summer, which attracted hackers from across the country;
+generally two dozen or so of the magazine's favorite contributors and readers.
+(Such conventions were common in the hacker community; 2600 Magazine,
+for instance, held public meetings of hackers in New York, every month.)
+LoD heavy-dudes were always a strong presence at these Phrack-sponsored
+"Summercons."
+
+In July 1988, an Arizona hacker named "Dictator" attended Summercon
+in Neidorf's home town of St. Louis. Dictator was one of Gail Thackeray's
+underground informants; Dictator's underground board in Phoenix was
+a sting operation for the Secret Service. Dictator brought an undercover
+crew of Secret Service agents to Summercon. The agents bored spyholes
+through the wall of Dictator's hotel room in St Louis, and videotaped
+the frolicking hackers through a one-way mirror. As it happened,
+however, nothing illegal had occurred on videotape, other than the
+guzzling of beer by a couple of minors. Summercons were social events,
+not sinister cabals. The tapes showed fifteen hours of raucous laughter,
+pizza-gobbling, in-jokes and back-slapping.
+
+Neidorf's lawyer, Sheldon Zenner, saw the Secret Service tapes
+before the trial. Zenner was shocked by the complete harmlessness
+of this meeting, which Cook had earlier characterized as a sinister
+interstate conspiracy to commit fraud. Zenner wanted to show the
+Summercon tapes to the jury. It took protracted maneuverings
+by the Task Force to keep the tapes from the jury as "irrelevant."
+
+The E911 Document was also proving a weak reed. It had originally
+been valued at $79,449. Unlike Shadowhawk's arcane Artificial Intelligence
+booty, the E911 Document was not software--it was written in English.
+Computer-knowledgeable people found this value--for a twelve-page
+bureaucratic document--frankly incredible. In his "Crime and Puzzlement"
+manifesto for EFF, Barlow commented: "We will probably never know how
+this figure was reached or by whom, though I like to imagine an appraisal
+team consisting of Franz Kafka, Joseph Heller, and Thomas Pynchon."
+
+As it happened, Barlow was unduly pessimistic. The EFF did, in fact,
+eventually discover exactly how this figure was reached, and by whom--
+but only in 1991, long after the Neidorf trial was over.
+
+Kim Megahee, a Southern Bell security manager,
+had arrived at the document's value by simply adding up
+the "costs associated with the production" of the E911 Document.
+Those "costs" were as follows:
+
+1. A technical writer had been hired to research and write the E911 Document.
+ 200 hours of work, at $35 an hour, cost : $7,000. A Project Manager had
+ overseen the technical writer. 200 hours, at $31 an hour, made: $6,200.
+
+2. A week of typing had cost $721 dollars. A week of formatting had
+ cost $721. A week of graphics formatting had cost $742.
+
+3. Two days of editing cost $367.
+
+4. A box of order labels cost five dollars.
+
+5. Preparing a purchase order for the Document, including typing
+ and the obtaining of an authorizing signature from within the
+ BellSouth bureaucracy, cost $129.
+
+6. Printing cost $313. Mailing the Document to fifty people
+ took fifty hours by a clerk, and cost $858.
+
+7. Placing the Document in an index took two clerks an hour each,
+ totalling $43.
+
+Bureaucratic overhead alone, therefore, was alleged to have cost
+a whopping $17,099. According to Mr. Megahee, the typing
+of a twelve-page document had taken a full week. Writing it
+had taken five weeks, including an overseer who apparently
+did nothing else but watch the author for five weeks.
+Editing twelve pages had taken two days. Printing and mailing
+an electronic document (which was already available on the
+Southern Bell Data Network to any telco employee who needed it),
+had cost over a thousand dollars.
+
+But this was just the beginning. There were also the HARDWARE EXPENSES.
+Eight hundred fifty dollars for a VT220 computer monitor.
+THIRTY-ONE THOUSAND DOLLARS for a sophisticated VAXstation II computer.
+Six thousand dollars for a computer printer. TWENTY-TWO THOUSAND DOLLARS
+for a copy of "Interleaf" software. Two thousand five hundred dollars
+for VMS software. All this to create the twelve-page Document.
+
+Plus ten percent of the cost of the software and the hardware, for maintenance.
+(Actually, the ten percent maintenance costs, though mentioned, had been left
+off the final $79,449 total, apparently through a merciful oversight).
+
+Mr. Megahee's letter had been mailed directly to William Cook himself,
+at the office of the Chicago federal attorneys. The United States Government
+accepted these telco figures without question.
+
+As incredulity mounted, the value of the E911 Document was officially
+revised downward. This time, Robert Kibler of BellSouth Security
+estimated the value of the twelve pages as a mere $24,639.05--based,
+purportedly, on "R&D costs." But this specific estimate,
+right down to the nickel, did not move the skeptics at all;
+in fact it provoked open scorn and a torrent of sarcasm.
+
+The financial issues concerning theft of proprietary information
+have always been peculiar. It could be argued that BellSouth
+had not "lost" its E911 Document at all in the first place,
+and therefore had not suffered any monetary damage from this "theft."
+And Sheldon Zenner did in fact argue this at Neidorf's trial--
+that Prophet's raid had not been "theft," but was better understood
+as illicit copying.
+
+The money, however, was not central to anyone's true purposes in this trial.
+It was not Cook's strategy to convince the jury that the E911 Document
+was a major act of theft and should be punished for that reason alone.
+His strategy was to argue that the E911 Document was DANGEROUS.
+It was his intention to establish that the E911 Document was "a road-map"
+to the Enhanced 911 System. Neidorf had deliberately and recklessly
+distributed a dangerous weapon. Neidorf and the Prophet did not care
+(or perhaps even gloated at the sinister idea) that the E911 Document
+could be used by hackers to disrupt 911 service, "a life line for every
+person certainly in the Southern Bell region of the United States,
+and indeed, in many communities throughout the United States,"
+in Cook's own words. Neidorf had put people's lives in danger.
+
+In pre-trial maneuverings, Cook had established that the E911 Document
+was too hot to appear in the public proceedings of the Neidorf trial.
+The JURY ITSELF would not be allowed to ever see this Document,
+lest it slip into the official court records, and thus into the hands
+of the general public, and, thus, somehow, to malicious hackers
+who might lethally abuse it.
+
+Hiding the E911 Document from the jury may have been a
+clever legal maneuver, but it had a severe flaw. There were,
+in point of fact, hundreds, perhaps thousands, of people,
+already in possession of the E911 Document, just as Phrack
+had published it. Its true nature was already obvious
+to a wide section of the interested public (all of whom,
+by the way, were, at least theoretically, party to
+a gigantic wire-fraud conspiracy). Most everyone
+in the electronic community who had a modem and any
+interest in the Neidorf case already had a copy of the Document.
+It had already been available in Phrack for over a year.
+
+People, even quite normal people without any particular
+prurient interest in forbidden knowledge, did not shut their eyes
+in terror at the thought of beholding a "dangerous" document
+from a telephone company. On the contrary, they tended to trust
+their own judgement and simply read the Document for themselves.
+And they were not impressed.
+
+One such person was John Nagle. Nagle was a forty-one-year-old
+professional programmer with a masters' degree in computer science
+from Stanford. He had worked for Ford Aerospace, where he had invented
+a computer-networking technique known as the "Nagle Algorithm,"
+and for the prominent Californian computer-graphics firm "Autodesk,"
+where he was a major stockholder.
+
+Nagle was also a prominent figure on the Well, much respected
+for his technical knowledgeability.
+
+Nagle had followed the civil-liberties debate closely,
+for he was an ardent telecommunicator. He was no particular friend
+of computer intruders, but he believed electronic publishing
+had a great deal to offer society at large, and attempts
+to restrain its growth, or to censor free electronic expression,
+strongly roused his ire.
+
+The Neidorf case, and the E911 Document, were both being discussed
+in detail on the Internet, in an electronic publication called Telecom Digest.
+Nagle, a longtime Internet maven, was a regular reader of Telecom Digest.
+Nagle had never seen a copy of Phrack, but the implications of the case
+disturbed him.
+
+While in a Stanford bookstore hunting books on robotics,
+Nagle happened across a book called The Intelligent Network.
+Thumbing through it at random, Nagle came across an entire chapter
+meticulously detailing the workings of E911 police emergency systems.
+This extensive text was being sold openly, and yet in Illinois
+a young man was in danger of going to prison for publishing
+a thin six-page document about 911 service.
+
+Nagle made an ironic comment to this effect in Telecom Digest.
+From there, Nagle was put in touch with Mitch Kapor,
+and then with Neidorf's lawyers.
+
+Sheldon Zenner was delighted to find a computer telecommunications expert
+willing to speak up for Neidorf, one who was not a wacky teenage "hacker."
+Nagle was fluent, mature, and respectable; he'd once had a federal
+security clearance.
+
+Nagle was asked to fly to Illinois to join the defense team.
+
+Having joined the defense as an expert witness, Nagle read the entire
+E911 Document for himself. He made his own judgement about its potential
+for menace.
+
+The time has now come for you yourself, the reader, to have a look
+at the E911 Document. This six-page piece of work was the pretext
+for a federal prosecution that could have sent an electronic publisher
+to prison for thirty, or even sixty, years. It was the pretext
+for the search and seizure of Steve Jackson Games, a legitimate publisher
+of printed books. It was also the formal pretext for the search
+and seizure of the Mentor's bulletin board, "Phoenix Project,"
+and for the raid on the home of Erik Bloodaxe. It also had much
+to do with the seizure of Richard Andrews' Jolnet node
+and the shutdown of Charles Boykin's AT&T node.
+The E911 Document was the single most important piece
+of evidence in the Hacker Crackdown. There can be no real
+and legitimate substitute for the Document itself.
+
+
+==Phrack Inc.==
+
+Volume Two, Issue 24, File 5 of 13
+
+Control Office Administration
+Of Enhanced 911 Services For
+Special Services and Account Centers
+
+by the Eavesdropper
+
+March, 1988
+
+
+Description of Service
+~~~~~~~~~~~~~~~~~~~~~
+The control office for Emergency 911 service is assigned in
+accordance with the existing standard guidelines to one of
+the following centers:
+
+o Special Services Center (SSC)
+o Major Accounts Center (MAC)
+o Serving Test Center (STC)
+o Toll Control Center (TCC)
+
+The SSC/MAC designation is used in this document interchangeably
+for any of these four centers. The Special Services Centers (SSCs)
+or Major Account Centers (MACs) have been designated as the trouble
+reporting contact for all E911 customer (PSAP) reported troubles.
+Subscribers who have trouble on an E911 call will continue
+to contact local repair service (CRSAB) who will refer the
+trouble to the SSC/MAC, when appropriate.
+
+Due to the critical nature of E911 service, the control
+and timely repair of troubles is demanded. As the primary
+E911 customer contact, the SSC/MAC is in the unique position
+to monitor the status of the trouble and insure its resolution.
+
+System Overview
+~~~~~~~~~~~~~~
+The number 911 is intended as a nationwide universal
+telephone number which provides the public with direct
+access to a Public Safety Answering Point (PSAP). A PSAP
+is also referred to as an Emergency Service Bureau (ESB).
+A PSAP is an agency or facility which is authorized by a
+municipality to receive and respond to police, fire and/or
+ambulance services. One or more attendants are located
+at the PSAP facilities to receive and handle calls of an
+emergency nature in accordance with the local municipal
+requirements.
+
+An important advantage of E911 emergency service is
+improved (reduced) response times for emergency
+services. Also close coordination among agencies
+providing various emergency services is a valuable
+capability provided by E911 service.
+
+1A ESS is used as the tandem office for the E911 network to
+route all 911 calls to the correct (primary) PSAP designated
+to serve the calling station. The E911 feature was
+developed primarily to provide routing to the correct PSAP
+for all 911 calls. Selective routing allows a 911 call
+originated from a particular station located in a particular
+district, zone, or town, to be routed to the primary PSAP
+designated to serve that customer station regardless of
+wire center boundaries. Thus, selective routing eliminates
+the problem of wire center boundaries not coinciding with
+district or other political boundaries.
+
+The services available with the E911 feature include:
+
+Forced Disconnect Default Routing
+Alternative Routing Night Service
+Selective Routing Automatic Number
+Identification (ANI)
+Selective Transfer Automatic Location
+Identification (ALI)
+
+
+Preservice/Installation Guidelines
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+When a contract for an E911 system has been signed, it is
+the responsibility of Network Marketing to establish an
+implementation/cutover committee which should include
+a representative from the SSC/MAC. Duties of the E911
+Implementation Team include coordination of all phases
+of the E911 system deployment and the formation of an
+on-going E911 maintenance subcommittee.
+
+Marketing is responsible for providing the following
+customer specific information to the SSC/MAC prior to
+the start of call through testing:
+
+o All PSAP's (name, address, local contact)
+o All PSAP circuit ID's
+o 1004 911 service request including PSAP details on each PSAP
+ (1004 Section K, L, M)
+o Network configuration
+o Any vendor information (name, telephone number, equipment)
+
+The SSC/MAC needs to know if the equipment and sets
+at the PSAP are maintained by the BOCs, an independent
+company, or an outside vendor, or any combination.
+This information is then entered on the PSAP profile sheets
+and reviewed quarterly for changes, additions and deletions.
+
+Marketing will secure the Major Account Number (MAN)
+and provide this number to Corporate Communications
+so that the initial issue of the service orders carry
+the MAN and can be tracked by the SSC/MAC via CORDNET.
+PSAP circuits are official services by definition.
+
+All service orders required for the installation of the E911
+system should include the MAN assigned to the city/county
+which has purchased the system.
+
+In accordance with the basic SSC/MAC strategy for provisioning,
+the SSC/MAC will be Overall Control Office (OCO) for all Node
+to PSAP circuits (official services) and any other services
+for this customer. Training must be scheduled for all SSC/MAC
+involved personnel during the pre-service stage of the project.
+
+The E911 Implementation Team will form the on-going
+maintenance subcommittee prior to the initial
+implementation of the E911 system. This sub-committee
+will establish post implementation quality assurance
+procedures to ensure that the E911 system continues to
+provide quality service to the customer.
+Customer/Company training, trouble reporting interfaces
+for the customer, telephone company and any involved
+independent telephone companies needs to be addressed
+and implemented prior to E911 cutover. These functions
+can be best addressed by the formation of a sub-
+committee of the E911 Implementation Team to set up
+guidelines for and to secure service commitments of
+interfacing organizations. A SSC/MAC supervisor should
+chair this subcommittee and include the following
+organizations:
+
+1) Switching Control Center
+ - E911 translations
+ - Trunking
+ - End office and Tandem office hardware/software
+2) Recent Change Memory Administration Center
+ - Daily RC update activity for TN/ESN translations
+ - Processes validity errors and rejects
+3) Line and Number Administration
+ - Verification of TN/ESN translations
+4) Special Service Center/Major Account Center
+ - Single point of contact for all PSAP and Node to host troubles
+ - Logs, tracks & statusing of all trouble reports
+ - Trouble referral, follow up, and escalation
+ - Customer notification of status and restoration
+ - Analyzation of "chronic" troubles
+ - Testing, installation and maintenance of E911 circuits
+5) Installation and Maintenance (SSIM/I&M)
+ - Repair and maintenance of PSAP equipment and Telco owned sets
+6) Minicomputer Maintenance Operations Center
+ - E911 circuit maintenance (where applicable)
+7) Area Maintenance Engineer
+ - Technical assistance on voice (CO-PSAP) network related E911 troubles
+
+
+Maintenance Guidelines
+~~~~~~~~~~~~~~~~~~~~~
+The CCNC will test the Node circuit from the 202T at the
+Host site to the 202T at the Node site. Since Host to Node
+(CCNC to MMOC) circuits are official company services,
+the CCNC will refer all Node circuit troubles to the
+SSC/MAC. The SSC/MAC is responsible for the testing
+and follow up to restoration of these circuit troubles.
+
+Although Node to PSAP circuit are official services, the
+MMOC will refer PSAP circuit troubles to the appropriate
+SSC/MAC. The SSC/MAC is responsible for testing and
+follow up to restoration of PSAP circuit troubles.
+
+The SSC/MAC will also receive reports from
+CRSAB/IMC(s) on subscriber 911 troubles when they are
+not line troubles. The SSC/MAC is responsible for testing
+and restoration of these troubles.
+
+Maintenance responsibilities are as follows:
+
+SCC@ Voice Network (ANI to PSAP)
+@SCC responsible for tandem switch
+
+SSIM/I&M PSAP Equipment (Modems, CIU's, sets)
+Vendor PSAP Equipment (when CPE)
+SSC/MAC PSAP to Node circuits, and tandem to
+ PSAP voice circuits (EMNT)
+MMOC Node site (Modems, cables, etc)
+
+Note: All above work groups are required to resolve troubles
+by interfacing with appropriate work groups for resolution.
+
+The Switching Control Center (SCC) is responsible for
+E911/1AESS translations in tandem central offices.
+These translations route E911 calls, selective transfer,
+default routing, speed calling, etc., for each PSAP.
+The SCC is also responsible for troubleshooting on
+the voice network (call originating to end office tandem equipment).
+
+For example, ANI failures in the originating offices would
+be a responsibility of the SCC.
+
+Recent Change Memory Administration Center (RCMAC) performs
+the daily tandem translation updates (recent change)
+for routing of individual telephone numbers.
+
+Recent changes are generated from service order activity
+(new service, address changes, etc.) and compiled into
+a daily file by the E911 Center (ALI/DMS E911 Computer).
+
+SSIM/I&M is responsible for the installation and repair of
+PSAP equipment. PSAP equipment includes ANI Controller,
+ALI Controller, data sets, cables, sets, and other peripheral
+equipment that is not vendor owned. SSIM/I&M is responsible
+for establishing maintenance test kits, complete with spare parts
+for PSAP maintenance. This includes test gear, data sets,
+and ANI/ALI Controller parts.
+
+Special Services Center (SSC) or Major Account Center
+(MAC) serves as the trouble reporting contact for all
+(PSAP) troubles reported by customer. The SSC/MAC
+refers troubles to proper organizations for handling and
+tracks status of troubles, escalating when necessary.
+The SSC/MAC will close out troubles with customer.
+The SSC/MAC will analyze all troubles and tracks "chronic"
+PSAP troubles.
+
+Corporate Communications Network Center (CCNC) will
+test and refer troubles on all node to host circuits.
+All E911 circuits are classified as official company property.
+
+The Minicomputer Maintenance Operations Center
+(MMOC) maintains the E911 (ALI/DMS) computer
+hardware at the Host site. This MMOC is also responsible
+for monitoring the system and reporting certain PSAP
+and system problems to the local MMOC's, SCC's or
+SSC/MAC's. The MMOC personnel also operate software
+programs that maintain the TN data base under the
+direction of the E911 Center. The maintenance of the
+NODE computer (the interface between the PSAP and the
+ALI/DMS computer) is a function of the MMOC at the
+NODE site. The MMOC's at the NODE sites may also be
+involved in the testing of NODE to Host circuits.
+The MMOC will also assist on Host to PSAP and data network
+related troubles not resolved through standard trouble
+clearing procedures.
+
+Installation And Maintenance Center (IMC) is responsible
+for referral of E911 subscriber troubles that are not subscriber
+line problems.
+
+E911 Center - Performs the role of System Administration
+and is responsible for overall operation of the E911
+computer software. The E911 Center does A-Z trouble
+analysis and provides statistical information on the
+performance of the system.
+
+This analysis includes processing PSAP inquiries (trouble
+reports) and referral of network troubles. The E911 Center
+also performs daily processing of tandem recent change
+and provides information to the RCMAC for tandem input.
+The E911 Center is responsible for daily processing
+of the ALI/DMS computer data base and provides error files,
+etc. to the Customer Services department for investigation and correction.
+The E911 Center participates in all system implementations and on-going
+maintenance effort and assists in the development of procedures,
+training and education of information to all groups.
+
+Any group receiving a 911 trouble from the SSC/MAC should
+close out the trouble with the SSC/MAC or provide a status
+if the trouble has been referred to another group.
+This will allow the SSC/MAC to provide a status back
+to the customer or escalate as appropriate.
+
+Any group receiving a trouble from the Host site (MMOC
+or CCNC) should close the trouble back to that group.
+
+The MMOC should notify the appropriate SSC/MAC
+when the Host, Node, or all Node circuits are down so that
+the SSC/MAC can reply to customer reports that may be
+called in by the PSAPs. This will eliminate duplicate
+reporting of troubles. On complete outages the MMOC
+will follow escalation procedures for a Node after two (2)
+hours and for a PSAP after four (4) hours. Additionally the
+MMOC will notify the appropriate SSC/MAC when the
+Host, Node, or all Node circuits are down.
+
+The PSAP will call the SSC/MAC to report E911 troubles.
+The person reporting the E911 trouble may not have a
+circuit I.D. and will therefore report the PSAP name and
+address. Many PSAP troubles are not circuit specific. In
+those instances where the caller cannot provide a circuit
+I.D., the SSC/MAC will be required to determine the
+circuit I.D. using the PSAP profile. Under no circumstances
+will the SSC/MAC Center refuse to take the trouble.
+The E911 trouble should be handled as quickly as possible,
+with the SSC/MAC providing as much assistance as
+possible while taking the trouble report from the caller.
+
+The SSC/MAC will screen/test the trouble to determine the
+appropriate handoff organization based on the following criteria:
+
+PSAP equipment problem: SSIM/I&M
+Circuit problem: SSC/MAC
+Voice network problem: SCC (report trunk group number)
+Problem affecting multiple PSAPs (No ALI report from
+all PSAPs): Contact the MMOC to check for NODE or
+Host computer problems before further testing.
+
+The SSC/MAC will track the status of reported troubles
+and escalate as appropriate. The SSC/MAC will close out
+customer/company reports with the initiating contact.
+Groups with specific maintenance responsibilities,
+defined above, will investigate "chronic" troubles upon
+request from the SSC/MAC and the ongoing maintenance subcommittee.
+
+All "out of service" E911 troubles are priority one type reports.
+One link down to a PSAP is considered a priority one trouble
+and should be handled as if the PSAP was isolated.
+
+The PSAP will report troubles with the ANI controller, ALI
+controller or set equipment to the SSC/MAC.
+
+NO ANI: Where the PSAP reports NO ANI (digital
+display screen is blank) ask if this condition exists on all
+screens and on all calls. It is important to differentiate
+between blank screens and screens displaying 911-00XX,
+or all zeroes.
+
+When the PSAP reports all screens on all calls, ask if there
+is any voice contact with callers. If there is no voice
+contact the trouble should be referred to the SCC
+immediately since 911 calls are not getting through which
+may require alternate routing of calls to another PSAP.
+
+When the PSAP reports this condition on all screens
+but not all calls and has voice contact with callers,
+the report should be referred to SSIM/I&M for dispatch.
+The SSC/MAC should verify with the SCC that ANI
+is pulsing before dispatching SSIM.
+
+When the PSAP reports this condition on one screen for
+all calls (others work fine) the trouble should be referred
+to SSIM/I&M for dispatch, because the trouble is isolated to
+one piece of equipment at the customer premise.
+
+An ANI failure (i.e. all zeroes) indicates that the ANI has
+not been received by the PSAP from the tandem office or
+was lost by the PSAP ANI controller. The PSAP may
+receive "02" alarms which can be caused by the ANI
+controller logging more than three all zero failures on the
+same trunk. The PSAP has been instructed to report this
+condition to the SSC/MAC since it could indicate an
+equipment trouble at the PSAP which might be affecting
+all subscribers calling into the PSAP. When all zeroes are
+being received on all calls or "02" alarms continue, a tester
+should analyze the condition to determine the appropriate
+action to be taken. The tester must perform cooperative
+testing with the SCC when there appears to be a problem
+on the Tandem-PSAP trunks before requesting dispatch.
+
+When an occasional all zero condition is reported,
+the SSC/MAC should dispatch SSIM/I&M to routine
+equipment on a "chronic" troublesweep.
+
+The PSAPs are instructed to report incidental ANI failures
+to the BOC on a PSAP inquiry trouble ticket (paper) that
+is sent to the Customer Services E911 group and forwarded
+to E911 center when required. This usually involves only a
+particular telephone number and is not a condition that
+would require a report to the SSC/MAC. Multiple ANI
+failures which our from the same end office (XX denotes
+end office), indicate a hard trouble condition may exist
+in the end office or end office tandem trunks. The PSAP will
+report this type of condition to the SSC/MAC and the
+SSC/MAC should refer the report to the SCC responsible
+for the tandem office. NOTE: XX is the ESCO (Emergency
+Service Number) associated with the incoming 911 trunks
+into the tandem. It is important that the C/MAC tell the
+SCC what is displayed at the PSAP (i.e. 911-0011) which
+indicates to the SCC which end office is in trouble.
+
+Note: It is essential that the PSAP fill out inquiry form
+on every ANI failure.
+
+The PSAP will report a trouble any time an address is not
+received on an address display (screen blank) E911 call.
+(If a record is not in the 911 data base or an ANI failure
+is encountered, the screen will provide a display noticing
+such condition). The SSC/MAC should verify with the PSAP
+whether the NO ALI condition is on one screen or all screens.
+
+When the condition is on one screen (other screens
+receive ALI information) the SSC/MAC will request
+SSIM/I&M to dispatch.
+
+If no screens are receiving ALI information, there is usually
+a circuit trouble between the PSAP and the Host computer.
+The SSC/MAC should test the trouble and refer for restoral.
+
+Note: If the SSC/MAC receives calls from multiple
+PSAP's, all of which are receiving NO ALI, there is a
+problem with the Node or Node to Host circuits or the
+Host computer itself. Before referring the trouble the
+SSC/MAC should call the MMOC to inquire if the Node
+or Host is in trouble.
+
+Alarm conditions on the ANI controller digital display at
+the PSAP are to be reported by the PSAP's. These alarms
+can indicate various trouble conditions so the SSC/MAC
+should ask the PSAP if any portion of the E911 system
+is not functioning properly.
+
+The SSC/MAC should verify with the PSAP attendant that
+the equipment's primary function is answering E911 calls.
+If it is, the SSC/MAC should request a dispatch SSIM/I&M.
+If the equipment is not primarily used for E911,
+then the SSC/MAC should advise PSAP to contact their CPE vendor.
+
+Note: These troubles can be quite confusing when the
+PSAP has vendor equipment mixed in with equipment
+that the BOC maintains. The Marketing representative
+should provide the SSC/MAC information concerning any
+unusual or exception items where the PSAP should
+contact their vendor. This information should be included
+in the PSAP profile sheets.
+
+ANI or ALI controller down: When the host computer sees
+the PSAP equipment down and it does not come back up,
+the MMOC will report the trouble to the SSC/MAC;
+the equipment is down at the PSAP, a dispatch will be required.
+
+PSAP link (circuit) down: The MMOC will provide the
+SSC/MAC with the circuit ID that the Host computer
+indicates in trouble. Although each PSAP has two circuits,
+when either circuit is down the condition must be treated
+as an emergency since failure of the second circuit will
+cause the PSAP to be isolated.
+
+Any problems that the MMOC identifies from the Node
+location to the Host computer will be handled directly
+with the appropriate MMOC(s)/CCNC.
+
+Note: The customer will call only when a problem is
+apparent to the PSAP. When only one circuit is down to
+the PSAP, the customer may not be aware there is a
+trouble, even though there is one link down,
+notification should appear on the PSAP screen.
+Troubles called into the SSC/MAC from the MMOC
+or other company employee should not be closed out
+by calling the PSAP since it may result in the
+customer responding that they do not have a trouble.
+These reports can only be closed out by receiving
+information that the trouble was fixed and by checking
+with the company employee that reported the trouble.
+The MMOC personnel will be able to verify that the
+trouble has cleared by reviewing a printout from the host.
+
+When the CRSAB receives a subscriber complaint
+(i.e., cannot dial 911) the RSA should obtain as much
+information as possible while the customer is on the line.
+
+For example, what happened when the subscriber dialed 911?
+The report is automatically directed to the IMC for subscriber line testing.
+When no line trouble is found, the IMC will refer the trouble condition
+to the SSC/MAC. The SSC/MAC will contact Customer Services E911 Group
+and verify that the subscriber should be able to call 911 and obtain the ESN.
+The SSC/MAC will verify the ESN via 2SCCS. When both verifications match,
+the SSC/MAC will refer the report to the SCC responsible for the 911 tandem
+office for investigation and resolution. The MAC is responsible for tracking
+the trouble and informing the IMC when it is resolved.
+
+
+For more information, please refer to E911 Glossary of Terms.
+End of Phrack File
+_____________________________________
+
+
+The reader is forgiven if he or she was entirely unable to read
+this document. John Perry Barlow had a great deal of fun at its expense,
+in "Crime and Puzzlement:" "Bureaucrat-ese of surpassing opacity. . . .
+To read the whole thing straight through without entering coma requires
+either a machine or a human who has too much practice thinking like one.
+Anyone who can understand it fully and fluidly had altered his consciousness
+beyond the ability to ever again read Blake, Whitman, or Tolstoy. . . .
+the document contains little of interest to anyone who is not a student
+of advanced organizational sclerosis."
+
+With the Document itself to hand, however, exactly as it was published
+(in its six-page edited form) in Phrack, the reader may be able to verify
+a few statements of fact about its nature. First, there is no software,
+no computer code, in the Document. It is not computer-programming language
+like FORTRAN or C++, it is English; all the sentences have nouns and verbs
+and punctuation. It does not explain how to break into the E911 system.
+It does not suggest ways to destroy or damage the E911 system.
+
+There are no access codes in the Document. There are no computer passwords.
+It does not explain how to steal long distance service. It does not explain
+how to break in to telco switching stations. There is nothing in it about
+using a personal computer or a modem for any purpose at all, good or bad.
+
+Close study will reveal that this document is not about machinery.
+The E911 Document is about ADMINISTRATION. It describes how one creates
+and administers certain units of telco bureaucracy:
+Special Service Centers and Major Account Centers (SSC/MAC).
+It describes how these centers should distribute responsibility
+for the E911 service, to other units of telco bureaucracy,
+in a chain of command, a formal hierarchy. It describes
+who answers customer complaints, who screens calls,
+who reports equipment failures, who answers those reports,
+who handles maintenance, who chairs subcommittees,
+who gives orders, who follows orders, WHO tells WHOM what to do.
+The Document is not a "roadmap" to computers.
+The Document is a roadmap to PEOPLE.
+
+As an aid to breaking into computer systems, the Document is USELESS.
+As an aid to harassing and deceiving telco people, however, the Document
+might prove handy (especially with its Glossary, which I have not included).
+An intense and protracted study of this Document and its Glossary,
+combined with many other such documents, might teach one to speak like
+a telco employee. And telco people live by SPEECH--they live by phone
+communication. If you can mimic their language over the phone,
+you can "social-engineer" them. If you can con telco people, you can
+wreak havoc among them. You can force them to no longer trust one another;
+you can break the telephonic ties that bind their community; you can make
+them paranoid. And people will fight harder to defend their community
+than they will fight to defend their individual selves.
+
+This was the genuine, gut-level threat posed by Phrack magazine.
+The real struggle was over the control of telco language,
+the control of telco knowledge. It was a struggle to defend the social
+"membrane of differentiation" that forms the walls of the telco
+community's ivory tower --the special jargon that allows telco
+professionals to recognize one another, and to exclude charlatans,
+thieves, and upstarts. And the prosecution brought out this fact.
+They repeatedly made reference to the threat posed to telco professionals
+by hackers using "social engineering."
+
+However, Craig Neidorf was not on trial for learning to speak like
+a professional telecommunications expert. Craig Neidorf was on trial
+for access device fraud and transportation of stolen property.
+He was on trial for stealing a document that was purportedly
+highly sensitive and purportedly worth tens of thousands of dollars.
+
+#
+
+John Nagle read the E911 Document. He drew his own conclusions.
+And he presented Zenner and his defense team with an overflowing box
+of similar material, drawn mostly from Stanford University's
+engineering libraries. During the trial, the defense team--Zenner,
+half-a-dozen other attorneys, Nagle, Neidorf, and computer-security
+expert Dorothy Denning, all pored over the E911 Document line-by-line.
+
+On the afternoon of July 25, 1990, Zenner began to cross-examine
+a woman named Billie Williams, a service manager for Southern Bell
+in Atlanta. Ms. Williams had been responsible for the E911 Document.
+(She was not its author--its original "author" was a Southern Bell
+staff manager named Richard Helms. However, Mr. Helms should not bear
+the entire blame; many telco staff people and maintenance personnel
+had amended the Document. It had not been so much "written" by a
+single author, as built by committee out of concrete-blocks of jargon.)
+
+Ms. Williams had been called as a witness for the prosecution,
+and had gamely tried to explain the basic technical structure
+of the E911 system, aided by charts.
+
+Now it was Zenner's turn. He first established that the
+"proprietary stamp" that BellSouth had used on the E911 Document
+was stamped on EVERY SINGLE DOCUMENT that BellSouth wrote--
+THOUSANDS of documents. "We do not publish anything other
+than for our own company," Ms. Williams explained.
+"Any company document of this nature is considered proprietary."
+Nobody was in charge of singling out special high-security publications
+for special high-security protection. They were ALL special,
+no matter how trivial, no matter what their subject matter--
+the stamp was put on as soon as any document was written,
+and the stamp was never removed.
+
+Zenner now asked whether the charts she had been using to explain
+the mechanics of E911 system were "proprietary," too.
+Were they PUBLIC INFORMATION, these charts, all about PSAPs,
+ALIs, nodes, local end switches? Could he take the charts out
+in the street and show them to anybody, "without violating
+some proprietary notion that BellSouth has?"
+
+Ms Williams showed some confusion, but finally areed that the charts were,
+in fact, public.
+
+"But isn't this what you said was basically what appeared in Phrack?"
+
+Ms. Williams denied this.
+
+Zenner now pointed out that the E911 Document as published in Phrack
+was only half the size of the original E911 Document (as Prophet
+had purloined it). Half of it had been deleted--edited by Neidorf.
+
+Ms. Williams countered that "Most of the information that is
+in the text file is redundant."
+
+Zenner continued to probe. Exactly what bits of knowledge in the Document
+were, in fact, unknown to the public? Locations of E911 computers?
+Phone numbers for telco personnel? Ongoing maintenance subcommittees?
+Hadn't Neidorf removed much of this?
+
+Then he pounced. "Are you familiar with Bellcore Technical Reference
+Document TR-TSY-000350?" It was, Zenner explained, officially titled
+"E911 Public Safety Answering Point Interface Between 1-1AESS Switch
+and Customer Premises Equipment." It contained highly detailed
+and specific technical information about the E911 System.
+It was published by Bellcore and publicly available for about $20.
+
+He showed the witness a Bellcore catalog which listed thousands
+of documents from Bellcore and from all the Baby Bells, BellSouth included.
+The catalog, Zenner pointed out, was free. Anyone with a credit card
+could call the Bellcore toll-free 800 number and simply order any
+of these documents, which would be shipped to any customer without question.
+Including, for instance, "BellSouth E911 Service Interfaces to
+Customer Premises Equipment at a Public Safety Answering Point."
+
+Zenner gave the witness a copy of "BellSouth E911 Service Interfaces,"
+which cost, as he pointed out, $13, straight from the catalog.
+"Look at it carefully," he urged Ms. Williams, "and tell me
+if it doesn't contain about twice as much detailed information
+about the E911 system of BellSouth than appeared anywhere in Phrack."
+
+"You want me to. . . ." Ms. Williams trailed off. "I don't understand."
+
+"Take a careful look," Zenner persisted. "Take a look at that document,
+and tell me when you're done looking at it if, indeed, it doesn't contain
+much more detailed information about the E911 system than appeared in Phrack."
+
+"Phrack wasn't taken from this," Ms. Williams said.
+
+"Excuse me?" said Zenner.
+
+"Phrack wasn't taken from this."
+
+"I can't hear you," Zenner said.
+
+"Phrack was not taken from this document. I don't understand
+your question to me."
+
+"I guess you don't," Zenner said.
+
+At this point, the prosecution's case had been gutshot.
+Ms. Williams was distressed. Her confusion was quite genuine.
+Phrack had not been taken from any publicly available Bellcore document.
+Phrack's E911 Document had been stolen from her own company's computers,
+from her own company's text files, that her own colleagues had written,
+and revised, with much labor.
+
+But the "value" of the Document had been blown to smithereens.
+It wasn't worth eighty grand. According to Bellcore it was worth
+thirteen bucks. And the looming menace that it supposedly posed
+had been reduced in instants to a scarecrow. Bellcore itself
+was selling material far more detailed and "dangerous,"
+to anybody with a credit card and a phone.
+
+Actually, Bellcore was not giving this information to just anybody.
+They gave it to ANYBODY WHO ASKED, but not many did ask.
+Not many people knew that Bellcore had a free catalog and an 800 number.
+John Nagle knew, but certainly the average teenage phreak didn't know.
+"Tuc," a friend of Neidorf's and sometime Phrack contributor, knew,
+and Tuc had been very helpful to the defense, behind the scenes.
+But the Legion of Doom didn't know--otherwise, they would never
+have wasted so much time raiding dumpsters. Cook didn't know.
+Foley didn't know. Kluepfel didn't know. The right hand
+of Bellcore knew not what the left hand was doing. The right
+hand was battering hackers without mercy, while the left hand
+was distributing Bellcore's intellectual property to anybody
+who was interested in telephone technical trivia--apparently,
+a pathetic few.
+
+The digital underground was so amateurish and poorly organized
+that they had never discovered this heap of unguarded riches.
+The ivory tower of the telcos was so wrapped-up in the fog
+of its own technical obscurity that it had left all the
+windows open and flung open the doors. No one had even noticed.
+
+Zenner sank another nail in the coffin. He produced a printed issue
+of Telephone Engineer & Management, a prominent industry journal
+that comes out twice a month and costs $27 a year. This particular issue
+of TE&M, called "Update on 911," featured a galaxy of technical details
+on 911 service and a glossary far more extensive than Phrack's.
+
+The trial rumbled on, somehow, through its own momentum.
+Tim Foley testified about his interrogations of Neidorf.
+Neidorf's written admission that he had known the E911 Document
+was pilfered was officially read into the court record.
+
+An interesting side issue came up: "Terminus" had once passed Neidorf
+a piece of UNIX AT&T software, a log-in sequence, that had been cunningly
+altered so that it could trap passwords. The UNIX software itself was
+illegally copied AT&T property, and the alterations "Terminus" had made to it,
+had transformed it into a device for facilitating computer break-ins. Terminus
+himself would eventually plead guilty to theft of this piece of software,
+and the Chicago group would send Terminus to prison for it. But it was
+of dubious relevance in the Neidorf case. Neidorf hadn't written the program.
+He wasn't accused of ever having used it. And Neidorf wasn't being charged
+with software theft or owning a password trapper.
+
+On the next day, Zenner took the offensive. The civil libertarians
+now had their own arcane, untried legal weaponry to launch into action--
+the Electronic Communications Privacy Act of 1986, 18 US Code,
+Section 2701 et seq. Section 2701 makes it a crime to intentionally
+access without authorization a facility in which an electronic communication
+service is provided--it is, at heart, an anti-bugging and anti-tapping law,
+intended to carry the traditional protections of telephones into other
+electronic channels of communication. While providing penalties for amateur
+snoops, however, Section 2703 of the ECPA also lays some formal difficulties
+on the bugging and tapping activities of police.
+
+The Secret Service, in the person of Tim Foley, had served Richard Andrews
+with a federal grand jury subpoena, in their pursuit of Prophet,
+the E911 Document, and the Terminus software ring. But according to
+the Electronic Communications Privacy Act, a "provider of remote
+computing service" was legally entitled to "prior notice" from
+the government if a subpoena was used. Richard Andrews and his
+basement UNIX node, Jolnet, had not received any "prior notice."
+Tim Foley had purportedly violated the ECPA and committed
+an electronic crime! Zenner now sought the judge's permission
+to cross-examine Foley on the topic of Foley's own electronic misdeeds.
+
+Cook argued that Richard Andrews' Jolnet was a privately owned
+bulletin board, and not within the purview of ECPA. Judge Bua
+granted the motion of the government to prevent cross-examination
+on that point, and Zenner's offensive fizzled. This, however,
+was the first direct assault on the legality of the actions
+of the Computer Fraud and Abuse Task Force itself--
+the first suggestion that they themselves had broken the law,
+and might, perhaps, be called to account.
+
+Zenner, in any case, did not really need the ECPA.
+Instead, he grilled Foley on the glaring contradictions in
+the supposed value of the E911 Document. He also brought up
+the embarrassing fact that the supposedly red-hot E911 Document
+had been sitting around for months, in Jolnet, with Kluepfel's knowledge,
+while Kluepfel had done nothing about it.
+
+In the afternoon, the Prophet was brought in to testify
+for the prosecution. (The Prophet, it will be recalled,
+had also been indicted in the case as partner in a fraud
+scheme with Neidorf.) In Atlanta, the Prophet had already
+pled guilty to one charge of conspiracy, one charge of wire fraud
+and one charge of interstate transportation of stolen property.
+The wire fraud charge, and the stolen property charge,
+were both directly based on the E911 Document.
+
+The twenty-year-old Prophet proved a sorry customer,
+answering questions politely but in a barely audible mumble,
+his voice trailing off at the ends of sentences.
+He was constantly urged to speak up.
+
+Cook, examining Prophet, forced him to admit that
+he had once had a "drug problem," abusing amphetamines,
+marijuana, cocaine, and LSD. This may have established
+to the jury that "hackers" are, or can be, seedy lowlife characters,
+but it may have damaged Prophet's credibility somewhat.
+Zenner later suggested that drugs might have damaged Prophet's memory.
+The interesting fact also surfaced that Prophet had never
+physically met Craig Neidorf. He didn't even know
+Neidorf's last name--at least, not until the trial.
+
+Prophet confirmed the basic facts of his hacker career.
+He was a member of the Legion of Doom. He had abused codes,
+he had broken into switching stations and re-routed calls,
+he had hung out on pirate bulletin boards. He had raided
+the BellSouth AIMSX computer, copied the E911 Document,
+stored it on Jolnet, mailed it to Neidorf. He and Neidorf
+had edited it, and Neidorf had known where it came from.
+
+Zenner, however, had Prophet confirm that Neidorf was not a member
+of the Legion of Doom, and had not urged Prophet to break into
+BellSouth computers. Neidorf had never urged Prophet to defraud anyone,
+or to steal anything. Prophet also admitted that he had never known Neidorf
+to break in to any computer. Prophet said that no one in the Legion of Doom
+considered Craig Neidorf a "hacker" at all. Neidorf was not a UNIX maven,
+and simply lacked the necessary skill and ability to break into computers.
+Neidorf just published a magazine.
+
+On Friday, July 27, 1990, the case against Neidorf collapsed.
+Cook moved to dismiss the indictment, citing "information currently
+available to us that was not available to us at the inception of the trial."
+Judge Bua praised the prosecution for this action, which he described as
+"very responsible," then dismissed a juror and declared a mistrial.
+
+Neidorf was a free man. His defense, however, had cost himself
+and his family dearly. Months of his life had been consumed in anguish;
+he had seen his closest friends shun him as a federal criminal.
+He owed his lawyers over a hundred thousand dollars, despite
+a generous payment to the defense by Mitch Kapor.
+
+Neidorf was not found innocent. The trial was simply dropped.
+Nevertheless, on September 9, 1991, Judge Bua granted Neidorf's
+motion for the "expungement and sealing" of his indictment record.
+The United States Secret Service was ordered to delete and destroy
+all fingerprints, photographs, and other records of arrest
+or processing relating to Neidorf's indictment, including
+their paper documents and their computer records.
+
+Neidorf went back to school, blazingly determined to become a lawyer.
+Having seen the justice system at work, Neidorf lost much of his enthusiasm
+for merely technical power. At this writing, Craig Neidorf is working
+in Washington as a salaried researcher for the American Civil Liberties Union.
+
+#
+
+The outcome of the Neidorf trial changed the EFF
+from voices-in-the-wilderness to the media darlings
+of the new frontier.
+
+Legally speaking, the Neidorf case was not a sweeping triumph
+for anyone concerned. No constitutional principles had been established.
+The issues of "freedom of the press" for electronic publishers remained
+in legal limbo. There were public misconceptions about the case.
+Many people thought Neidorf had been found innocent and relieved
+of all his legal debts by Kapor. The truth was that the government
+had simply dropped the case, and Neidorf's family had gone deeply
+into hock to support him.
+
+But the Neidorf case did provide a single, devastating, public sound-bite:
+THE FEDS SAID IT WAS WORTH EIGHTY GRAND, AND IT WAS ONLY WORTH THIRTEEN BUCKS.
+
+This is the Neidorf case's single most memorable element. No serious report
+of the case missed this particular element. Even cops could not read this
+without a wince and a shake of the head. It left the public credibility
+of the crackdown agents in tatters.
+
+The crackdown, in fact, continued, however. Those two charges
+against Prophet, which had been based on the E911 Document,
+were quietly forgotten at his sentencing--even though Prophet
+had already pled guilty to them. Georgia federal prosecutors
+strongly argued for jail time for the Atlanta Three, insisting on
+"the need to send a message to the community," "the message that
+hackers around the country need to hear."
+
+There was a great deal in their sentencing memorandum
+about the awful things that various other hackers had done
+(though the Atlanta Three themselves had not, in fact,
+actually committed these crimes). There was also much
+speculation about the awful things that the Atlanta Three
+MIGHT have done and WERE CAPABLE of doing (even though
+they had not, in fact, actually done them).
+The prosecution's argument carried the day.
+The Atlanta Three were sent to prison:
+Urvile and Leftist both got 14 months each,
+while Prophet (a second offender) got 21 months.
+
+The Atlanta Three were also assessed staggering fines as "restitution":
+$233,000 each. BellSouth claimed that the defendants had "stolen"
+"approximately $233,880 worth" of "proprietary computer access information"--
+specifically, $233,880 worth of computer passwords and connect addresses.
+BellSouth's astonishing claim of the extreme value of its own computer
+passwords and addresses was accepted at face value by the Georgia court.
+Furthermore (as if to emphasize its theoretical nature) this enormous sum
+was not divvied up among the Atlanta Three, but each of them had to pay
+all of it.
+
+A striking aspect of the sentence was that the Atlanta Three were
+specifically forbidden to use computers, except for work or under supervision.
+Depriving hackers of home computers and modems makes some sense if one
+considers hackers as "computer addicts," but EFF, filing an amicus brief
+in the case, protested that this punishment was unconstitutional--
+it deprived the Atlanta Three of their rights of free association
+and free expression through electronic media.
+
+Terminus, the "ultimate hacker," was finally sent to prison for a year
+through the dogged efforts of the Chicago Task Force. His crime,
+to which he pled guilty, was the transfer of the UNIX password trapper,
+which was officially valued by AT&T at $77,000, a figure which aroused
+intense skepticism among those familiar with UNIX "login.c" programs.
+
+The jailing of Terminus and the Atlanta Legionnaires of Doom, however,
+did not cause the EFF any sense of embarrassment or defeat.
+On the contrary, the civil libertarians were rapidly gathering strength.
+
+An early and potent supporter was Senator Patrick Leahy,
+Democrat from Vermont, who had been a Senate sponsor
+of the Electronic Communications Privacy Act. Even before
+the Neidorf trial, Leahy had spoken out in defense of hacker-power
+and freedom of the keyboard: "We cannot unduly inhibit the inquisitive
+13-year-old who, if left to experiment today, may tomorrow develop
+the telecommunications or computer technology to lead the United States
+into the 21st century. He represents our future and our best hope
+to remain a technologically competitive nation."
+
+It was a handsome statement, rendered perhaps rather more effective
+by the fact that the crackdown raiders DID NOT HAVE any Senators
+speaking out for THEM. On the contrary, their highly secretive
+actions and tactics, all "sealed search warrants" here and
+"confidential ongoing investigations" there, might have won
+them a burst of glamorous publicity at first, but were crippling
+them in the on-going propaganda war. Gail Thackeray was reduced
+to unsupported bluster: "Some of these people who are loudest
+on the bandwagon may just slink into the background,"
+she predicted in Newsweek--when all the facts came out,
+and the cops were vindicated.
+
+But all the facts did not come out. Those facts that did,
+were not very flattering. And the cops were not vindicated.
+And Gail Thackeray lost her job. By the end of 1991,
+William Cook had also left public employment.
+
+1990 had belonged to the crackdown, but by '91 its agents
+were in severe disarray, and the libertarians were on a roll.
+People were flocking to the cause.
+
+A particularly interesting ally had been Mike Godwin of Austin, Texas.
+Godwin was an individual almost as difficult to describe as Barlow;
+he had been editor of the student newspaper of the University of Texas,
+and a computer salesman, and a programmer, and in 1990 was back
+in law school, looking for a law degree.
+
+Godwin was also a bulletin board maven. He was very well-known
+in the Austin board community under his handle "Johnny Mnemonic,"
+which he adopted from a cyberpunk science fiction story by William Gibson.
+Godwin was an ardent cyberpunk science fiction fan. As a fellow Austinite
+of similar age and similar interests, I myself had known Godwin socially
+for many years. When William Gibson and myself had been writing our
+collaborative SF novel, The Difference Engine, Godwin had been our
+technical advisor in our effort to link our Apple word-processors
+from Austin to Vancouver. Gibson and I were so pleased by his generous
+expert help that we named a character in the novel "Michael Godwin"
+in his honor.
+
+The handle "Mnemonic" suited Godwin very well. His erudition
+and his mastery of trivia were impressive to the point of stupor;
+his ardent curiosity seemed insatiable, and his desire to debate
+and argue seemed the central drive of his life. Godwin had even
+started his own Austin debating society, wryly known as the
+"Dull Men's Club." In person, Godwin could be overwhelming;
+a flypaper-brained polymath who could not seem to let any idea go.
+On bulletin boards, however, Godwin's closely reasoned,
+highly grammatical, erudite posts suited the medium well,
+and he became a local board celebrity.
+
+Mike Godwin was the man most responsible for the public national exposure
+of the Steve Jackson case. The Izenberg seizure in Austin had received
+no press coverage at all. The March 1 raids on Mentor, Bloodaxe, and
+Steve Jackson Games had received a brief front-page splash in the
+front page of the Austin American-Statesman, but it was confused
+and ill-informed: the warrants were sealed, and the Secret Service
+wasn't talking. Steve Jackson seemed doomed to obscurity.
+Jackson had not been arrested; he was not charged with any crime;
+he was not on trial. He had lost some computers in an ongoing
+investigation--so what? Jackson tried hard to attract attention
+to the true extent of his plight, but he was drawing a blank;
+no one in a position to help him seemed able to get a mental grip
+on the issues.
+
+Godwin, however, was uniquely, almost magically, qualified
+to carry Jackson's case to the outside world. Godwin was
+a board enthusiast, a science fiction fan, a former journalist,
+a computer salesman, a lawyer-to-be, and an Austinite.
+Through a coincidence yet more amazing, in his last year
+of law school Godwin had specialized in federal prosecutions
+and criminal procedure. Acting entirely on his own, Godwin made
+up a press packet which summarized the issues and provided useful
+contacts for reporters. Godwin's behind-the-scenes effort
+(which he carried out mostly to prove a point in a local board debate)
+broke the story again in the Austin American-Statesman and then in Newsweek.
+
+Life was never the same for Mike Godwin after that. As he joined the growing
+civil liberties debate on the Internet, it was obvious to all parties involved
+that here was one guy who, in the midst of complete murk and confusion,
+GENUINELY UNDERSTOOD EVERYTHING HE WAS TALKING ABOUT. The disparate elements
+of Godwin's dilettantish existence suddenly fell together as neatly as
+the facets of a Rubik's cube.
+
+When the time came to hire a full-time EFF staff attorney,
+Godwin was the obvious choice. He took the Texas bar exam,
+left Austin, moved to Cambridge, became a full-time, professional,
+computer civil libertarian, and was soon touring the nation on behalf
+of EFF, delivering well-received addresses on the issues to crowds
+as disparate as academics, industrialists, science fiction fans,
+and federal cops.
+
+Michael Godwin is currently the chief legal counsel of
+the Electronic Frontier Foundation in Cambridge, Massachusetts.
+
+#
+
+Another early and influential participant in the controversy
+was Dorothy Denning. Dr. Denning was unique among investigators
+of the computer underground in that she did not enter the debate
+with any set of politicized motives. She was a professional
+cryptographer and computer security expert whose primary interest
+in hackers was SCHOLARLY. She had a B.A. and M.A. in mathematics,
+and a Ph.D. in computer science from Purdue. She had worked for SRI
+International, the California think-tank that was also the home of
+computer-security maven Donn Parker, and had authored an influential text
+called Cryptography and Data Security. In 1990, Dr. Denning was working for
+Digital Equipment Corporation in their Systems Reseach Center. Her husband,
+Peter Denning, was also a computer security expert, working for NASA's
+Research Institute for Advanced Computer Science. He had edited the
+well-received Computers Under Attack: Intruders, Worms and Viruses.
+
+Dr. Denning took it upon herself to contact the digital underground,
+more or less with an anthropological interest. There she discovered
+that these computer-intruding hackers, who had been characterized
+as unethical, irresponsible, and a serious danger to society,
+did in fact have their own subculture and their own rules.
+They were not particularly well-considered rules, but they were,
+in fact, rules. Basically, they didn't take money and they
+didn't break anything.
+
+Her dispassionate reports on her researches did a great deal
+to influence serious-minded computer professionals--the sort
+of people who merely rolled their eyes at the cyberspace
+rhapsodies of a John Perry Barlow.
+
+For young hackers of the digital underground, meeting Dorothy Denning
+was a genuinely mind-boggling experience. Here was this neatly coiffed,
+conservatively dressed, dainty little personage, who reminded most
+hackers of their moms or their aunts. And yet she was an IBM systems
+programmer with profound expertise in computer architectures
+and high-security information flow, who had personal friends
+in the FBI and the National Security Agency.
+
+Dorothy Denning was a shining example of the American mathematical
+intelligentsia, a genuinely brilliant person from the central ranks
+of the computer-science elite. And here she was, gently questioning
+twenty-year-old hairy-eyed phone-phreaks over the deeper ethical
+implications of their behavior.
+
+Confronted by this genuinely nice lady, most hackers sat up very straight
+and did their best to keep the anarchy-file stuff down to a faint whiff
+of brimstone. Nevertheless, the hackers WERE in fact prepared to seriously
+discuss serious issues with Dorothy Denning. They were willing to speak
+the unspeakable and defend the indefensible, to blurt out their convictions
+that information cannot be owned, that the databases of governments and large
+corporations were a threat to the rights and privacy of individuals.
+
+Denning's articles made it clear to many that "hacking"
+was not simple vandalism by some evil clique of psychotics.
+"Hacking" was not an aberrant menace that could be charmed away
+by ignoring it, or swept out of existence by jailing a few ringleaders.
+Instead, "hacking" was symptomatic of a growing, primal struggle over
+knowledge and power in the age of information.
+
+Denning pointed out that the attitude of hackers were at least partially
+shared by forward-looking management theorists in the business community:
+people like Peter Drucker and Tom Peters. Peter Drucker, in his book
+The New Realities, had stated that "control of information by the government
+is no longer possible. Indeed, information is now transnational.
+Like money, it has no `fatherland.'"
+
+And management maven Tom Peters had chided large corporations for uptight,
+proprietary attitudes in his bestseller, Thriving on Chaos:
+"Information hoarding, especially by politically motivated,
+power-seeking staffs, had been commonplace throughout American industry,
+service and manufacturing alike. It will be an impossible
+millstone aroung the neck of tomorrow's organizations."
+
+Dorothy Denning had shattered the social membrane of the
+digital underground. She attended the Neidorf trial,
+where she was prepared to testify for the defense as an expert witness.
+She was a behind-the-scenes organizer of two of the most important
+national meetings of the computer civil libertarians. Though not
+a zealot of any description, she brought disparate elements of the
+electronic community into a surprising and fruitful collusion.
+
+Dorothy Denning is currently the Chair of the Computer Science Department
+at Georgetown University in Washington, DC.
+
+#
+
+There were many stellar figures in the civil libertarian community.
+There's no question, however, that its single most influential figure
+was Mitchell D. Kapor. Other people might have formal titles,
+or governmental positions, have more experience with crime,
+or with the law, or with the arcanities of computer security
+or constitutional theory. But by 1991 Kapor had transcended
+any such narrow role. Kapor had become "Mitch."
+
+Mitch had become the central civil-libertarian ad-hocrat.
+Mitch had stood up first, he had spoken out loudly, directly,
+vigorously and angrily, he had put his own reputation,
+and his very considerable personal fortune, on the line.
+By mid-'91 Kapor was the best-known advocate of his cause
+and was known PERSONALLY by almost every single human being in America
+with any direct influence on the question of civil liberties in cyberspace.
+Mitch had built bridges, crossed voids, changed paradigms, forged metaphors,
+made phone-calls and swapped business cards to such spectacular effect
+that it had become impossible for anyone to take any action in the
+"hacker question" without wondering what Mitch might think--
+and say--and tell his friends.
+
+The EFF had simply NETWORKED the situation into an entirely new status quo.
+And in fact this had been EFF's deliberate strategy from the beginning.
+Both Barlow and Kapor loathed bureaucracies and had deliberately
+chosen to work almost entirely through the electronic spiderweb of
+"valuable personal contacts."
+
+After a year of EFF, both Barlow and Kapor had every reason
+to look back with satisfaction. EFF had established its own Internet node,
+"eff.org," with a well-stocked electronic archive of documents on
+electronic civil rights, privacy issues, and academic freedom.
+EFF was also publishing EFFector, a quarterly printed journal,
+as well as EFFector Online, an electronic newsletter with
+over 1,200 subscribers. And EFF was thriving on the Well.
+
+EFF had a national headquarters in Cambridge and a full-time staff.
+It had become a membership organization and was attracting
+grass-roots support. It had also attracted the support
+of some thirty civil-rights lawyers, ready and eager
+to do pro bono work in defense of the Constitution in Cyberspace.
+
+EFF had lobbied successfully in Washington and in Massachusetts
+to change state and federal legislation on computer networking.
+Kapor in particular had become a veteran expert witness,
+and had joined the Computer Science and Telecommunications Board
+of the National Academy of Science and Engineering.
+
+EFF had sponsored meetings such as "Computers, Freedom and Privacy"
+and the CPSR Roundtable. It had carried out a press offensive that,
+in the words of EFFector, "has affected the climate of opinion about
+computer networking and begun to reverse the slide into
+`hacker hysteria' that was beginning to grip the nation."
+
+It had helped Craig Neidorf avoid prison.
+
+And, last but certainly not least, the Electronic Frontier Foundation
+had filed a federal lawsuit in the name of Steve Jackson,
+Steve Jackson Games Inc., and three users of the Illuminati
+bulletin board system. The defendants were, and are,
+the United States Secret Service, William Cook, Tim Foley,
+Barbara Golden and Henry Kleupfel.
+
+The case, which is in pre-trial procedures in an Austin federal court
+as of this writing, is a civil action for damages to redress
+alleged violations of the First and Fourth Amendments to the
+United States Constitution, as well as the Privacy Protection Act
+of 1980 (42 USC 2000aa et seq.), and the Electronic Communications
+Privacy Act (18 USC 2510 et seq and 2701 et seq).
+
+EFF had established that it had credibility. It had also established
+that it had teeth.
+
+In the fall of 1991 I travelled to Massachusetts to speak personally
+with Mitch Kapor. It was my final interview for this book.
+
+#
+
+The city of Boston has always been one of the major intellectual centers
+of the American republic. It is a very old city by American standards,
+a place of skyscrapers overshadowing seventeenth-century graveyards,
+where the high-tech start-up companies of Route 128 co-exist with the
+hand-wrought pre-industrial grace of "Old Ironsides," the USS CONSTITUTION.
+
+The Battle of Bunker Hill, one of the first and bitterest armed clashes
+of the American Revolution, was fought in Boston's environs. Today there is
+a monumental spire on Bunker Hill, visible throughout much of the city.
+The willingness of the republican revolutionaries to take up arms and fire
+on their oppressors has left a cultural legacy that two full centuries
+have not effaced. Bunker Hill is still a potent center of American political
+symbolism, and the Spirit of '76 is still a potent image for those who seek
+to mold public opinion.
+
+Of course, not everyone who wraps himself in the flag is necessarily
+a patriot. When I visited the spire in September 1991, it bore a huge,
+badly-erased, spray-can grafitto around its bottom reading
+"BRITS OUT--IRA PROVOS." Inside this hallowed edifice was
+a glass-cased diorama of thousands of tiny toy soldiers,
+rebels and redcoats, fighting and dying over the green hill,
+the riverside marshes, the rebel trenchworks. Plaques indicated the
+movement of troops, the shiftings of strategy. The Bunker Hill Monument
+is occupied at its very center by the toy soldiers of a military
+war-game simulation.
+
+The Boston metroplex is a place of great universities,
+prominent among the Massachusetts Institute of Technology,
+where the term "computer hacker" was first coined. The Hacker Crackdown
+of 1990 might be interpreted as a political struggle among American cities:
+traditional strongholds of longhair intellectual liberalism,
+such as Boston, San Francisco, and Austin, versus the bare-knuckle
+industrial pragmatism of Chicago and Phoenix (with Atlanta and New York
+wrapped in internal struggle).
+
+The headquarters of the Electronic Frontier Foundation is on
+155 Second Street in Cambridge, a Bostonian suburb north
+of the River Charles. Second Street has weedy sidewalks of dented,
+sagging brick and elderly cracked asphalt; large street-signs warn
+"NO PARKING DURING DECLARED SNOW EMERGENCY." This is an old area
+of modest manufacturing industries; the EFF is catecorner from the
+Greene Rubber Company. EFF's building is two stories of red brick;
+its large wooden windows feature gracefully arched tops and stone sills.
+
+The glass window beside the Second Street entrance bears three sheets
+of neatly laser-printed paper, taped against the glass. They read:
+ON Technology. EFF. KEI.
+
+"ON Technology" is Kapor's software company, which currently specializes
+in "groupware" for the Apple Macintosh computer. "Groupware" is intended
+to promote efficient social interaction among office-workers linked
+by computers. ON Technology's most successful software products to date
+are "Meeting Maker" and "Instant Update."
+
+"KEI" is Kapor Enterprises Inc., Kapor's personal holding company,
+the commercial entity that formally controls his extensive investments
+in other hardware and software corporations.
+
+"EFF" is a political action group--of a special sort.
+
+Inside, someone's bike has been chained to the handrails
+of a modest flight of stairs. A wall of modish glass brick
+separates this anteroom from the offices. Beyond the brick,
+there's an alarm system mounted on the wall, a sleek, complex little
+number that resembles a cross between a thermostat and a CD player.
+Piled against the wall are box after box of a recent special issue
+of Scientific American, "How to Work, Play, and Thrive in Cyberspace,"
+with extensive coverage of electronic networking techniques
+and political issues, including an article by Kapor himself.
+These boxes are addressed to Gerard Van der Leun, EFF's
+Director of Communications, who will shortly mail those magazines
+to every member of the EFF.
+
+The joint headquarters of EFF, KEI, and ON Technology,
+which Kapor currently rents, is a modestly bustling place.
+It's very much the same physical size as Steve Jackson's gaming company.
+It's certainly a far cry from the gigantic gray steel-sided railway
+shipping barn, on the Monsignor O'Brien Highway, that is owned
+by Lotus Development Corporation.
+
+Lotus is, of course, the software giant that Mitchell Kapor founded
+in the late 70s. The software program Kapor co-authored,
+"Lotus 1-2-3," is still that company's most profitable product.
+"Lotus 1-2-3" also bears a singular distinction in the
+digital underground: it's probably the most pirated piece
+of application software in world history.
+
+Kapor greets me cordially in his own office, down a hall.
+Kapor, whose name is pronounced KAY-por, is in his early forties,
+married and the father of two. He has a round face, high forehead,
+straight nose, a slightly tousled mop of black hair peppered with gray.
+His large brown eyes are wideset, reflective, one might almost say soulful.
+He disdains ties, and commonly wears Hawaiian shirts and tropical prints,
+not so much garish as simply cheerful and just that little bit anomalous.
+
+There is just the whiff of hacker brimstone about Mitch Kapor.
+He may not have the hard-riding, hell-for-leather, guitar-strumming
+charisma of his Wyoming colleague John Perry Barlow, but there's
+something about the guy that still stops one short. He has the air
+of the Eastern city dude in the bowler hat, the dreamy,
+Longfellow-quoting poker shark who only HAPPENS to know
+the exact mathematical odds against drawing to an inside straight.
+Even among his computer-community colleagues, who are hardly known
+for mental sluggishness, Kapor strikes one forcefully as a very
+intelligent man. He speaks rapidly, with vigorous gestures,
+his Boston accent sometimes slipping to the sharp nasal tang
+of his youth in Long Island.
+
+Kapor, whose Kapor Family Foundation does much of his philanthropic work,
+is a strong supporter of Boston's Computer Museum. Kapor's interest
+in the history of his industry has brought him some remarkable curios,
+such as the "byte" just outside his office door. This "byte"--
+eight digital bits--has been salvaged from the wreck of an
+electronic computer of the pre-transistor age. It's a standing gunmetal
+rack about the size of a small toaster-oven: with eight slots
+of hand-soldered breadboarding featuring thumb-sized vacuum tubes.
+If it fell off a table it could easily break your foot,
+but it was state-of-the-art computation in the 1940s.
+(It would take exactly 157,184 of these primordial toasters
+to hold the first part of this book.)
+
+There's also a coiling, multicolored, scaly dragon that some
+inspired techno-punk artist has cobbled up entirely out of transistors,
+capacitors, and brightly plastic-coated wiring.
+
+Inside the office, Kapor excuses himself briefly to do a little
+mouse-whizzing housekeeping on his personal Macintosh IIfx.
+If its giant screen were an open window, an agile person
+could climb through it without much trouble at all.
+There's a coffee-cup at Kapor's elbow, a memento of his
+recent trip to Eastern Europe, which has a black-and-white
+stencilled photo and the legend CAPITALIST FOOLS TOUR.
+It's Kapor, Barlow, and two California venture-capitalist luminaries
+of their acquaintance, four windblown, grinning Baby Boomer
+dudes in leather jackets, boots, denim, travel bags,
+standing on airport tarmac somewhere behind the formerly Iron Curtain.
+They look as if they're having the absolute time of their lives.
+
+Kapor is in a reminiscent mood. We talk a bit about his youth--
+high school days as a "math nerd," Saturdays attending Columbia University's
+high-school science honors program, where he had his first experience
+programming computers. IBM 1620s, in 1965 and '66. "I was very interested,"
+says Kapor, "and then I went off to college and got distracted by drugs sex
+and rock and roll, like anybody with half a brain would have then!"
+After college he was a progressive-rock DJ in Hartford, Connecticut,
+for a couple of years.
+
+I ask him if he ever misses his rock and roll days--if he ever wished
+he could go back to radio work.
+
+He shakes his head flatly. "I stopped thinking about going back
+to be a DJ the day after Altamont."
+
+Kapor moved to Boston in 1974 and got a job programming mainframes in COBOL.
+He hated it. He quit and became a teacher of transcendental meditation.
+(It was Kapor's long flirtation with Eastern mysticism that gave the
+world "Lotus.")
+
+In 1976 Kapor went to Switzerland, where the Transcendental Meditation
+movement had rented a gigantic Victorian hotel in St-Moritz. It was
+an all-male group--a hundred and twenty of them--determined upon
+Enlightenment or Bust. Kapor had given the transcendant his best shot.
+He was becoming disenchanted by "the nuttiness in the organization."
+"They were teaching people to levitate," he says, staring at the floor.
+His voice drops an octave, becomes flat. "THEY DON'T LEVITATE."
+
+Kapor chose Bust. He went back to the States and acquired a degree
+in counselling psychology. He worked a while in a hospital,
+couldn't stand that either. "My rep was," he says "a very bright kid
+with a lot of potential who hasn't found himself. Almost thirty.
+Sort of lost."
+
+Kapor was unemployed when he bought his first personal computer--an Apple II.
+He sold his stereo to raise cash and drove to New Hampshire to avoid the
+sales tax.
+
+"The day after I purchased it," Kapor tells me, "I was hanging out
+in a computer store and I saw another guy, a man in his forties,
+well-dressed guy, and eavesdropped on his conversation with the salesman.
+He didn't know anything about computers. I'd had a year programming.
+And I could program in BASIC. I'd taught myself. So I went up to him,
+and I actually sold myself to him as a consultant." He pauses.
+"I don't know where I got the nerve to do this. It was uncharacteristic.
+I just said, `I think I can help you, I've been listening,
+this is what you need to do and I think I can do it for you.'
+And he took me on! He was my first client! I became a computer
+consultant the first day after I bought the Apple II."
+
+Kapor had found his true vocation. He attracted more clients
+for his consultant service, and started an Apple users' group.
+
+A friend of Kapor's, Eric Rosenfeld, a graduate student at MIT,
+had a problem. He was doing a thesis on an arcane form of
+financial statistics, but could not wedge himself into the crowded queue
+for time on MIT's mainframes. (One might note at this point that if
+Mr. Rosenfeld had dishonestly broken into the MIT mainframes,
+Kapor himself might have never invented Lotus 1-2-3 and
+the PC business might have been set back for years!)
+Eric Rosenfeld did have an Apple II, however,
+and he thought it might be possible to scale the problem down.
+Kapor, as favor, wrote a program for him in BASIC that did the job.
+
+It then occurred to the two of them, out of the blue,
+that it might be possible to SELL this program.
+They marketed it themselves, in plastic baggies,
+for about a hundred bucks a pop, mail order.
+"This was a total cottage industry by a marginal consultant,"
+Kapor says proudly. "That's how I got started, honest to God."
+
+Rosenfeld, who later became a very prominent figure on Wall Street,
+urged Kapor to go to MIT's business school for an MBA.
+Kapor did seven months there, but never got his MBA.
+He picked up some useful tools--mainly a firm grasp
+of the principles of accounting--and, in his own words,
+"learned to talk MBA." Then he dropped out and went to Silicon Valley.
+
+The inventors of VisiCalc, the Apple computer's premier business program,
+had shown an interest in Mitch Kapor. Kapor worked diligently for them
+for six months, got tired of California, and went back to Boston
+where they had better bookstores. The VisiCalc group had made
+the critical error of bringing in "professional management."
+"That drove them into the ground," Kapor says.
+
+"Yeah, you don't hear a lot about VisiCalc these days," I muse.
+
+Kapor looks surprised. "Well, Lotus. . . we BOUGHT it."
+
+"Oh. You BOUGHT it?"
+
+"Yeah."
+
+"Sort of like the Bell System buying Western Union?"
+
+Kapor grins. "Yep! Yep! Yeah, exactly!"
+
+Mitch Kapor was not in full command of the destiny of himself
+or his industry. The hottest software commodities of the early 1980s
+were COMPUTER GAMES--the Atari seemed destined to enter every teenage home
+in America. Kapor got into business software simply because he didn't have
+any particular feeling for computer games. But he was supremely fast
+on his feet, open to new ideas and inclined to trust his instincts.
+And his instincts were good. He chose good people to deal with--
+gifted programmer Jonathan Sachs (the co-author of Lotus 1-2-3).
+Financial wizard Eric Rosenfeld, canny Wall Street analyst
+and venture capitalist Ben Rosen. Kapor was the founder and CEO of Lotus,
+one of the most spectacularly successful business ventures of the
+later twentieth century.
+
+He is now an extremely wealthy man. I ask him if he actually
+knows how much money he has.
+
+"Yeah," he says. "Within a percent or two."
+
+How much does he actually have, then?
+
+He shakes his head. "A lot. A lot. Not something I talk about.
+Issues of money and class are things that cut pretty close to the bone."
+
+I don't pry. It's beside the point. One might presume, impolitely,
+that Kapor has at least forty million--that's what he got the year
+he left Lotus. People who ought to know claim Kapor has about
+a hundred and fifty million, give or take a market swing
+in his stock holdings. If Kapor had stuck with Lotus,
+as his colleague friend and rival Bill Gates has stuck
+with his own software start-up, Microsoft, then Kapor
+would likely have much the same fortune Gates has--
+somewhere in the neighborhood of three billion,
+give or take a few hundred million. Mitch Kapor
+has all the money he wants. Money has lost whatever charm
+it ever held for him--probably not much in the first place.
+When Lotus became too uptight, too bureaucratic, too far
+from the true sources of his own satisfaction, Kapor walked.
+He simply severed all connections with the company and went out the door.
+It stunned everyone--except those who knew him best.
+
+Kapor has not had to strain his resources to wreak a thorough
+transformation in cyberspace politics. In its first year,
+EFF's budget was about a quarter of a million dollars.
+Kapor is running EFF out of his pocket change.
+
+Kapor takes pains to tell me that he does not consider himself
+a civil libertarian per se. He has spent quite some time
+with true-blue civil libertarians lately, and there's a
+political-correctness to them that bugs him. They seem
+to him to spend entirely too much time in legal nitpicking
+and not enough vigorously exercising civil rights in the
+everyday real world.
+
+Kapor is an entrepreneur. Like all hackers, he prefers his involvements
+direct, personal, and hands-on. "The fact that EFF has a node on the
+Internet is a great thing. We're a publisher. We're a distributor
+of information." Among the items the eff.org Internet node carries
+is back issues of Phrack. They had an internal debate about that in EFF,
+and finally decided to take the plunge. They might carry other
+digital underground publications--but if they do, he says,
+"we'll certainly carry Donn Parker, and anything Gail Thackeray
+wants to put up. We'll turn it into a public library, that has
+the whole spectrum of use. Evolve in the direction of people making up
+their own minds." He grins. "We'll try to label all the editorials."
+
+Kapor is determined to tackle the technicalities of the Internet
+in the service of the public interest. "The problem with being a node
+on the Net today is that you've got to have a captive technical specialist.
+We have Chris Davis around, for the care and feeding of the balky beast!
+We couldn't do it ourselves!"
+
+He pauses. "So one direction in which technology has to evolve
+is much more standardized units, that a non-technical person
+can feel comfortable with. It's the same shift as from minicomputers to PCs.
+I can see a future in which any person can have a Node on the Net.
+Any person can be a publisher. It's better than the media we now have.
+It's possible. We're working actively."
+
+Kapor is in his element now, fluent, thoroughly in command in his material.
+"You go tell a hardware Internet hacker that everyone should have a node
+on the Net," he says, "and the first thing they're going to say is,
+`IP doesn't scale!'" ("IP" is the interface protocol for the Internet.
+As it currently exists, the IP software is simply not capable of
+indefinite expansion; it will run out of usable addresses, it will saturate.)
+"The answer," Kapor says, "is: evolve the protocol! Get the smart people
+together and figure out what to do. Do we add ID? Do we add new protocol?
+Don't just say, WE CAN'T DO IT."
+
+Getting smart people together to figure out what to do is a skill
+at which Kapor clearly excels. I counter that people on the Internet
+rather enjoy their elite technical status, and don't seem particularly
+anxious to democratize the Net.
+
+Kapor agrees, with a show of scorn. "I tell them that this is the snobbery
+of the people on the Mayflower looking down their noses at the people
+who came over ON THE SECOND BOAT! Just because they got here a year,
+or five years, or ten years before everybody else, that doesn't give
+them ownership of cyberspace! By what right?"
+
+I remark that the telcos are an electronic network, too,
+and they seem to guard their specialized knowledge pretty closely.
+
+Kapor ripostes that the telcos and the Internet are entirely
+different animals. "The Internet is an open system,
+everything is published, everything gets argued about,
+basically by anybody who can get in. Mostly, it's exclusive
+and elitist just because it's so difficult. Let's make it easier to use."
+
+On the other hand, he allows with a swift change of emphasis,
+the so-called elitists do have a point as well. "Before people start coming in,
+who are new, who want to make suggestions, and criticize the Net as
+`all screwed up'. . . . They should at least take the time to understand
+the culture on its own terms. It has its own history--show some respect
+for it. I'm a conservative, to that extent."
+
+The Internet is Kapor's paradigm for the future of telecommunications.
+The Internet is decentralized, non-hierarchical, almost anarchic.
+There are no bosses, no chain of command, no secret data.
+If each node obeys the general interface standards,
+there's simply no need for any central network authority.
+
+Wouldn't that spell the doom of AT&T as an institution? I ask.
+
+That prospect doesn't faze Kapor for a moment. "Their big advantage,
+that they have now, is that they have all of the wiring.
+But two things are happening. Anyone with right-of-way
+is putting down fiber--Southern Pacific Railroad,
+people like that--there's enormous `dark fiber' laid in."
+("Dark Fiber" is fiber-optic cable, whose enormous capacity
+so exceeds the demands of current usage that much of the
+fiber still has no light-signals on it--it's still `dark,'
+awaiting future use.)
+
+"The other thing that's happening is the local-loop stuff
+is going to go wireless. Everyone from Bellcore to the cable TV
+companies to AT&T wants to put in these things called
+`personal communication systems.' So you could have local competition--
+you could have multiplicity of people, a bunch of neighborhoods,
+sticking stuff up on poles. And a bunch of other people laying in dark fiber.
+So what happens to the telephone companies? There's enormous pressure
+on them from both sides.
+
+"The more I look at this, the more I believe that in a post-industrial,
+digital world, the idea of regulated monopolies is bad. People will
+look back on it and say that in the 19th and 20th centuries
+the idea of public utilities was an okay compromise.
+You needed one set of wires in the ground. It was too economically
+inefficient, otherwise. And that meant one entity running it.
+But now, with pieces being wireless--the connections are going
+to be via high-level interfaces, not via wires. I mean, ULTIMATELY
+there are going to be wires--but the wires are just a commodity.
+Fiber, wireless. You no longer NEED a utility."
+
+Water utilities? Gas utilities?
+
+Of course we still need those, he agrees. "But when what you're moving
+is information, instead of physical substances, then you can play by
+a different set of rules. We're evolving those rules now!
+Hopefully you can have a much more decentralized system,
+and one in which there's more competition in the marketplace.
+
+"The role of government will be to make sure that nobody cheats.
+The proverbial `level playing field.' A policy that prevents monopolization.
+It should result in better service, lower prices, more choices,
+and local empowerment." He smiles. "I'm very big on local empowerment."
+
+Kapor is a man with a vision. It's a very novel vision which he
+and his allies are working out in considerable detail and with great energy.
+Dark, cynical, morbid cyberpunk that I am, I cannot avoid considering
+some of the darker implications of "decentralized, nonhierarchical,
+locally empowered" networking.
+
+I remark that some pundits have suggested that electronic networking--faxes,
+phones, small-scale photocopiers--played a strong role in dissolving
+the power of centralized communism and causing the collapse of the Warsaw Pact.
+
+Socialism is totally discredited, says Kapor, fresh back from
+the Eastern Bloc. The idea that faxes did it, all by themselves,
+is rather wishful thinking.
+
+Has it occurred to him that electronic networking might corrode
+America's industrial and political infrastructure to the point
+where the whole thing becomes untenable, unworkable--and the old order
+just collapses headlong, like in Eastern Europe?
+
+"No," Kapor says flatly. "I think that's extraordinarily unlikely.
+In part, because ten or fifteen years ago, I had similar hopes
+about personal computers--which utterly failed to materialize."
+He grins wryly, then his eyes narrow. "I'm VERY opposed to techno-utopias.
+Every time I see one, I either run away, or try to kill it."
+
+It dawns on me then that Mitch Kapor is not trying to
+make the world safe for democracy. He certainly is not
+trying to make it safe for anarchists or utopians--
+least of all for computer intruders or electronic rip-off artists.
+What he really hopes to do is make the world safe for
+future Mitch Kapors. This world of decentralized, small-scale nodes,
+with instant global access for the best and brightest,
+would be a perfect milieu for the shoestring attic capitalism
+that made Mitch Kapor what he is today.
+
+Kapor is a very bright man. He has a rare combination
+of visionary intensity with a strong practical streak.
+The Board of the EFF: John Barlow, Jerry Berman of the ACLU,
+Stewart Brand, John Gilmore, Steve Wozniak, and Esther Dyson,
+the doyenne of East-West computer entrepreneurism--share his gift,
+his vision, and his formidable networking talents.
+They are people of the 1960s, winnowed-out by its turbulence
+and rewarded with wealth and influence. They are some of the best
+and the brightest that the electronic community has to offer.
+But can they do it, in the real world? Or are they only dreaming?
+They are so few. And there is so much against them.
+
+I leave Kapor and his networking employees struggling cheerfully
+with the promising intricacies of their newly installed Macintosh
+System 7 software. The next day is Saturday. EFF is closed.
+I pay a few visits to points of interest downtown.
+
+One of them is the birthplace of the telephone.
+
+It's marked by a bronze plaque in a plinth of black-and-white speckled granite. It sits in the
+plaza of the John F. Kennedy Federal Building, the very place where Kapor was
+once fingerprinted by the FBI.
+
+The plaque has a bas-relief picture of Bell's original telephone.
+"BIRTHPLACE OF THE TELEPHONE," it reads. "Here, on June 2, 1875,
+Alexander Graham Bell and Thomas A. Watson first transmitted sound over wires.
+
+"This successful experiment was completed in a fifth floor garret
+at what was then 109 Court Street and marked the beginning of
+world-wide telephone service."
+
+109 Court Street is long gone. Within sight of Bell's plaque,
+across a street, is one of the central offices of NYNEX,
+the local Bell RBOC, on 6 Bowdoin Square.
+
+I cross the street and circle the telco building, slowly,
+hands in my jacket pockets. It's a bright, windy, New England
+autumn day. The central office is a handsome 1940s-era megalith
+in late Art Deco, eight stories high.
+
+Parked outside the back is a power-generation truck.
+The generator strikes me as rather anomalous. Don't they
+already have their own generators in this eight-story monster?
+Then the suspicion strikes me that NYNEX must have heard
+of the September 17 AT&T power-outage which crashed New York City.
+Belt-and-suspenders, this generator. Very telco.
+
+Over the glass doors of the front entrance is a handsome bronze
+bas-relief of Art Deco vines, sunflowers, and birds, entwining
+the Bell logo and the legend NEW ENGLAND TELEPHONE AND TELEGRAPH COMPANY
+--an entity which no longer officially exists.
+
+The doors are locked securely. I peer through the shadowed glass.
+Inside is an official poster reading:
+
+
+"New England Telephone a NYNEX Company
+
+ATTENTION
+
+"All persons while on New England Telephone
+Company premises are required to visibly wear their
+identification cards (C.C.P. Section 2, Page 1).
+
+"Visitors, vendors, contractors, and all others are
+required to visibly wear a daily pass.
+
+"Thank you.
+
+Kevin C. Stanton.
+Building Security Coordinator."
+
+
+Outside, around the corner, is a pull-down ribbed metal security door,
+a locked delivery entrance. Some passing stranger has grafitti-tagged
+this door, with a single word in red spray-painted cursive:
+
+Fury
+
+#
+
+My book on the Hacker Crackdown is almost over now.
+I have deliberately saved the best for last.
+
+In February 1991, I attended the CPSR Public Policy Roundtable,
+in Washington, DC. CPSR, Computer Professionals for Social Responsibility,
+was a sister organization of EFF, or perhaps its aunt, being older
+and perhaps somewhat wiser in the ways of the world of politics.
+
+Computer Professionals for Social Responsibility began in 1981
+in Palo Alto, as an informal discussion group of Californian
+computer scientists and technicians, united by nothing more
+than an electronic mailing list. This typical high-tech
+ad-hocracy received the dignity of its own acronym in 1982,
+and was formally incorporated in 1983.
+
+CPSR lobbied government and public alike with an educational
+outreach effort, sternly warning against any foolish
+and unthinking trust in complex computer systems.
+CPSR insisted that mere computers should never be
+considered a magic panacea for humanity's social,
+ethical or political problems. CPSR members were especially
+troubled about the stability, safety, and dependability
+of military computer systems, and very especially troubled
+by those systems controlling nuclear arsenals. CPSR was
+best-known for its persistent and well-publicized attacks on the
+scientific credibility of the Strategic Defense Initiative ("Star Wars").
+
+In 1990, CPSR was the nation's veteran cyber-political activist group,
+with over two thousand members in twenty- one local chapters across the US.
+It was especially active in Boston, Silicon Valley, and Washington DC,
+where its Washington office sponsored the Public Policy Roundtable.
+
+The Roundtable, however, had been funded by EFF, which had passed CPSR
+an extensive grant for operations. This was the first large-scale,
+official meeting of what was to become the electronic civil
+libertarian community.
+
+Sixty people attended, myself included--in this instance, not so much
+as a journalist as a cyberpunk author. Many of the luminaries
+of the field took part: Kapor and Godwin as a matter of course.
+Richard Civille and Marc Rotenberg of CPSR. Jerry Berman of the ACLU.
+John Quarterman, author of The Matrix. Steven Levy, author of Hackers.
+George Perry and Sandy Weiss of Prodigy Services, there to network
+about the civil-liberties troubles their young commercial
+network was experiencing. Dr. Dorothy Denning. Cliff Figallo,
+manager of the Well. Steve Jackson was there, having finally
+found his ideal target audience, and so was Craig Neidorf,
+"Knight Lightning" himself, with his attorney, Sheldon Zenner.
+Katie Hafner, science journalist, and co-author of Cyberpunk:
+Outlaws and Hackers on the Computer Frontier. Dave Farber,
+ARPAnet pioneer and fabled Internet guru. Janlori Goldman
+of the ACLU's Project on Privacy and Technology. John Nagle
+of Autodesk and the Well. Don Goldberg of the House Judiciary Committee.
+Tom Guidoboni, the defense attorney in the Internet Worm case.
+Lance Hoffman, computer-science professor at The George Washington
+University. Eli Noam of Columbia. And a host of others no less distinguished.
+
+Senator Patrick Leahy delivered the keynote address,
+expressing his determination to keep ahead of the curve
+on the issue of electronic free speech. The address was
+well-received, and the sense of excitement was palpable.
+Every panel discussion was interesting--some were entirely
+compelling. People networked with an almost frantic interest.
+
+I myself had a most interesting and cordial lunch discussion with
+Noel and Jeanne Gayler, Admiral Gayler being a former director
+of the National Security Agency. As this was the first known encounter
+between an actual no-kidding cyberpunk and a chief executive of
+America's largest and best-financed electronic espionage apparat,
+there was naturally a bit of eyebrow-raising on both sides.
+
+Unfortunately, our discussion was off-the-record. In fact
+all the discussions at the CPSR were officially off-the-record,
+the idea being to do some serious networking in an atmosphere
+of complete frankness, rather than to stage a media circus.
+
+In any case, CPSR Roundtable, though interesting and intensely valuable,
+was as nothing compared to the truly mind-boggling event that transpired
+a mere month later.
+
+#
+
+"Computers, Freedom and Privacy." Four hundred people from
+every conceivable corner of America's electronic community.
+As a science fiction writer, I have been to some weird gigs in my day,
+but this thing is truly BEYOND THE PALE. Even "Cyberthon,"
+Point Foundation's "Woodstock of Cyberspace" where Bay Area
+psychedelia collided headlong with the emergent world
+of computerized virtual reality, was like a Kiwanis Club gig
+compared to this astonishing do.
+
+The "electronic community" had reached an apogee.
+Almost every principal in this book is in attendance.
+Civil Libertarians. Computer Cops. The Digital Underground.
+Even a few discreet telco people. Colorcoded dots
+for lapel tags are distributed. Free Expression issues.
+Law Enforcement. Computer Security. Privacy. Journalists.
+Lawyers. Educators. Librarians. Programmers.
+Stylish punk-black dots for the hackers and phone phreaks.
+Almost everyone here seems to wear eight or nine dots,
+to have six or seven professional hats.
+
+It is a community. Something like Lebanon perhaps,
+but a digital nation. People who had feuded all year
+in the national press, people who entertained the deepest
+suspicions of one another's motives and ethics, are now
+in each others' laps. "Computers, Freedom and Privacy"
+had every reason in the world to turn ugly, and yet except
+for small irruptions of puzzling nonsense from the
+convention's token lunatic, a surprising bonhomie reigned.
+CFP was like a wedding-party in which two lovers,
+unstable bride and charlatan groom, tie the knot
+in a clearly disastrous matrimony.
+
+It is clear to both families--even to neighbors and random guests--
+that this is not a workable relationship, and yet the young couple's
+desperate attraction can brook no further delay. They simply cannot
+help themselves. Crockery will fly, shrieks from their newlywed home
+will wake the city block, divorce waits in the wings like a vulture
+over the Kalahari, and yet this is a wedding, and there is going
+to be a child from it. Tragedies end in death; comedies in marriage.
+The Hacker Crackdown is ending in marriage. And there will be a child.
+
+From the beginning, anomalies reign. John Perry Barlow,
+cyberspace ranger, is here. His color photo in
+The New York Times Magazine, Barlow scowling
+in a grim Wyoming snowscape, with long black coat,
+dark hat, a Macintosh SE30 propped on a fencepost
+and an awesome frontier rifle tucked under one arm,
+will be the single most striking visual image
+of the Hacker Crackdown. And he is CFP's guest of honor--
+along with Gail Thackeray of the FCIC! What on earth do
+they expect these dual guests to do with each other? Waltz?
+
+Barlow delivers the first address. Uncharacteristically,
+he is hoarse--the sheer volume of roadwork has worn him down.
+He speaks briefly, congenially, in a plea for conciliation,
+and takes his leave to a storm of applause.
+
+Then Gail Thackeray takes the stage. She's visibly nervous.
+She's been on the Well a lot lately. Reading those Barlow posts.
+Following Barlow is a challenge to anyone. In honor of the famous
+lyricist for the Grateful Dead, she announces reedily, she is going to read--
+A POEM. A poem she has composed herself.
+
+It's an awful poem, doggerel in the rollicking meter of Robert W. Service's
+The Cremation of Sam McGee, but it is in fact, a poem. It's the Ballad
+of the Electronic Frontier! A poem about the Hacker Crackdown and the
+sheer unlikelihood of CFP. It's full of in-jokes. The score or so cops
+in the audience, who are sitting together in a nervous claque,
+are absolutely cracking-up. Gail's poem is the funniest goddamn thing
+they've ever heard. The hackers and civil-libs, who had this woman figured
+for Ilsa She-Wolf of the SS, are staring with their jaws hanging loosely.
+Never in the wildest reaches of their imagination had they figured
+Gail Thackeray was capable of such a totally off-the-wall move.
+You can see them punching their mental CONTROL-RESET buttons.
+Jesus! This woman's a hacker weirdo! She's JUST LIKE US!
+God, this changes everything!
+
+Al Bayse, computer technician for the FBI, had been the only cop
+at the CPSR Roundtable, dragged there with his arm bent by
+Dorothy Denning. He was guarded and tightlipped at CPSR Roundtable;
+a "lion thrown to the Christians."
+
+At CFP, backed by a claque of cops, Bayse suddenly waxes eloquent
+and even droll, describing the FBI's "NCIC 2000", a gigantic digital catalog
+of criminal records, as if he has suddenly become some weird hybrid
+of George Orwell and George Gobel. Tentatively, he makes an arcane
+joke about statistical analysis. At least a third of the crowd laughs aloud.
+
+"They didn't laugh at that at my last speech," Bayse observes.
+He had been addressing cops--STRAIGHT cops, not computer people.
+It had been a worthy meeting, useful one supposes, but nothing like THIS.
+There has never been ANYTHING like this. Without any prodding,
+without any preparation, people in the audience simply begin to ask questions.
+Longhairs, freaky people, mathematicians. Bayse is answering, politely,
+frankly, fully, like a man walking on air. The ballroom's atmosphere
+crackles with surreality. A female lawyer behind me breaks into a sweat
+and a hot waft of surprisingly potent and musky perfume flows off
+her pulse-points.
+
+People are giddy with laughter. People are interested,
+fascinated, their eyes so wide and dark that they seem eroticized.
+Unlikely daisy-chains form in the halls, around the bar, on the escalators:
+cops with hackers, civil rights with FBI, Secret Service with phone phreaks.
+
+Gail Thackeray is at her crispest in a white wool sweater with a
+tiny Secret Service logo. "I found Phiber Optik at the payphones,
+and when he saw my sweater, he turned into a PILLAR OF SALT!" she chortles.
+
+Phiber discusses his case at much length with his arresting officer,
+Don Delaney of the New York State Police. After an hour's chat,
+the two of them look ready to begin singing "Auld Lang Syne."
+Phiber finally finds the courage to get his worst complaint off his chest.
+It isn't so much the arrest. It was the CHARGE. Pirating service
+off 900 numbers. I'm a PROGRAMMER, Phiber insists. This lame charge
+is going to hurt my reputation. It would have been cool to be busted
+for something happening, like Section 1030 computer intrusion.
+Maybe some kind of crime that's scarcely been invented yet.
+Not lousy phone fraud. Phooey.
+
+Delaney seems regretful. He had a mountain of possible criminal charges
+against Phiber Optik. The kid's gonna plead guilty anyway. He's a
+first timer, they always plead. Coulda charged the kid with most anything,
+and gotten the same result in the end. Delaney seems genuinely sorry
+not to have gratified Phiber in this harmless fashion. Too late now.
+Phiber's pled already. All water under the bridge. Whaddya gonna do?
+
+Delaney's got a good grasp on the hacker mentality.
+He held a press conference after he busted a bunch of
+Masters of Deception kids. Some journo had asked him:
+"Would you describe these people as GENIUSES?"
+Delaney's deadpan answer, perfect: "No, I would describe
+these people as DEFENDANTS." Delaney busts a kid for
+hacking codes with repeated random dialling. Tells the
+press that NYNEX can track this stuff in no time flat nowadays,
+and a kid has to be STUPID to do something so easy to catch.
+Dead on again: hackers don't mind being thought of as Genghis Khan
+by the straights, but if there's anything that really gets 'em
+where they live, it's being called DUMB.
+
+Won't be as much fun for Phiber next time around.
+As a second offender he's gonna see prison.
+Hackers break the law. They're not geniuses, either.
+They're gonna be defendants. And yet, Delaney muses over
+a drink in the hotel bar, he has found it impossible to treat
+them as common criminals. Delaney knows criminals. These kids,
+by comparison, are clueless--there is just no crook vibe off of them,
+they don't smell right, they're just not BAD.
+
+Delaney has seen a lot of action. He did Vietnam.
+He's been shot at, he has shot people. He's a homicide
+cop from New York. He has the appearance of a man who
+has not only seen the shit hit the fan but has seen it splattered
+across whole city blocks and left to ferment for years.
+This guy has been around.
+
+He listens to Steve Jackson tell his story. The dreamy
+game strategist has been dealt a bad hand. He has played
+it for all he is worth. Under his nerdish SF-fan exterior
+is a core of iron. Friends of his say Steve Jackson believes
+in the rules, believes in fair play. He will never compromise
+his principles, never give up. "Steve," Delaney says to
+Steve Jackson, "they had some balls, whoever busted you.
+You're all right!" Jackson, stunned, falls silent and
+actually blushes with pleasure.
+
+Neidorf has grown up a lot in the past year. The kid is
+a quick study, you gotta give him that. Dressed by his mom,
+the fashion manager for a national clothing chain,
+Missouri college techie-frat Craig Neidorf out-dappers
+everyone at this gig but the toniest East Coast lawyers.
+The iron jaws of prison clanged shut without him and now
+law school beckons for Neidorf. He looks like a larval Congressman.
+
+Not a "hacker," our Mr. Neidorf. He's not interested
+in computer science. Why should he be? He's not
+interested in writing C code the rest of his life,
+and besides, he's seen where the chips fall.
+To the world of computer science he and Phrack
+were just a curiosity. But to the world of law. . . .
+The kid has learned where the bodies are buried.
+He carries his notebook of press clippings wherever he goes.
+
+Phiber Optik makes fun of Neidorf for a Midwestern geek,
+for believing that "Acid Phreak" does acid and listens to acid rock.
+Hell no. Acid's never done ACID! Acid's into ACID HOUSE MUSIC.
+Jesus. The very idea of doing LSD. Our PARENTS did LSD, ya clown.
+
+Thackeray suddenly turns upon Craig Neidorf the full lighthouse
+glare of her attention and begins a determined half-hour attempt
+to WIN THE BOY OVER. The Joan of Arc of Computer Crime is
+GIVING CAREER ADVICE TO KNIGHT LIGHTNING! "Your experience
+would be very valuable--a real asset," she tells him with
+unmistakeable sixty-thousand-watt sincerity. Neidorf is fascinated.
+He listens with unfeigned attention. He's nodding and saying yes ma'am.
+Yes, Craig, you too can forget all about money and enter the glamorous
+and horribly underpaid world of PROSECUTING COMPUTER CRIME!
+You can put your former friends in prison--ooops. . . .
+
+You cannot go on dueling at modem's length indefinitely.
+You cannot beat one another senseless with rolled-up press-clippings.
+Sooner or later you have to come directly to grips.
+And yet the very act of assembling here has changed
+the entire situation drastically. John Quarterman,
+author of The Matrix, explains the Internet at his symposium.
+It is the largest news network in the world, it is growing
+by leaps and bounds, and yet you cannot measure Internet because
+you cannot stop it in place. It cannot stop, because there
+is no one anywhere in the world with the authority to stop Internet.
+It changes, yes, it grows, it embeds itself across the post-industrial,
+postmodern world and it generates community wherever it
+touches, and it is doing this all by itself.
+
+Phiber is different. A very fin de siecle kid, Phiber Optik.
+Barlow says he looks like an Edwardian dandy. He does rather.
+Shaven neck, the sides of his skull cropped hip-hop close,
+unruly tangle of black hair on top that looks pomaded,
+he stays up till four a.m. and misses all the sessions,
+then hangs out in payphone booths with his acoustic coupler
+gutsily CRACKING SYSTEMS RIGHT IN THE MIDST OF THE HEAVIEST
+LAW ENFORCEMENT DUDES IN THE U.S., or at least PRETENDING to. . . .
+Unlike "Frank Drake." Drake, who wrote Dorothy Denning out
+of nowhere, and asked for an interview for his cheapo
+cyberpunk fanzine, and then started grilling her on her ethics.
+She was squirmin', too. . . . Drake, scarecrow-tall with his
+floppy blond mohawk, rotting tennis shoes and black leather jacket
+lettered ILLUMINATI in red, gives off an unmistakeable air
+of the bohemian literatus. Drake is the kind of guy
+who reads British industrial design magazines and appreciates
+William Gibson because the quality of the prose is so tasty.
+Drake could never touch a phone or a keyboard again,
+and he'd still have the nose-ring and the blurry photocopied
+fanzines and the sampled industrial music. He's a radical punk
+with a desktop-publishing rig and an Internet address.
+Standing next to Drake, the diminutive Phiber looks like he's
+been physically coagulated out of phone-lines. Born to phreak.
+
+Dorothy Denning approaches Phiber suddenly. The two of them
+are about the same height and body-build. Denning's blue eyes
+flash behind the round window-frames of her glasses.
+"Why did you say I was `quaint?'" she asks Phiber, quaintly.
+
+It's a perfect description but Phiber is nonplussed. . .
+"Well, I uh, you know. . . ."
+
+"I also think you're quaint, Dorothy," I say, novelist to the rescue,
+the journo gift of gab. . . . She is neat and dapper and yet there's
+an arcane quality to her, something like a Pilgrim Maiden behind
+leaded glass; if she were six inches high Dorothy Denning would look
+great inside a china cabinet. . .The Cryptographeress. . .
+The Cryptographrix. . .whatever. . . . Weirdly, Peter Denning looks
+just like his wife, you could pick this gentleman out of a thousand guys
+as the soulmate of Dorothy Denning. Wearing tailored slacks,
+a spotless fuzzy varsity sweater, and a neatly knotted academician's tie. . . .
+This fineboned, exquisitely polite, utterly civilized and hyperintelligent
+couple seem to have emerged from some cleaner and finer parallel universe,
+where humanity exists to do the Brain Teasers column in Scientific American.
+Why does this Nice Lady hang out with these unsavory characters?
+
+Because the time has come for it, that's why.
+Because she's the best there is at what she does.
+
+Donn Parker is here, the Great Bald Eagle of Computer Crime. . . .
+With his bald dome, great height, and enormous Lincoln-like hands,
+the great visionary pioneer of the field plows through the lesser mortals
+like an icebreaker. . . . His eyes are fixed on the future with the
+rigidity of a bronze statue. . . . Eventually, he tells his audience,
+all business crime will be computer crime, because businesses will do
+everything through computers. "Computer crime" as a category will vanish.
+
+In the meantime, passing fads will flourish and fail and evaporate. . . .
+Parker's commanding, resonant voice is sphinxlike, everything is viewed
+from some eldritch valley of deep historical abstraction. . . .
+Yes, they've come and they've gone, these passing flaps in the world
+of digital computation. . . . The radio-frequency emanation scandal. . .
+KGB and MI5 and CIA do it every day, it's easy, but nobody else ever has. . . .
+The salami-slice fraud, mostly mythical. . . . "Crimoids," he calls them. . . .
+Computer viruses are the current crimoid champ, a lot less dangerous than
+most people let on, but the novelty is fading and there's a crimoid vacuum at
+the moment, the press is visibly hungering for something more outrageous. . . .
+The Great Man shares with us a few speculations on the coming crimoids. . . .
+Desktop Forgery! Wow. . . . Computers stolen just for the sake of the
+information within them--data-napping! Happened in Britain a while ago,
+could be the coming thing. . . . Phantom nodes in the Internet!
+
+Parker handles his overhead projector sheets with an ecclesiastical air. . . .
+He wears a grey double-breasted suit, a light blue shirt, and a
+very quiet tie of understated maroon and blue paisley. . . .
+Aphorisms emerge from him with slow, leaden emphasis. . . .
+There is no such thing as an adequately secure computer
+when one faces a sufficiently powerful adversary. . . .
+Deterrence is the most socially useful aspect of security. . . .
+People are the primary weakness in all information systems. . . .
+The entire baseline of computer security must be shifted upward. . . .
+Don't ever violate your security by publicly describing
+your security measures. . . .
+
+People in the audience are beginning to squirm, and yet
+there is something about the elemental purity of this guy's
+philosophy that compels uneasy respect. . . . Parker sounds
+like the only sane guy left in the lifeboat, sometimes.
+The guy who can prove rigorously, from deep moral principles,
+that Harvey there, the one with the broken leg and the checkered past,
+is the one who has to be, err. . .that is, Mr. Harvey is best placed
+to make the necessary sacrifice for the security and indeed
+the very survival of the rest of this lifeboat's crew. . . .
+Computer security, Parker informs us mournfully, is a
+nasty topic, and we wish we didn't have to have it. . . .
+The security expert, armed with method and logic, must think--imagine--
+everything that the adversary might do before the adversary might
+actually do it. It is as if the criminal's dark brain were an
+extensive subprogram within the shining cranium of Donn Parker.
+He is a Holmes whose Moriarty does not quite yet exist
+and so must be perfectly simulated.
+
+CFP is a stellar gathering, with the giddiness of a wedding.
+It is a happy time, a happy ending, they know their world
+is changing forever tonight, and they're proud to have been there
+to see it happen, to talk, to think, to help.
+
+And yet as night falls, a certain elegiac quality manifests itself,
+as the crowd gathers beneath the chandeliers with their wineglasses
+and dessert plates. Something is ending here, gone forever,
+and it takes a while to pinpoint it.
+
+It is the End of the Amateurs.
+
+
+
+
+
+
+
+
+
+End of the Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling
+
+*** END OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN ***
+
+***** This file should be named 101.txt or 101.zip *****
+This and all associated files of various formats will be found in:
+ http://www.gutenberg.org/1/0/101/
+
+
+
+Updated editions will replace the previous one--the old editions will be
+renamed.
+
+Creating the works from public domain print editions means that no one
+owns a United States copyright in these works, so the Foundation (and
+you!) can copy and distribute it in the United States without permission
+and without paying copyright royalties. Special rules, set forth in the
+General Terms of Use part of this license, apply to copying and
+distributing Project Gutenberg-tm electronic works to protect the
+PROJECT GUTENBERG-tm concept and trademark. Project Gutenberg is a
+registered trademark, and may not be used if you charge for the eBooks,
+unless you receive specific permission. If you do not charge anything
+for copies of this eBook, complying with the rules is very easy. You may
+use this eBook for nearly any purpose such as creation of derivative
+works, reports, performances and research. They may be modified and
+printed and given away--you may do practically ANYTHING with public
+domain eBooks. Redistribution is subject to the trademark license,
+especially commercial redistribution.
+
+
+
+*** START: FULL LICENSE ***
+
+THE FULL PROJECT GUTENBERG LICENSE
+PLEASE READ THIS BEFORE YOU DISTRIBUTE OR USE THIS WORK
+
+To protect the Project Gutenberg-tm mission of promoting the free
+distribution of electronic works, by using or distributing this work
+(or any other work associated in any way with the phrase "Project
+Gutenberg"), you agree to comply with all the terms of the Full Project
+Gutenberg-tm License (available with this file or online at
+http://www.gutenberg.org/license).
+
+
+Section 1. General Terms of Use and Redistributing Project Gutenberg-tm
+electronic works
+
+1.A. By reading or using any part of this Project Gutenberg-tm
+electronic work, you indicate that you have read, understand, agree to
+and accept all the terms of this license and intellectual property
+(trademark/copyright) agreement. If you do not agree to abide by all
+the terms of this agreement, you must cease using and return or destroy
+all copies of Project Gutenberg-tm electronic works in your possession.
+If you paid a fee for obtaining a copy of or access to a Project
+Gutenberg-tm electronic work and you do not agree to be bound by the
+terms of this agreement, you may obtain a refund from the person or
+entity to whom you paid the fee as set forth in paragraph 1.E.8.
+
+1.B. "Project Gutenberg" is a registered trademark. It may only be
+used on or associated in any way with an electronic work by people who
+agree to be bound by the terms of this agreement. There are a few
+things that you can do with most Project Gutenberg-tm electronic works
+even without complying with the full terms of this agreement. See
+paragraph 1.C below. There are a lot of things you can do with Project
+Gutenberg-tm electronic works if you follow the terms of this agreement
+and help preserve free future access to Project Gutenberg-tm electronic
+works. See paragraph 1.E below.
+
+1.C. The Project Gutenberg Literary Archive Foundation ("the Foundation"
+or PGLAF), owns a compilation copyright in the collection of Project
+Gutenberg-tm electronic works. Nearly all the individual works in the
+collection are in the public domain in the United States. If an
+individual work is in the public domain in the United States and you are
+located in the United States, we do not claim a right to prevent you from
+copying, distributing, performing, displaying or creating derivative
+works based on the work as long as all references to Project Gutenberg
+are removed. Of course, we hope that you will support the Project
+Gutenberg-tm mission of promoting free access to electronic works by
+freely sharing Project Gutenberg-tm works in compliance with the terms of
+this agreement for keeping the Project Gutenberg-tm name associated with
+the work. You can easily comply with the terms of this agreement by
+keeping this work in the same format with its attached full Project
+Gutenberg-tm License when you share it without charge with others.
+This particular work is one of the few copyrighted individual works
+included with the permission of the copyright holder. Information on
+the copyright owner for this particular work and the terms of use
+imposed by the copyright holder on this work are set forth at the
+beginning of this work.
+
+1.D. The copyright laws of the place where you are located also govern
+what you can do with this work. Copyright laws in most countries are in
+a constant state of change. If you are outside the United States, check
+the laws of your country in addition to the terms of this agreement
+before downloading, copying, displaying, performing, distributing or
+creating derivative works based on this work or any other Project
+Gutenberg-tm work. The Foundation makes no representations concerning
+the copyright status of any work in any country outside the United
+States.
+
+1.E. Unless you have removed all references to Project Gutenberg:
+
+1.E.1. The following sentence, with active links to, or other immediate
+access to, the full Project Gutenberg-tm License must appear prominently
+whenever any copy of a Project Gutenberg-tm work (any work on which the
+phrase "Project Gutenberg" appears, or with which the phrase "Project
+Gutenberg" is associated) is accessed, displayed, performed, viewed,
+copied or distributed:
+
+This eBook is for the use of anyone anywhere at no cost and with
+almost no restrictions whatsoever. You may copy it, give it away or
+re-use it under the terms of the Project Gutenberg License included
+with this eBook or online at www.gutenberg.org
+
+1.E.2. If an individual Project Gutenberg-tm electronic work is derived
+from the public domain (does not contain a notice indicating that it is
+posted with permission of the copyright holder), the work can be copied
+and distributed to anyone in the United States without paying any fees
+or charges. If you are redistributing or providing access to a work
+with the phrase "Project Gutenberg" associated with or appearing on the
+work, you must comply either with the requirements of paragraphs 1.E.1
+through 1.E.7 or obtain permission for the use of the work and the
+Project Gutenberg-tm trademark as set forth in paragraphs 1.E.8 or
+1.E.9.
+
+1.E.3. If an individual Project Gutenberg-tm electronic work is posted
+with the permission of the copyright holder, your use and distribution
+must comply with both paragraphs 1.E.1 through 1.E.7 and any additional
+terms imposed by the copyright holder. Additional terms will be linked
+to the Project Gutenberg-tm License for all works posted with the
+permission of the copyright holder found at the beginning of this work.
+
+1.E.4. Do not unlink or detach or remove the full Project Gutenberg-tm
+License terms from this work, or any files containing a part of this
+work or any other work associated with Project Gutenberg-tm.
+
+1.E.5. Do not copy, display, perform, distribute or redistribute this
+electronic work, or any part of this electronic work, without
+prominently displaying the sentence set forth in paragraph 1.E.1 with
+active links or immediate access to the full terms of the Project
+Gutenberg-tm License.
+
+1.E.6. You may convert to and distribute this work in any binary,
+compressed, marked up, nonproprietary or proprietary form, including any
+word processing or hypertext form. However, if you provide access to or
+distribute copies of a Project Gutenberg-tm work in a format other than
+"Plain Vanilla ASCII" or other format used in the official version
+posted on the official Project Gutenberg-tm web site (www.gutenberg.org),
+you must, at no additional cost, fee or expense to the user, provide a
+copy, a means of exporting a copy, or a means of obtaining a copy upon
+request, of the work in its original "Plain Vanilla ASCII" or other
+form. Any alternate format must include the full Project Gutenberg-tm
+License as specified in paragraph 1.E.1.
+
+1.E.7. Do not charge a fee for access to, viewing, displaying,
+performing, copying or distributing any Project Gutenberg-tm works
+unless you comply with paragraph 1.E.8 or 1.E.9.
+
+1.E.8. You may charge a reasonable fee for copies of or providing
+access to or distributing Project Gutenberg-tm electronic works provided
+that
+
+- You pay a royalty fee of 20% of the gross profits you derive from
+ the use of Project Gutenberg-tm works calculated using the method
+ you already use to calculate your applicable taxes. The fee is
+ owed to the owner of the Project Gutenberg-tm trademark, but he
+ has agreed to donate royalties under this paragraph to the
+ Project Gutenberg Literary Archive Foundation. Royalty payments
+ must be paid within 60 days following each date on which you
+ prepare (or are legally required to prepare) your periodic tax
+ returns. Royalty payments should be clearly marked as such and
+ sent to the Project Gutenberg Literary Archive Foundation at the
+ address specified in Section 4, "Information about donations to
+ the Project Gutenberg Literary Archive Foundation."
+
+- You provide a full refund of any money paid by a user who notifies
+ you in writing (or by e-mail) within 30 days of receipt that s/he
+ does not agree to the terms of the full Project Gutenberg-tm
+ License. You must require such a user to return or
+ destroy all copies of the works possessed in a physical medium
+ and discontinue all use of and all access to other copies of
+ Project Gutenberg-tm works.
+
+- You provide, in accordance with paragraph 1.F.3, a full refund of any
+ money paid for a work or a replacement copy, if a defect in the
+ electronic work is discovered and reported to you within 90 days
+ of receipt of the work.
+
+- You comply with all other terms of this agreement for free
+ distribution of Project Gutenberg-tm works.
+
+1.E.9. If you wish to charge a fee or distribute a Project Gutenberg-tm
+electronic work or group of works on different terms than are set
+forth in this agreement, you must obtain permission in writing from
+both the Project Gutenberg Literary Archive Foundation and Michael
+Hart, the owner of the Project Gutenberg-tm trademark. Contact the
+Foundation as set forth in Section 3 below.
+
+1.F.
+
+1.F.1. Project Gutenberg volunteers and employees expend considerable
+effort to identify, do copyright research on, transcribe and proofread
+public domain works in creating the Project Gutenberg-tm
+collection. Despite these efforts, Project Gutenberg-tm electronic
+works, and the medium on which they may be stored, may contain
+"Defects," such as, but not limited to, incomplete, inaccurate or
+corrupt data, transcription errors, a copyright or other intellectual
+property infringement, a defective or damaged disk or other medium, a
+computer virus, or computer codes that damage or cannot be read by
+your equipment.
+
+1.F.2. LIMITED WARRANTY, DISCLAIMER OF DAMAGES - Except for the "Right
+of Replacement or Refund" described in paragraph 1.F.3, the Project
+Gutenberg Literary Archive Foundation, the owner of the Project
+Gutenberg-tm trademark, and any other party distributing a Project
+Gutenberg-tm electronic work under this agreement, disclaim all
+liability to you for damages, costs and expenses, including legal
+fees. YOU AGREE THAT YOU HAVE NO REMEDIES FOR NEGLIGENCE, STRICT
+LIABILITY, BREACH OF WARRANTY OR BREACH OF CONTRACT EXCEPT THOSE
+PROVIDED IN PARAGRAPH 1.F.3. YOU AGREE THAT THE FOUNDATION, THE
+TRADEMARK OWNER, AND ANY DISTRIBUTOR UNDER THIS AGREEMENT WILL NOT BE
+LIABLE TO YOU FOR ACTUAL, DIRECT, INDIRECT, CONSEQUENTIAL, PUNITIVE OR
+INCIDENTAL DAMAGES EVEN IF YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+1.F.3. LIMITED RIGHT OF REPLACEMENT OR REFUND - If you discover a
+defect in this electronic work within 90 days of receiving it, you can
+receive a refund of the money (if any) you paid for it by sending a
+written explanation to the person you received the work from. If you
+received the work on a physical medium, you must return the medium with
+your written explanation. The person or entity that provided you with
+the defective work may elect to provide a replacement copy in lieu of a
+refund. If you received the work electronically, the person or entity
+providing it to you may choose to give you a second opportunity to
+receive the work electronically in lieu of a refund. If the second copy
+is also defective, you may demand a refund in writing without further
+opportunities to fix the problem.
+
+1.F.4. Except for the limited right of replacement or refund set forth
+in paragraph 1.F.3, this work is provided to you 'AS-IS,' WITH NO OTHER
+WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+WARRANTIES OF MERCHANTIBILITY OR FITNESS FOR ANY PURPOSE.
+
+1.F.5. Some states do not allow disclaimers of certain implied
+warranties or the exclusion or limitation of certain types of damages.
+If any disclaimer or limitation set forth in this agreement violates the
+law of the state applicable to this agreement, the agreement shall be
+interpreted to make the maximum disclaimer or limitation permitted by
+the applicable state law. The invalidity or unenforceability of any
+provision of this agreement shall not void the remaining provisions.
+
+1.F.6. INDEMNITY - You agree to indemnify and hold the Foundation, the
+trademark owner, any agent or employee of the Foundation, anyone
+providing copies of Project Gutenberg-tm electronic works in accordance
+with this agreement, and any volunteers associated with the production,
+promotion and distribution of Project Gutenberg-tm electronic works,
+harmless from all liability, costs and expenses, including legal fees,
+that arise directly or indirectly from any of the following which you do
+or cause to occur: (a) distribution of this or any Project Gutenberg-tm
+work, (b) alteration, modification, or additions or deletions to any
+Project Gutenberg-tm work, and (c) any Defect you cause.
+
+
+Section 2. Information about the Mission of Project Gutenberg-tm
+
+Project Gutenberg-tm is synonymous with the free distribution of
+electronic works in formats readable by the widest variety of computers
+including obsolete, old, middle-aged and new computers. It exists
+because of the efforts of hundreds of volunteers and donations from
+people in all walks of life.
+
+Volunteers and financial support to provide volunteers with the
+assistance they need are critical to reaching Project Gutenberg-tm's
+goals and ensuring that the Project Gutenberg-tm collection will
+remain freely available for generations to come. In 2001, the Project
+Gutenberg Literary Archive Foundation was created to provide a secure
+and permanent future for Project Gutenberg-tm and future generations.
+To learn more about the Project Gutenberg Literary Archive Foundation
+and how your efforts and donations can help, see Sections 3 and 4
+and the Foundation web page at http://www.pglaf.org.
+
+
+Section 3. Information about the Project Gutenberg Literary Archive
+Foundation
+
+The Project Gutenberg Literary Archive Foundation is a non profit
+501(c)(3) educational corporation organized under the laws of the
+state of Mississippi and granted tax exempt status by the Internal
+Revenue Service. The Foundation's EIN or federal tax identification
+number is 64-6221541. Its 501(c)(3) letter is posted at
+http://pglaf.org/fundraising. Contributions to the Project Gutenberg
+Literary Archive Foundation are tax deductible to the full extent
+permitted by U.S. federal laws and your state's laws.
+
+The Foundation's principal office is located at 4557 Melan Dr. S.
+Fairbanks, AK, 99712., but its volunteers and employees are scattered
+throughout numerous locations. Its business office is located at
+809 North 1500 West, Salt Lake City, UT 84116, (801) 596-1887, email
+business@pglaf.org. Email contact links and up to date contact
+information can be found at the Foundation's web site and official
+page at http://pglaf.org
+
+For additional contact information:
+ Dr. Gregory B. Newby
+ Chief Executive and Director
+ gbnewby@pglaf.org
+
+Section 4. Information about Donations to the Project Gutenberg
+Literary Archive Foundation
+
+Project Gutenberg-tm depends upon and cannot survive without wide
+spread public support and donations to carry out its mission of
+increasing the number of public domain and licensed works that can be
+freely distributed in machine readable form accessible by the widest
+array of equipment including outdated equipment. Many small donations
+($1 to $5,000) are particularly important to maintaining tax exempt
+status with the IRS.
+
+The Foundation is committed to complying with the laws regulating
+charities and charitable donations in all 50 states of the United
+States. Compliance requirements are not uniform and it takes a
+considerable effort, much paperwork and many fees to meet and keep up
+with these requirements. We do not solicit donations in locations
+where we have not received written confirmation of compliance. To
+SEND DONATIONS or determine the status of compliance for any
+particular state visit http://pglaf.org
+
+While we cannot and do not solicit contributions from states where we
+have not met the solicitation requirements, we know of no prohibition
+against accepting unsolicited donations from donors in such states who
+approach us with offers to donate.
+
+International donations are gratefully accepted, but we cannot make
+any statements concerning tax treatment of donations received from
+outside the United States. U.S. laws alone swamp our small staff.
+
+Please check the Project Gutenberg Web pages for current donation
+methods and addresses. Donations are accepted in a number of other
+ways including checks, online payments and credit card donations.
+To donate, please visit: http://pglaf.org/donate
+
+
+Section 5. General Information About Project Gutenberg-tm electronic
+works.
+
+Professor Michael S. Hart is the originator of the Project Gutenberg-tm
+concept of a library of electronic works that could be freely shared
+with anyone. For thirty years, he produced and distributed Project
+Gutenberg-tm eBooks with only a loose network of volunteer support.
+
+Project Gutenberg-tm eBooks are often created from several printed
+editions, all of which are confirmed as Public Domain in the U.S.
+unless a copyright notice is included. Thus, we do not necessarily
+keep eBooks in compliance with any particular paper edition.
+
+Each eBook is in a subdirectory of the same number as the eBook's
+eBook number, often in several formats including plain vanilla ASCII,
+compressed (zipped), HTML and others.
+
+Corrected EDITIONS of our eBooks replace the old file and take over
+the old filename and etext number. The replaced older file is renamed.
+VERSIONS based on separate sources are treated as new eBooks receiving
+new filenames and etext numbers.
+
+Most people start at our Web site which has the main PG search facility:
+
+http://www.gutenberg.org
+
+This Web site includes information about Project Gutenberg-tm,
+including how to make donations to the Project Gutenberg Literary
+Archive Foundation, how to help produce our new eBooks, and how to
+subscribe to our email newsletter to hear about new eBooks.
+
+EBooks posted prior to November 2003, with eBook numbers BELOW #10000,
+are filed in directories based on their release date. If you want to
+download any of these eBooks directly, rather than using the regular
+search system you may utilize the following addresses and just
+download by the etext year.
+
+http://www.ibiblio.org/gutenberg/etext06
+
+ (Or /etext 05, 04, 03, 02, 01, 00, 99,
+ 98, 97, 96, 95, 94, 93, 92, 92, 91 or 90)
+
+EBooks posted since November 2003, with etext numbers OVER #10000, are
+filed in a different way. The year of a release date is no longer part
+of the directory path. The path is based on the etext number (which is
+identical to the filename). The path to the file is made up of single
+digits corresponding to all but the last digit in the filename. For
+example an eBook of filename 10234 would be found at:
+
+http://www.gutenberg.org/1/0/2/3/10234
+
+or filename 24689 would be found at:
+http://www.gutenberg.org/2/4/6/8/24689
+
+An alternative method of locating eBooks:
+http://www.gutenberg.org/GUTINDEX.ALL
+
+*** END: FULL LICENSE ***
diff --git a/common/src/leap/soledad/common/tests/test_async.py b/common/src/leap/soledad/common/tests/test_async.py
new file mode 100644
index 00000000..03b8c553
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_async.py
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+# test_async.py
+# Copyright (C) 2013, 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import hashlib
+
+from twisted.internet import defer
+
+from leap.soledad.common.tests.util import BaseSoledadTest
+from leap.soledad.client import adbapi
+from leap.soledad.client.sqlcipher import SQLCipherOptions
+
+
+class ASyncSQLCipherRetryTestCase(BaseSoledadTest):
+ """
+ Test asynchronous SQLCipher operation.
+ """
+
+ NUM_DOCS = 5000
+
+ def _get_dbpool(self):
+ tmpdb = os.path.join(self.tempdir, "test.soledad")
+ opts = SQLCipherOptions(tmpdb, "secret", create=True)
+ return adbapi.getConnectionPool(opts)
+
+ def _get_sample(self):
+ if not getattr(self, "_sample", None):
+ dirname = os.path.dirname(os.path.realpath(__file__))
+ sample_file = os.path.join(dirname, "hacker_crackdown.txt")
+ with open(sample_file) as f:
+ self._sample = f.readlines()
+ return self._sample
+
+ def test_concurrent_puts_fail_with_few_retries_and_small_timeout(self):
+ """
+ Test if concurrent updates to the database with small timeout and
+ small number of retries fail with "database is locked" error.
+
+ Many concurrent write attempts to the same sqlcipher database may fail
+ when the timeout is small and there are no retries. This test will
+ pass if any of the attempts to write the database fail.
+
+ This test is much dependent on the environment and its result intends
+ to contrast with the test for the workaround for the "database is
+ locked" problem, which is addressed by the "test_concurrent_puts" test
+ below.
+
+ If this test ever fails, it means that either (1) the platform where
+ you are running is it very powerful and you should try with an even
+ lower timeout value, or (2) the bug has been solved by a better
+ implementation of the underlying database pool, and thus this test
+ should be removed from the test suite.
+ """
+
+ old_timeout = adbapi.SQLCIPHER_CONNECTION_TIMEOUT
+ old_max_retries = adbapi.SQLCIPHER_MAX_RETRIES
+
+ adbapi.SQLCIPHER_CONNECTION_TIMEOUT = 1
+ adbapi.SQLCIPHER_MAX_RETRIES = 1
+
+ dbpool = self._get_dbpool()
+
+ def _create_doc(doc):
+ return dbpool.runU1DBQuery("create_doc", doc)
+
+ def _insert_docs():
+ deferreds = []
+ for i in range(self.NUM_DOCS):
+ payload = self._get_sample()[i]
+ chash = hashlib.sha256(payload).hexdigest()
+ doc = {"number": i, "payload": payload, 'chash': chash}
+ d = _create_doc(doc)
+ deferreds.append(d)
+ return defer.gatherResults(deferreds, consumeErrors=True)
+
+ def _errback(e):
+ if e.value[0].getErrorMessage() == "database is locked":
+ adbapi.SQLCIPHER_CONNECTION_TIMEOUT = old_timeout
+ adbapi.SQLCIPHER_MAX_RETRIES = old_max_retries
+ return defer.succeed("")
+ raise Exception
+
+ d = _insert_docs()
+ d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs"))
+ d.addErrback(_errback)
+ return d
+
+ def test_concurrent_puts(self):
+ """
+ Test that many concurrent puts succeed.
+
+ Currently, there's a known problem with the concurrent database pool
+ which is that many concurrent attempts to write to the database may
+ fail when the lock timeout is small and when there are no (or few)
+ retries. We currently workaround this problem by increasing the
+ timeout and the number of retries.
+
+ Should this test ever fail, it probably means that the timeout and/or
+ number of retries should be increased for the platform you're running
+ the test. If the underlying database pool is ever fixed, then the test
+ above will fail and we should remove this comment from here.
+ """
+
+ dbpool = self._get_dbpool()
+
+ def _create_doc(doc):
+ return dbpool.runU1DBQuery("create_doc", doc)
+
+ def _insert_docs():
+ deferreds = []
+ for i in range(self.NUM_DOCS):
+ payload = self._get_sample()[i]
+ chash = hashlib.sha256(payload).hexdigest()
+ doc = {"number": i, "payload": payload, 'chash': chash}
+ d = _create_doc(doc)
+ deferreds.append(d)
+ return defer.gatherResults(deferreds, consumeErrors=True)
+
+
+ def _count_docs(results):
+ _, docs = results
+ if self.NUM_DOCS == len(docs):
+ return defer.succeed("")
+ raise Exception
+
+ d = _insert_docs()
+ d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs"))
+ d.addCallback(_count_docs)
+ return d
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 10d6c136..d2aef9bb 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -20,134 +20,21 @@
Test ObjectStore and Couch backend bits.
"""
-import re
-import copy
-import shutil
-from base64 import b64decode
-from mock import Mock
-from urlparse import urljoin
+import simplejson as json
+
+from urlparse import urljoin
from u1db import errors as u1db_errors
from couchdb.client import Server
-from leap.common.files import mkdir_p
+from testscenarios import TestWithScenarios
+
+from leap.soledad.common import couch, errors
from leap.soledad.common.tests import u1db_tests as tests
from leap.soledad.common.tests.u1db_tests import test_backends
from leap.soledad.common.tests.u1db_tests import test_sync
-from leap.soledad.common import couch, errors
-import simplejson as json
-
-
-#-----------------------------------------------------------------------------
-# A wrapper for running couchdb locally.
-#-----------------------------------------------------------------------------
-
-import re
-import os
-import tempfile
-import subprocess
-import time
-import unittest
-
-
-# from: https://github.com/smcq/paisley/blob/master/paisley/test/util.py
-# TODO: include license of above project.
-class CouchDBWrapper(object):
- """
- Wrapper for external CouchDB instance which is started and stopped for
- testing.
- """
-
- def start(self):
- """
- Start a CouchDB instance for a test.
- """
- self.tempdir = tempfile.mkdtemp(suffix='.couch.test')
-
- path = os.path.join(os.path.dirname(__file__),
- 'couchdb.ini.template')
- handle = open(path)
- conf = handle.read() % {
- 'tempdir': self.tempdir,
- }
- handle.close()
-
- confPath = os.path.join(self.tempdir, 'test.ini')
- handle = open(confPath, 'w')
- handle.write(conf)
- handle.close()
-
- # create the dirs from the template
- mkdir_p(os.path.join(self.tempdir, 'lib'))
- mkdir_p(os.path.join(self.tempdir, 'log'))
- args = ['couchdb', '-n', '-a', confPath]
- null = open('/dev/null', 'w')
-
- self.process = subprocess.Popen(
- args, env=None, stdout=null.fileno(), stderr=null.fileno(),
- close_fds=True)
- # find port
- logPath = os.path.join(self.tempdir, 'log', 'couch.log')
- while not os.path.exists(logPath):
- if self.process.poll() is not None:
- got_stdout, got_stderr = "", ""
- if self.process.stdout is not None:
- got_stdout = self.process.stdout.read()
-
- if self.process.stderr is not None:
- got_stderr = self.process.stderr.read()
- raise Exception("""
-couchdb exited with code %d.
-stdout:
-%s
-stderr:
-%s""" % (
- self.process.returncode, got_stdout, got_stderr))
- time.sleep(0.01)
- while os.stat(logPath).st_size == 0:
- time.sleep(0.01)
- PORT_RE = re.compile(
- 'Apache CouchDB has started on http://127.0.0.1:(?P<port>\d+)')
-
- handle = open(logPath)
- line = handle.read()
- handle.close()
- m = PORT_RE.search(line)
- if not m:
- self.stop()
- raise Exception("Cannot find port in line %s" % line)
- self.port = int(m.group('port'))
-
- def stop(self):
- """
- Terminate the CouchDB instance.
- """
- self.process.terminate()
- self.process.communicate()
- shutil.rmtree(self.tempdir)
-
-
-class CouchDBTestCase(unittest.TestCase):
- """
- TestCase base class for tests against a real CouchDB server.
- """
-
- @classmethod
- def setUpClass(cls):
- """
- Make sure we have a CouchDB instance for a test.
- """
- cls.wrapper = CouchDBWrapper()
- cls.wrapper.start()
- #self.db = self.wrapper.db
-
- @classmethod
- def tearDownClass(cls):
- """
- Stop CouchDB instance for test.
- """
- cls.wrapper.stop()
+from leap.soledad.common.tests.util import CouchDBTestCase
#-----------------------------------------------------------------------------
@@ -239,7 +126,8 @@ COUCH_SCENARIOS = [
]
-class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase):
+class CouchTests(
+ TestWithScenarios, test_backends.AllDatabaseTests, CouchDBTestCase):
scenarios = COUCH_SCENARIOS
@@ -262,7 +150,8 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase):
test_backends.AllDatabaseTests.tearDown(self)
-class CouchDatabaseTests(test_backends.LocalDatabaseTests, CouchDBTestCase):
+class CouchDatabaseTests(
+ TestWithScenarios, test_backends.LocalDatabaseTests, CouchDBTestCase):
scenarios = COUCH_SCENARIOS
@@ -271,7 +160,7 @@ class CouchDatabaseTests(test_backends.LocalDatabaseTests, CouchDBTestCase):
test_backends.LocalDatabaseTests.tearDown(self)
-class CouchValidateGenNTransIdTests(
+class CouchValidateGenNTransIdTests(TestWithScenarios,
test_backends.LocalDatabaseValidateGenNTransIdTests, CouchDBTestCase):
scenarios = COUCH_SCENARIOS
@@ -281,7 +170,7 @@ class CouchValidateGenNTransIdTests(
test_backends.LocalDatabaseValidateGenNTransIdTests.tearDown(self)
-class CouchValidateSourceGenTests(
+class CouchValidateSourceGenTests(TestWithScenarios,
test_backends.LocalDatabaseValidateSourceGenTests, CouchDBTestCase):
scenarios = COUCH_SCENARIOS
@@ -291,7 +180,7 @@ class CouchValidateSourceGenTests(
test_backends.LocalDatabaseValidateSourceGenTests.tearDown(self)
-class CouchWithConflictsTests(
+class CouchWithConflictsTests(TestWithScenarios,
test_backends.LocalDatabaseWithConflictsTests, CouchDBTestCase):
scenarios = COUCH_SCENARIOS
@@ -325,23 +214,11 @@ simple_doc = tests.simple_doc
nested_doc = tests.nested_doc
-class CouchDatabaseSyncTargetTests(test_sync.DatabaseSyncTargetTests,
- CouchDBTestCase):
+class CouchDatabaseSyncTargetTests(
+ TestWithScenarios, test_sync.DatabaseSyncTargetTests, CouchDBTestCase):
scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios))
- def setUp(self):
- # we implement parents' setUp methods here to prevent from launching
- # more couch instances then needed.
- tests.TestCase.setUp(self)
- self.server = self.server_thread = None
- self.db, self.st = self.create_db_and_target(self)
- self.other_changes = []
-
- def tearDown(self):
- self.db.delete_database()
- test_sync.DatabaseSyncTargetTests.tearDown(self)
-
def test_sync_exchange_returns_many_new_docs(self):
# This test was replicated to allow dictionaries to be compared after
# JSON expansion (because one dictionary may have many different
@@ -372,7 +249,7 @@ from u1db.backends.inmemory import InMemoryIndex
class IndexedCouchDatabase(couch.CouchDatabase):
def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True):
- old_class.__init__(self, url, dbname, replica_uid=replica_uid,
+ old_class.__init__(self, url, dbname, replica_uid=replica_uid,
ensure_ddocs=ensure_ddocs)
self._indexes = {}
@@ -458,7 +335,8 @@ for name, scenario in COUCH_SCENARIOS:
scenario = dict(scenario)
-class CouchDatabaseSyncTests(test_sync.DatabaseSyncTests, CouchDBTestCase):
+class CouchDatabaseSyncTests(
+ TestWithScenarios, test_sync.DatabaseSyncTests, CouchDBTestCase):
scenarios = sync_scenarios
@@ -498,6 +376,7 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
def tearDown(self):
self.db.delete_database()
self.db.close()
+ CouchDBTestCase.tearDown(self)
def test_missing_design_doc_raises(self):
"""
@@ -670,6 +549,3 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
self.assertRaises(
errors.MissingDesignDocDeletedError,
self.db._do_set_replica_gen_and_trans_id, 1, 2, 3)
-
-
-load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
index 6465eb80..3de4da1c 100644
--- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
+++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
@@ -15,26 +15,25 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-Test atomocity for couch operations.
+Test atomicity of couch operations.
"""
import os
-import mock
import tempfile
import threading
-
from urlparse import urljoin
-
+from twisted.internet import defer
from leap.soledad.client import Soledad
from leap.soledad.common.couch import CouchDatabase, CouchServerState
-from leap.soledad.common.tests.test_couch import CouchDBTestCase
-from leap.soledad.common.tests.u1db_tests import TestCaseWithServer
-from leap.soledad.common.tests.test_sync_target import (
+
+from leap.soledad.common.tests.util import (
make_token_soledad_app,
- make_leap_document_for_test,
- token_leap_sync_target,
+ make_soledad_document_for_test,
+ token_soledad_sync_target,
)
+from leap.soledad.common.tests.test_couch import CouchDBTestCase
+from leap.soledad.common.tests.u1db_tests import TestCaseWithServer
from leap.soledad.common.tests.test_server import _couch_ensure_database
@@ -52,15 +51,15 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
def make_app_after_state(state):
return make_token_soledad_app(state)
- make_document_for_test = make_leap_document_for_test
+ make_document_for_test = make_soledad_document_for_test
- sync_target = token_leap_sync_target
+ sync_target = token_soledad_sync_target
def _soledad_instance(self, user='user-uuid', passphrase=u'123',
prefix='',
- secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME,
+ secrets_path='secrets.json',
local_db_path='soledad.u1db', server_url='',
- cert_file=None, auth_token=None, secret_id=None):
+ cert_file=None, auth_token=None):
"""
Instantiate Soledad.
"""
@@ -70,19 +69,6 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
def _put_doc_side_effect(doc):
self._doc_put = doc
- # we need a mocked shared db or else Soledad will try to access the
- # network to find if there are uploaded secrets.
- class MockSharedDB(object):
-
- get_doc = mock.Mock(return_value=None)
- put_doc = mock.Mock(side_effect=_put_doc_side_effect)
- lock = mock.Mock(return_value=('atoken', 300))
- unlock = mock.Mock()
-
- def __call__(self):
- return self
-
- Soledad._shared_db = MockSharedDB()
return Soledad(
user,
passphrase,
@@ -92,11 +78,10 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
server_url=server_url,
cert_file=cert_file,
auth_token=auth_token,
- secret_id=secret_id)
+ shared_db=self.get_default_shared_mock(_put_doc_side_effect))
def make_app(self):
- self.request_state = CouchServerState(self._couch_url, 'shared',
- 'tokens')
+ self.request_state = CouchServerState(self._couch_url)
return self.make_app_after_state(self.request_state)
def setUp(self):
@@ -126,7 +111,6 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
puts.
"""
doc = self.db.create_doc({'ops': 0})
- ops = 1
docs = [doc.doc_id]
for i in range(0, REPEAT_TIMES):
self.assertEqual(
@@ -183,24 +167,27 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
auth_token='auth-token',
server_url=self.getURL())
- def _create_docs_and_sync(sol, syncs):
- # create a lot of documents
- for i in range(0, REPEAT_TIMES):
- sol.create_doc({})
+ def _create_docs(results):
+ deferreds = []
+ for i in xrange(0, REPEAT_TIMES):
+ deferreds.append(sol.create_doc({}))
+ return defer.DeferredList(deferreds)
+
+ def _assert_transaction_and_sync_logs(results, sync_idx):
# assert sizes of transaction and sync logs
self.assertEqual(
- syncs*REPEAT_TIMES,
+ sync_idx*REPEAT_TIMES,
len(self.db._get_transaction_log()))
self.assertEqual(
- 1 if syncs > 0 else 0,
+ 1 if sync_idx > 0 else 0,
len(self.db._database.view('syncs/log').rows))
- # sync to the remote db
- sol.sync()
- gen, docs = self.db.get_all_docs()
- self.assertEqual((syncs+1)*REPEAT_TIMES, gen)
- self.assertEqual((syncs+1)*REPEAT_TIMES, len(docs))
+
+ def _assert_sync(results, sync_idx):
+ gen, docs = results
+ self.assertEqual((sync_idx+1)*REPEAT_TIMES, gen)
+ self.assertEqual((sync_idx+1)*REPEAT_TIMES, len(docs))
# assert sizes of transaction and sync logs
- self.assertEqual((syncs+1)*REPEAT_TIMES,
+ self.assertEqual((sync_idx+1)*REPEAT_TIMES,
len(self.db._get_transaction_log()))
sync_log_rows = self.db._database.view('syncs/log').rows
sync_log = sync_log_rows[0].value
@@ -210,14 +197,32 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
# assert sync_log has exactly 1 row
self.assertEqual(1, len(sync_log_rows))
# assert it has the correct replica_uid, gen and trans_id
- self.assertEqual(sol._db._replica_uid, replica_uid)
- sol_gen, sol_trans_id = sol._db._get_generation_info()
+ self.assertEqual(sol._dbpool.replica_uid, replica_uid)
+ conn_key = sol._dbpool._u1dbconnections.keys().pop()
+ conn = sol._dbpool._u1dbconnections[conn_key]
+ sol_gen, sol_trans_id = conn._get_generation_info()
self.assertEqual(sol_gen, known_gen)
self.assertEqual(sol_trans_id, known_trans_id)
+
+ # create some documents
+ d = _create_docs(None)
- _create_docs_and_sync(sol, 0)
- _create_docs_and_sync(sol, 1)
- sol.close()
+ # sync first time and assert success
+ d.addCallback(_assert_transaction_and_sync_logs, 0)
+ d.addCallback(lambda _: sol.sync())
+ d.addCallback(lambda _: sol.get_all_docs())
+ d.addCallback(_assert_sync, 0)
+
+ # create more docs, sync second time and assert success
+ d.addCallback(_create_docs)
+ d.addCallback(_assert_transaction_and_sync_logs, 1)
+ d.addCallback(lambda _: sol.sync())
+ d.addCallback(lambda _: sol.get_all_docs())
+ d.addCallback(_assert_sync, 1)
+
+ d.addCallback(lambda _: sol.close())
+
+ return d
#
# Concurrency tests
@@ -313,86 +318,76 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
"""
Assert that the sync_log is correct after concurrent syncs.
"""
- threads = []
docs = []
- pool = threading.BoundedSemaphore(value=1)
+
self.startServer()
+
sol = self._soledad_instance(
auth_token='auth-token',
server_url=self.getURL())
- def _run_method(self):
- # create a lot of documents
- doc = self._params['sol'].create_doc({})
- pool.acquire()
- docs.append(doc.doc_id)
- pool.release()
+ def _save_doc_ids(results):
+ for doc in results:
+ docs.append(doc.doc_id)
- # launch threads to create documents in parallel
+ # create documents in parallel
+ deferreds = []
for i in range(0, REPEAT_TIMES):
- thread = self._WorkerThread(
- {'sol': sol, 'syncs': i},
- _run_method)
- thread.start()
- threads.append(thread)
+ d = sol.create_doc({})
+ deferreds.append(d)
- # wait for threads to finish
- for thread in threads:
- thread.join()
+ # wait for documents creation and sync
+ d = defer.gatherResults(deferreds)
+ d.addCallback(_save_doc_ids)
+ d.addCallback(lambda _: sol.sync())
- # do the sync!
- sol.sync()
+ def _assert_logs(results):
+ transaction_log = self.db._get_transaction_log()
+ self.assertEqual(REPEAT_TIMES, len(transaction_log))
+ # assert all documents are in the remote log
+ self.assertEqual(REPEAT_TIMES, len(docs))
+ for doc_id in docs:
+ self.assertEqual(
+ 1,
+ len(filter(lambda t: t[0] == doc_id, transaction_log)))
- transaction_log = self.db._get_transaction_log()
- self.assertEqual(REPEAT_TIMES, len(transaction_log))
- # assert all documents are in the remote log
- self.assertEqual(REPEAT_TIMES, len(docs))
- for doc_id in docs:
- self.assertEqual(
- 1,
- len(filter(lambda t: t[0] == doc_id, transaction_log)))
- sol.close()
+ d.addCallback(_assert_logs)
+ d.addCallback(lambda _: sol.close())
+
+ return d
def test_concurrent_syncs_do_not_fail(self):
"""
Assert that concurrent attempts to sync end up being executed
sequentially and do not fail.
"""
- threads = []
docs = []
- pool = threading.BoundedSemaphore(value=1)
+
self.startServer()
+
sol = self._soledad_instance(
auth_token='auth-token',
server_url=self.getURL())
- def _run_method(self):
- # create a lot of documents
- doc = self._params['sol'].create_doc({})
- # do the sync!
- sol.sync()
- pool.acquire()
- docs.append(doc.doc_id)
- pool.release()
-
- # launch threads to create documents in parallel
- for i in range(0, REPEAT_TIMES):
- thread = self._WorkerThread(
- {'sol': sol, 'syncs': i},
- _run_method)
- thread.start()
- threads.append(thread)
-
- # wait for threads to finish
- for thread in threads:
- thread.join()
-
- transaction_log = self.db._get_transaction_log()
- self.assertEqual(REPEAT_TIMES, len(transaction_log))
- # assert all documents are in the remote log
- self.assertEqual(REPEAT_TIMES, len(docs))
- for doc_id in docs:
- self.assertEqual(
- 1,
- len(filter(lambda t: t[0] == doc_id, transaction_log)))
- sol.close()
+ deferreds = []
+ for i in xrange(0, REPEAT_TIMES):
+ d = sol.create_doc({})
+ d.addCallback(lambda doc: docs.append(doc.doc_id))
+ d.addCallback(lambda _: sol.sync())
+ deferreds.append(d)
+
+ def _assert_logs(results):
+ transaction_log = self.db._get_transaction_log()
+ self.assertEqual(REPEAT_TIMES, len(transaction_log))
+ # assert all documents are in the remote log
+ self.assertEqual(REPEAT_TIMES, len(docs))
+ for doc_id in docs:
+ self.assertEqual(
+ 1,
+ len(filter(lambda t: t[0] == doc_id, transaction_log)))
+
+ d = defer.gatherResults(deferreds)
+ d.addCallback(_assert_logs)
+ d.addCallback(lambda _: sol.close())
+
+ return d
diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py
index 1071af14..fdad8aac 100644
--- a/common/src/leap/soledad/common/tests/test_crypto.py
+++ b/common/src/leap/soledad/common/tests/test_crypto.py
@@ -23,8 +23,14 @@ import binascii
from leap.soledad.client import crypto
from leap.soledad.common.document import SoledadDocument
-from leap.soledad.common.tests import BaseSoledadTest
-from leap.soledad.common.crypto import WrongMac, UnknownMacMethod
+from leap.soledad.common.tests.util import BaseSoledadTest
+from leap.soledad.common.crypto import WrongMacError
+from leap.soledad.common.crypto import UnknownMacMethodError
+from leap.soledad.common.crypto import EncryptionMethods
+from leap.soledad.common.crypto import ENC_JSON_KEY
+from leap.soledad.common.crypto import ENC_SCHEME_KEY
+from leap.soledad.common.crypto import MAC_KEY
+from leap.soledad.common.crypto import MAC_METHOD_KEY
class EncryptedSyncTestCase(BaseSoledadTest):
@@ -46,8 +52,8 @@ class EncryptedSyncTestCase(BaseSoledadTest):
self.assertNotEqual(
simpledoc, doc1.content,
'incorrect document encryption')
- self.assertTrue(crypto.ENC_JSON_KEY in doc1.content)
- self.assertTrue(crypto.ENC_SCHEME_KEY in doc1.content)
+ self.assertTrue(ENC_JSON_KEY in doc1.content)
+ self.assertTrue(ENC_SCHEME_KEY in doc1.content)
# decrypt doc
doc1.set_json(crypto.decrypt_doc(self._soledad._crypto, doc1))
self.assertEqual(
@@ -57,23 +63,28 @@ class EncryptedSyncTestCase(BaseSoledadTest):
class RecoveryDocumentTestCase(BaseSoledadTest):
def test_export_recovery_document_raw(self):
- rd = self._soledad.export_recovery_document()
- secret_id = rd[self._soledad.STORAGE_SECRETS_KEY].items()[0][0]
- secret = rd[self._soledad.STORAGE_SECRETS_KEY][secret_id]
- self.assertEqual(secret_id, self._soledad._secret_id)
- self.assertEqual(secret, self._soledad._secrets[secret_id])
- self.assertTrue(self._soledad.CIPHER_KEY in secret)
- self.assertTrue(secret[self._soledad.CIPHER_KEY] == 'aes256')
- self.assertTrue(self._soledad.LENGTH_KEY in secret)
- self.assertTrue(self._soledad.SECRET_KEY in secret)
+ rd = self._soledad.secrets._export_recovery_document()
+ secret_id = rd[self._soledad.secrets.STORAGE_SECRETS_KEY].items()[0][0]
+ # assert exported secret is the same
+ secret = self._soledad.secrets._decrypt_storage_secret(
+ rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id])
+ self.assertEqual(secret_id, self._soledad.secrets._secret_id)
+ self.assertEqual(secret, self._soledad.secrets._secrets[secret_id])
+ # assert recovery document structure
+ encrypted_secret = rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id]
+ self.assertTrue(self._soledad.secrets.CIPHER_KEY in encrypted_secret)
+ self.assertTrue(
+ encrypted_secret[self._soledad.secrets.CIPHER_KEY] == 'aes256')
+ self.assertTrue(self._soledad.secrets.LENGTH_KEY in encrypted_secret)
+ self.assertTrue(self._soledad.secrets.SECRET_KEY in encrypted_secret)
def test_import_recovery_document(self):
- rd = self._soledad.export_recovery_document()
+ rd = self._soledad.secrets._export_recovery_document()
s = self._soledad_instance()
- s.import_recovery_document(rd)
- s._set_secret_id(self._soledad._secret_id)
- self.assertEqual(self._soledad._get_storage_secret(),
- s._get_storage_secret(),
+ s.secrets._import_recovery_document(rd)
+ s.secrets.set_secret_id(self._soledad.secrets._secret_id)
+ self.assertEqual(self._soledad.storage_secret,
+ s.storage_secret,
'Failed settinng secret for symmetric encryption.')
s.close()
@@ -83,32 +94,32 @@ class SoledadSecretsTestCase(BaseSoledadTest):
def test__gen_secret(self):
# instantiate and save secret_id
sol = self._soledad_instance(user='user@leap.se')
- self.assertTrue(len(sol._secrets) == 1)
- secret_id_1 = sol.secret_id
+ self.assertTrue(len(sol.secrets._secrets) == 1)
+ secret_id_1 = sol.secrets.secret_id
# assert id is hash of secret
self.assertTrue(
secret_id_1 == hashlib.sha256(sol.storage_secret).hexdigest())
# generate new secret
- secret_id_2 = sol._gen_secret()
+ secret_id_2 = sol.secrets._gen_secret()
self.assertTrue(secret_id_1 != secret_id_2)
sol.close()
# re-instantiate
- sol = self._soledad_instance(
- user='user@leap.se',
- secret_id=secret_id_1)
+ sol = self._soledad_instance(user='user@leap.se')
+ sol.secrets.set_secret_id(secret_id_1)
# assert ids are valid
- self.assertTrue(len(sol._secrets) == 2)
- self.assertTrue(secret_id_1 in sol._secrets)
- self.assertTrue(secret_id_2 in sol._secrets)
+ self.assertTrue(len(sol.secrets._secrets) == 2)
+ self.assertTrue(secret_id_1 in sol.secrets._secrets)
+ self.assertTrue(secret_id_2 in sol.secrets._secrets)
# assert format of secret 1
self.assertTrue(sol.storage_secret is not None)
self.assertIsInstance(sol.storage_secret, str)
- self.assertTrue(len(sol.storage_secret) == sol.GENERATED_SECRET_LENGTH)
+ secret_length = sol.secrets.GEN_SECRET_LENGTH
+ self.assertTrue(len(sol.storage_secret) == secret_length)
# assert format of secret 2
- sol._set_secret_id(secret_id_2)
+ sol.secrets.set_secret_id(secret_id_2)
self.assertTrue(sol.storage_secret is not None)
self.assertIsInstance(sol.storage_secret, str)
- self.assertTrue(len(sol.storage_secret) == sol.GENERATED_SECRET_LENGTH)
+ self.assertTrue(len(sol.storage_secret) == secret_length)
# assert id is hash of new secret
self.assertTrue(
secret_id_2 == hashlib.sha256(sol.storage_secret).hexdigest())
@@ -117,16 +128,18 @@ class SoledadSecretsTestCase(BaseSoledadTest):
def test__has_secret(self):
sol = self._soledad_instance(
user='user@leap.se', prefix=self.rand_prefix)
- self.assertTrue(sol._has_secret(), "Should have a secret at "
- "this point")
+ self.assertTrue(
+ sol.secrets._has_secret(),
+ "Should have a secret at this point")
# setting secret id to None should not interfere in the fact we have a
# secret.
- sol._set_secret_id(None)
- self.assertTrue(sol._has_secret(), "Should have a secret at "
- "this point")
+ sol.secrets.set_secret_id(None)
+ self.assertTrue(
+ sol.secrets._has_secret(),
+ "Should have a secret at this point")
# but not being able to decrypt correctly should
- sol._secrets[sol.secret_id][sol.SECRET_KEY] = None
- self.assertFalse(sol._has_secret())
+ sol.secrets._secrets[sol.secrets.secret_id] = None
+ self.assertFalse(sol.secrets._has_secret())
sol.close()
@@ -141,13 +154,13 @@ class MacAuthTestCase(BaseSoledadTest):
doc.content = simpledoc
# encrypt doc
doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc))
- self.assertTrue(crypto.MAC_KEY in doc.content)
- self.assertTrue(crypto.MAC_METHOD_KEY in doc.content)
+ self.assertTrue(MAC_KEY in doc.content)
+ self.assertTrue(MAC_METHOD_KEY in doc.content)
# mess with MAC
- doc.content[crypto.MAC_KEY] = '1234567890ABCDEF'
+ doc.content[MAC_KEY] = '1234567890ABCDEF'
# try to decrypt doc
self.assertRaises(
- WrongMac,
+ WrongMacError,
crypto.decrypt_doc, self._soledad._crypto, doc)
def test_decrypt_with_unknown_mac_method_raises(self):
@@ -159,13 +172,13 @@ class MacAuthTestCase(BaseSoledadTest):
doc.content = simpledoc
# encrypt doc
doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc))
- self.assertTrue(crypto.MAC_KEY in doc.content)
- self.assertTrue(crypto.MAC_METHOD_KEY in doc.content)
+ self.assertTrue(MAC_KEY in doc.content)
+ self.assertTrue(MAC_METHOD_KEY in doc.content)
# mess with MAC method
- doc.content[crypto.MAC_METHOD_KEY] = 'mymac'
+ doc.content[MAC_METHOD_KEY] = 'mymac'
# try to decrypt doc
self.assertRaises(
- UnknownMacMethod,
+ UnknownMacMethodError,
crypto.decrypt_doc, self._soledad._crypto, doc)
@@ -176,20 +189,20 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):
key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
- method=crypto.EncryptionMethods.AES_256_CTR)
+ method=EncryptionMethods.AES_256_CTR)
self.assertTrue(cyphertext is not None)
self.assertTrue(cyphertext != '')
self.assertTrue(cyphertext != 'data')
plaintext = self._soledad._crypto.decrypt_sym(
cyphertext, key, iv=iv,
- method=crypto.EncryptionMethods.AES_256_CTR)
+ method=EncryptionMethods.AES_256_CTR)
self.assertEqual('data', plaintext)
def test_decrypt_with_wrong_iv_fails(self):
key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
- method=crypto.EncryptionMethods.AES_256_CTR)
+ method=EncryptionMethods.AES_256_CTR)
self.assertTrue(cyphertext is not None)
self.assertTrue(cyphertext != '')
self.assertTrue(cyphertext != 'data')
@@ -200,14 +213,14 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):
wrongiv = os.urandom(1) + rawiv[1:]
plaintext = self._soledad._crypto.decrypt_sym(
cyphertext, key, iv=binascii.b2a_base64(wrongiv),
- method=crypto.EncryptionMethods.AES_256_CTR)
+ method=EncryptionMethods.AES_256_CTR)
self.assertNotEqual('data', plaintext)
def test_decrypt_with_wrong_key_fails(self):
key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
- method=crypto.EncryptionMethods.AES_256_CTR)
+ method=EncryptionMethods.AES_256_CTR)
self.assertTrue(cyphertext is not None)
self.assertTrue(cyphertext != '')
self.assertTrue(cyphertext != 'data')
@@ -217,7 +230,7 @@ class SoledadCryptoAESTestCase(BaseSoledadTest):
wrongkey = os.urandom(32)
plaintext = self._soledad._crypto.decrypt_sym(
cyphertext, wrongkey, iv=iv,
- method=crypto.EncryptionMethods.AES_256_CTR)
+ method=EncryptionMethods.AES_256_CTR)
self.assertNotEqual('data', plaintext)
@@ -228,20 +241,20 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest):
key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
- method=crypto.EncryptionMethods.XSALSA20)
+ method=EncryptionMethods.XSALSA20)
self.assertTrue(cyphertext is not None)
self.assertTrue(cyphertext != '')
self.assertTrue(cyphertext != 'data')
plaintext = self._soledad._crypto.decrypt_sym(
cyphertext, key, iv=iv,
- method=crypto.EncryptionMethods.XSALSA20)
+ method=EncryptionMethods.XSALSA20)
self.assertEqual('data', plaintext)
def test_decrypt_with_wrong_iv_fails(self):
key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
- method=crypto.EncryptionMethods.XSALSA20)
+ method=EncryptionMethods.XSALSA20)
self.assertTrue(cyphertext is not None)
self.assertTrue(cyphertext != '')
self.assertTrue(cyphertext != 'data')
@@ -252,14 +265,14 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest):
wrongiv = os.urandom(1) + rawiv[1:]
plaintext = self._soledad._crypto.decrypt_sym(
cyphertext, key, iv=binascii.b2a_base64(wrongiv),
- method=crypto.EncryptionMethods.XSALSA20)
+ method=EncryptionMethods.XSALSA20)
self.assertNotEqual('data', plaintext)
def test_decrypt_with_wrong_key_fails(self):
key = os.urandom(32)
iv, cyphertext = self._soledad._crypto.encrypt_sym(
'data', key,
- method=crypto.EncryptionMethods.XSALSA20)
+ method=EncryptionMethods.XSALSA20)
self.assertTrue(cyphertext is not None)
self.assertTrue(cyphertext != '')
self.assertTrue(cyphertext != 'data')
@@ -269,5 +282,5 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest):
wrongkey = os.urandom(32)
plaintext = self._soledad._crypto.decrypt_sym(
cyphertext, wrongkey, iv=iv,
- method=crypto.EncryptionMethods.XSALSA20)
+ method=EncryptionMethods.XSALSA20)
self.assertNotEqual('data', plaintext)
diff --git a/common/src/leap/soledad/common/tests/test_http.py b/common/src/leap/soledad/common/tests/test_http.py
index d21470e0..1f661b77 100644
--- a/common/src/leap/soledad/common/tests/test_http.py
+++ b/common/src/leap/soledad/common/tests/test_http.py
@@ -20,8 +20,6 @@ Test Leap backend bits: test http database
from u1db.remote import http_database
from leap.soledad.client import auth
-
-from leap.soledad.common.tests import u1db_tests as tests
from leap.soledad.common.tests.u1db_tests import test_http_database
@@ -59,6 +57,3 @@ class TestHTTPDatabaseWithCreds(
'token': 'auth-token',
}})
self.assertIn('token', db1._creds)
-
-
-load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_http_client.py b/common/src/leap/soledad/common/tests/test_http_client.py
index 3169398b..db731c32 100644
--- a/common/src/leap/soledad/common/tests/test_http_client.py
+++ b/common/src/leap/soledad/common/tests/test_http_client.py
@@ -21,8 +21,9 @@ import json
from u1db.remote import http_client
+from testscenarios import TestWithScenarios
+
from leap.soledad.client import auth
-from leap.soledad.common.tests import u1db_tests as tests
from leap.soledad.common.tests.u1db_tests import test_http_client
from leap.soledad.server.auth import SoledadTokenAuthMiddleware
@@ -31,7 +32,9 @@ from leap.soledad.server.auth import SoledadTokenAuthMiddleware
# The following tests come from `u1db.tests.test_http_client`.
#-----------------------------------------------------------------------------
-class TestSoledadClientBase(test_http_client.TestHTTPClientBase):
+class TestSoledadClientBase(
+ TestWithScenarios,
+ test_http_client.TestHTTPClientBase):
"""
This class should be used to test Token auth.
"""
@@ -90,7 +93,7 @@ class TestSoledadClientBase(test_http_client.TestHTTPClientBase):
"message": e.message})]
uuid, token = encoded.decode('base64').split(':', 1)
if uuid != 'user-uuid' and token != 'auth-token':
- return unauth_err("Incorrect address or token.")
+ return Exception("Incorrect address or token.")
start_response("200 OK", [('Content-Type', 'application/json')])
return [json.dumps([environ['PATH_INFO'], uuid, token])]
@@ -112,5 +115,3 @@ class TestSoledadClientBase(test_http_client.TestHTTPClientBase):
res, headers = cli._request('GET', ['doc', 'token'])
self.assertEqual(
['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res))
-
-load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_https.py b/common/src/leap/soledad/common/tests/test_https.py
index b6288188..4dd55754 100644
--- a/common/src/leap/soledad/common/tests/test_https.py
+++ b/common/src/leap/soledad/common/tests/test_https.py
@@ -14,30 +14,35 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
"""
Test Leap backend bits: https
"""
-from leap.soledad.common.tests import BaseSoledadTest
-from leap.soledad.common.tests import test_sync_target as test_st
-from leap.soledad.common.tests import u1db_tests as tests
-from leap.soledad.common.tests.u1db_tests import test_backends
-from leap.soledad.common.tests.u1db_tests import test_https
-from leap.soledad import client
-from leap.soledad.server import SoledadApp
from u1db.remote import http_client
+from leap.soledad import client
+
+from testscenarios import TestWithScenarios
+
+from leap.soledad.common.tests.u1db_tests import test_backends
+from leap.soledad.common.tests.u1db_tests import test_https
+from leap.soledad.common.tests.util import (
+ BaseSoledadTest,
+ make_soledad_document_for_test,
+ make_soledad_app,
+ make_token_soledad_app,
+)
-def make_soledad_app(state):
- return SoledadApp(state)
LEAP_SCENARIOS = [
('http', {
'make_database_for_test': test_backends.make_http_database_for_test,
'copy_database_for_test': test_backends.copy_http_database_for_test,
- 'make_document_for_test': test_st.make_leap_document_for_test,
- 'make_app_with_state': test_st.make_soledad_app}),
+ 'make_document_for_test': make_soledad_document_for_test,
+ 'make_app_with_state': make_soledad_app}),
]
@@ -55,14 +60,15 @@ def token_leap_https_sync_target(test, host, path):
class TestSoledadSyncTargetHttpsSupport(
+ TestWithScenarios,
test_https.TestHttpSyncTargetHttpsSupport,
BaseSoledadTest):
scenarios = [
('token_soledad_https',
{'server_def': test_https.https_server_def,
- 'make_app_with_state': test_st.make_token_soledad_app,
- 'make_document_for_test': test_st.make_leap_document_for_test,
+ 'make_app_with_state': make_token_soledad_app,
+ 'make_document_for_test': make_soledad_document_for_test,
'sync_target': token_leap_https_sync_target}),
]
@@ -71,8 +77,8 @@ class TestSoledadSyncTargetHttpsSupport(
# run smoothly with standard u1db.
test_https.TestHttpSyncTargetHttpsSupport.setUp(self)
# so here monkey patch again to test our functionality.
- http_client._VerifiedHTTPSConnection = client.VerifiedHTTPSConnection
- client.SOLEDAD_CERT = http_client.CA_CERTS
+ http_client._VerifiedHTTPSConnection = client.api.VerifiedHTTPSConnection
+ client.api.SOLEDAD_CERT = http_client.CA_CERTS
def test_working(self):
"""
@@ -83,7 +89,7 @@ class TestSoledadSyncTargetHttpsSupport(
"""
self.startServer()
db = self.request_state._create_database('test')
- self.patch(client, 'SOLEDAD_CERT', self.cacert_pem)
+ self.patch(client.api, 'SOLEDAD_CERT', self.cacert_pem)
remote_target = self.getSyncTarget('localhost', 'test')
remote_target.record_sync_info('other-id', 2, 'T-id')
self.assertEqual(
@@ -99,10 +105,8 @@ class TestSoledadSyncTargetHttpsSupport(
"""
self.startServer()
self.request_state._create_database('test')
- self.patch(client, 'SOLEDAD_CERT', self.cacert_pem)
+ self.patch(client.api, 'SOLEDAD_CERT', self.cacert_pem)
remote_target = self.getSyncTarget('127.0.0.1', 'test')
self.assertRaises(
http_client.CertificateError, remote_target.record_sync_info,
'other-id', 2, 'T-id')
-
-load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index cb5348b4..2b653a1c 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -19,29 +19,28 @@ Tests for server-related functionality.
"""
import os
import tempfile
-import simplejson as json
import mock
import time
import binascii
from urlparse import urljoin
+from twisted.internet import defer
-from leap.common.testing.basetest import BaseLeapTest
from leap.soledad.common.couch import (
CouchServerState,
CouchDatabase,
)
-from leap.soledad.common.tests.u1db_tests import (
- TestCaseWithServer,
- simple_doc,
-)
+from leap.soledad.common.tests.u1db_tests import TestCaseWithServer
from leap.soledad.common.tests.test_couch import CouchDBTestCase
-from leap.soledad.common.tests.test_target_soledad import (
+from leap.soledad.common.tests.util import (
make_token_soledad_app,
- make_leap_document_for_test,
+ make_soledad_document_for_test,
+ token_soledad_sync_target,
+ BaseSoledadTest,
)
-from leap.soledad.common.tests.test_sync_target import token_leap_sync_target
-from leap.soledad.client import Soledad, crypto
+
+from leap.soledad.common import crypto
+from leap.soledad.client import Soledad
from leap.soledad.server import LockResource
from leap.soledad.server.auth import URLToAuthorization
@@ -58,7 +57,7 @@ def _couch_ensure_database(self, dbname):
CouchServerState.ensure_database = _couch_ensure_database
-class ServerAuthorizationTestCase(BaseLeapTest):
+class ServerAuthorizationTestCase(BaseSoledadTest):
"""
Tests related to Soledad server authorization.
"""
@@ -272,19 +271,24 @@ class EncryptedSyncTestCase(
Tests for encrypted sync using Soledad server backed by a couch database.
"""
+ # increase twisted.trial's timeout because large files syncing might take
+ # some time to finish.
+ timeout = 500
+
@staticmethod
def make_app_with_state(state):
return make_token_soledad_app(state)
- make_document_for_test = make_leap_document_for_test
+ make_document_for_test = make_soledad_document_for_test
- sync_target = token_leap_sync_target
+ sync_target = token_soledad_sync_target
def _soledad_instance(self, user='user-uuid', passphrase=u'123',
prefix='',
- secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME,
- local_db_path='soledad.u1db', server_url='',
- cert_file=None, auth_token=None, secret_id=None):
+ secrets_path='secrets.json',
+ local_db_path='soledad.u1db',
+ server_url='',
+ cert_file=None, auth_token=None):
"""
Instantiate Soledad.
"""
@@ -294,19 +298,15 @@ class EncryptedSyncTestCase(
def _put_doc_side_effect(doc):
self._doc_put = doc
- # we need a mocked shared db or else Soledad will try to access the
- # network to find if there are uploaded secrets.
- class MockSharedDB(object):
-
- get_doc = mock.Mock(return_value=None)
- put_doc = mock.Mock(side_effect=_put_doc_side_effect)
- lock = mock.Mock(return_value=('atoken', 300))
- unlock = mock.Mock()
-
- def __call__(self):
- return self
+ if not server_url:
+ # attempt to find the soledad server url
+ server_address = None
+ server = getattr(self, 'server', None)
+ if server:
+ server_address = getattr(self.server, 'server_address', None)
+ if server_address:
+ server_url = 'http://%s:%d' % (server_address)
- Soledad._shared_db = MockSharedDB()
return Soledad(
user,
passphrase,
@@ -316,78 +316,129 @@ class EncryptedSyncTestCase(
server_url=server_url,
cert_file=cert_file,
auth_token=auth_token,
- secret_id=secret_id)
+ shared_db=self.get_default_shared_mock(_put_doc_side_effect))
def make_app(self):
- self.request_state = CouchServerState(self._couch_url, 'shared',
- 'tokens')
+ self.request_state = CouchServerState(self._couch_url)
return self.make_app_with_state(self.request_state)
def setUp(self):
- TestCaseWithServer.setUp(self)
+ # the order of the following initializations is crucial because of
+ # dependencies.
+ # XXX explain better
CouchDBTestCase.setUp(self)
- self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
self._couch_url = 'http://localhost:' + str(self.wrapper.port)
+ self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
+ TestCaseWithServer.setUp(self)
def tearDown(self):
CouchDBTestCase.tearDown(self)
TestCaseWithServer.tearDown(self)
- def test_encrypted_sym_sync(self):
+ def _test_encrypted_sym_sync(self, passphrase=u'123', doc_size=2,
+ number_of_docs=1):
"""
Test the complete syncing chain between two soledad dbs using a
Soledad server backed by a couch database.
"""
self.startServer()
+
# instantiate soledad and create a document
sol1 = self._soledad_instance(
# token is verified in test_target.make_token_soledad_app
- auth_token='auth-token'
- )
- _, doclist = sol1.get_all_docs()
- self.assertEqual([], doclist)
- doc1 = sol1.create_doc(json.loads(simple_doc))
+ auth_token='auth-token',
+ passphrase=passphrase)
+
+ # instantiate another soledad using the same secret as the previous
+ # one (so we can correctly verify the mac of the synced document)
+ sol2 = self._soledad_instance(
+ prefix='x',
+ auth_token='auth-token',
+ secrets_path=sol1._secrets_path,
+ passphrase=passphrase)
+
# ensure remote db exists before syncing
db = CouchDatabase.open_database(
urljoin(self._couch_url, 'user-user-uuid'),
create=True,
ensure_ddocs=True)
- # sync with server
- sol1._server_url = self.getURL()
- sol1.sync()
- # assert doc was sent to couch db
- _, doclist = db.get_all_docs()
- self.assertEqual(1, len(doclist))
- couchdoc = doclist[0]
- # assert document structure in couch server
- self.assertEqual(doc1.doc_id, couchdoc.doc_id)
- self.assertEqual(doc1.rev, couchdoc.rev)
- self.assertEqual(6, len(couchdoc.content))
- self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content)
- self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content)
- self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content)
- self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content)
- self.assertTrue(crypto.MAC_KEY in couchdoc.content)
- self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content)
- # instantiate soledad with empty db, but with same secrets path
- sol2 = self._soledad_instance(prefix='x', auth_token='auth-token')
- _, doclist = sol2.get_all_docs()
- self.assertEqual([], doclist)
- sol2._secrets_path = sol1.secrets_path
- sol2._load_secrets()
- sol2._set_secret_id(sol1._secret_id)
- # sync the new instance
- sol2._server_url = self.getURL()
- sol2.sync()
- _, doclist = sol2.get_all_docs()
- self.assertEqual(1, len(doclist))
- doc2 = doclist[0]
- # assert incoming doc is equal to the first sent doc
- self.assertEqual(doc1, doc2)
- db.delete_database()
- db.close()
- sol1.close()
- sol2.close()
+
+ def _db1AssertEmptyDocList(results):
+ _, doclist = results
+ self.assertEqual([], doclist)
+
+ def _db1CreateDocs(results):
+ deferreds = []
+ for i in xrange(number_of_docs):
+ content = binascii.hexlify(os.urandom(doc_size/2))
+ deferreds.append(sol1.create_doc({'data': content}))
+ return defer.DeferredList(deferreds)
+
+ def _db1AssertDocsSyncedToServer(results):
+ _, sol_doclist = results
+ self.assertEqual(number_of_docs, len(sol_doclist))
+ # assert doc was sent to couch db
+ _, couch_doclist = db.get_all_docs()
+ self.assertEqual(number_of_docs, len(couch_doclist))
+ for i in xrange(number_of_docs):
+ soldoc = sol_doclist.pop()
+ couchdoc = couch_doclist.pop()
+ # assert document structure in couch server
+ self.assertEqual(soldoc.doc_id, couchdoc.doc_id)
+ self.assertEqual(soldoc.rev, couchdoc.rev)
+ self.assertEqual(6, len(couchdoc.content))
+ self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content)
+ self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content)
+ self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content)
+ self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content)
+ self.assertTrue(crypto.MAC_KEY in couchdoc.content)
+ self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content)
+
+ d = sol1.get_all_docs()
+ d.addCallback(_db1AssertEmptyDocList)
+ d.addCallback(_db1CreateDocs)
+ d.addCallback(lambda _: sol1.sync())
+ d.addCallback(lambda _: sol1.get_all_docs())
+ d.addCallback(_db1AssertDocsSyncedToServer)
+
+ def _db2AssertEmptyDocList(results):
+ _, doclist = results
+ self.assertEqual([], doclist)
+
+ def _getAllDocsFromBothDbs(results):
+ d1 = sol1.get_all_docs()
+ d2 = sol2.get_all_docs()
+ return defer.DeferredList([d1, d2])
+
+ d.addCallback(lambda _: sol2.get_all_docs())
+ d.addCallback(_db2AssertEmptyDocList)
+ d.addCallback(lambda _: sol2.sync())
+ d.addCallback(_getAllDocsFromBothDbs)
+
+ def _assertDocSyncedFromDb1ToDb2(results):
+ r1, r2 = results
+ _, (gen1, doclist1) = r1
+ _, (gen2, doclist2) = r2
+ self.assertEqual(number_of_docs, gen1)
+ self.assertEqual(number_of_docs, gen2)
+ self.assertEqual(number_of_docs, len(doclist1))
+ self.assertEqual(number_of_docs, len(doclist2))
+ self.assertEqual(doclist1[0], doclist2[0])
+
+ d.addCallback(_assertDocSyncedFromDb1ToDb2)
+
+ def _cleanUp(results):
+ db.delete_database()
+ db.close()
+ sol1.close()
+ sol2.close()
+
+ d.addCallback(_cleanUp)
+
+ return d
+
+ def test_encrypted_sym_sync(self):
+ return self._test_encrypted_sym_sync()
def test_encrypted_sym_sync_with_unicode_passphrase(self):
"""
@@ -395,152 +446,20 @@ class EncryptedSyncTestCase(
Soledad server backed by a couch database, using an unicode
passphrase.
"""
- self.startServer()
- # instantiate soledad and create a document
- sol1 = self._soledad_instance(
- # token is verified in test_target.make_token_soledad_app
- auth_token='auth-token',
- passphrase=u'ãáàäéàëíìïóòöõúùüñç',
- )
- _, doclist = sol1.get_all_docs()
- self.assertEqual([], doclist)
- doc1 = sol1.create_doc(json.loads(simple_doc))
- # ensure remote db exists before syncing
- db = CouchDatabase.open_database(
- urljoin(self._couch_url, 'user-user-uuid'),
- create=True,
- ensure_ddocs=True)
- # sync with server
- sol1._server_url = self.getURL()
- sol1.sync()
- # assert doc was sent to couch db
- _, doclist = db.get_all_docs()
- self.assertEqual(1, len(doclist))
- couchdoc = doclist[0]
- # assert document structure in couch server
- self.assertEqual(doc1.doc_id, couchdoc.doc_id)
- self.assertEqual(doc1.rev, couchdoc.rev)
- self.assertEqual(6, len(couchdoc.content))
- self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content)
- self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content)
- self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content)
- self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content)
- self.assertTrue(crypto.MAC_KEY in couchdoc.content)
- self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content)
- # instantiate soledad with empty db, but with same secrets path
- sol2 = self._soledad_instance(
- prefix='x',
- auth_token='auth-token',
- passphrase=u'ãáàäéàëíìïóòöõúùüñç',
- )
- _, doclist = sol2.get_all_docs()
- self.assertEqual([], doclist)
- sol2._secrets_path = sol1.secrets_path
- sol2._load_secrets()
- sol2._set_secret_id(sol1._secret_id)
- # sync the new instance
- sol2._server_url = self.getURL()
- sol2.sync()
- _, doclist = sol2.get_all_docs()
- self.assertEqual(1, len(doclist))
- doc2 = doclist[0]
- # assert incoming doc is equal to the first sent doc
- self.assertEqual(doc1, doc2)
- db.delete_database()
- db.close()
- sol1.close()
- sol2.close()
+ return self._test_encrypted_sym_sync(passphrase=u'ãáàäéàëíìïóòöõúùüñç')
def test_sync_very_large_files(self):
"""
Test if Soledad can sync very large files.
"""
- # define the size of the "very large file"
length = 100*(10**6) # 100 MB
- self.startServer()
- # instantiate soledad and create a document
- sol1 = self._soledad_instance(
- # token is verified in test_target.make_token_soledad_app
- auth_token='auth-token'
- )
- _, doclist = sol1.get_all_docs()
- self.assertEqual([], doclist)
- content = binascii.hexlify(os.urandom(length/2)) # len() == length
- doc1 = sol1.create_doc({'data': content})
- # ensure remote db exists before syncing
- db = CouchDatabase.open_database(
- urljoin(self._couch_url, 'user-user-uuid'),
- create=True,
- ensure_ddocs=True)
- # sync with server
- sol1._server_url = self.getURL()
- sol1.sync()
- # instantiate soledad with empty db, but with same secrets path
- sol2 = self._soledad_instance(prefix='x', auth_token='auth-token')
- _, doclist = sol2.get_all_docs()
- self.assertEqual([], doclist)
- sol2._secrets_path = sol1.secrets_path
- sol2._load_secrets()
- sol2._set_secret_id(sol1._secret_id)
- # sync the new instance
- sol2._server_url = self.getURL()
- sol2.sync()
- _, doclist = sol2.get_all_docs()
- self.assertEqual(1, len(doclist))
- doc2 = doclist[0]
- # assert incoming doc is equal to the first sent doc
- self.assertEqual(doc1, doc2)
- # delete remote database
- db.delete_database()
- db.close()
- sol1.close()
- sol2.close()
+ return self._test_encrypted_sym_sync(doc_size=length, number_of_docs=1)
def test_sync_many_small_files(self):
"""
Test if Soledad can sync many smallfiles.
"""
- number_of_docs = 100
- self.startServer()
- # instantiate soledad and create a document
- sol1 = self._soledad_instance(
- # token is verified in test_target.make_token_soledad_app
- auth_token='auth-token'
- )
- _, doclist = sol1.get_all_docs()
- self.assertEqual([], doclist)
- # create many small files
- for i in range(0, number_of_docs):
- sol1.create_doc(json.loads(simple_doc))
- # ensure remote db exists before syncing
- db = CouchDatabase.open_database(
- urljoin(self._couch_url, 'user-user-uuid'),
- create=True,
- ensure_ddocs=True)
- # sync with server
- sol1._server_url = self.getURL()
- sol1.sync()
- # instantiate soledad with empty db, but with same secrets path
- sol2 = self._soledad_instance(prefix='x', auth_token='auth-token')
- _, doclist = sol2.get_all_docs()
- self.assertEqual([], doclist)
- sol2._secrets_path = sol1.secrets_path
- sol2._load_secrets()
- sol2._set_secret_id(sol1._secret_id)
- # sync the new instance
- sol2._server_url = self.getURL()
- sol2.sync()
- _, doclist = sol2.get_all_docs()
- self.assertEqual(number_of_docs, len(doclist))
- # assert incoming docs are equal to sent docs
- for doc in doclist:
- self.assertEqual(sol1.get_doc(doc.doc_id), doc)
- # delete remote database
- db.delete_database()
- db.close()
- sol1.close()
- sol2.close()
-
+ return self._test_encrypted_sym_sync(doc_size=2, number_of_docs=100)
class LockResourceTestCase(
CouchDBTestCase, TestCaseWithServer):
@@ -552,15 +471,18 @@ class LockResourceTestCase(
def make_app_with_state(state):
return make_token_soledad_app(state)
- make_document_for_test = make_leap_document_for_test
+ make_document_for_test = make_soledad_document_for_test
- sync_target = token_leap_sync_target
+ sync_target = token_soledad_sync_target
def setUp(self):
- TestCaseWithServer.setUp(self)
+ # the order of the following initializations is crucial because of
+ # dependencies.
+ # XXX explain better
CouchDBTestCase.setUp(self)
- self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
self._couch_url = 'http://localhost:' + str(self.wrapper.port)
+ self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
+ TestCaseWithServer.setUp(self)
# create the databases
CouchDatabase.open_database(
urljoin(self._couch_url, 'shared'),
@@ -570,18 +492,17 @@ class LockResourceTestCase(
urljoin(self._couch_url, 'tokens'),
create=True,
ensure_ddocs=True)
- self._state = CouchServerState(
- self._couch_url, 'shared', 'tokens')
+ self._state = CouchServerState(self._couch_url)
def tearDown(self):
- CouchDBTestCase.tearDown(self)
- TestCaseWithServer.tearDown(self)
# delete remote database
db = CouchDatabase.open_database(
urljoin(self._couch_url, 'shared'),
create=True,
ensure_ddocs=True)
db.delete_database()
+ CouchDBTestCase.tearDown(self)
+ TestCaseWithServer.tearDown(self)
def test__try_obtain_filesystem_lock(self):
responder = mock.Mock()
diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py
index 11e43423..1cd74dad 100644
--- a/common/src/leap/soledad/common/tests/test_soledad.py
+++ b/common/src/leap/soledad/common/tests/test_soledad.py
@@ -20,46 +20,43 @@ Tests for general Soledad functionality.
import os
from mock import Mock
-
-from leap.common.events import events_pb2 as proto
-from leap.soledad.common.tests import (
+from leap.common.events import catalog
+from leap.soledad.common.tests.util import (
BaseSoledadTest,
ADDRESS,
)
from leap import soledad
from leap.soledad.common.document import SoledadDocument
-from leap.soledad.common.crypto import WrongMac
-from leap.soledad.client import Soledad, PassphraseTooShort
-from leap.soledad.client.crypto import SoledadCrypto
+from leap.soledad.common.crypto import WrongMacError
+from leap.soledad.client import Soledad
+from leap.soledad.client.adbapi import U1DBConnectionPool
+from leap.soledad.client.secrets import PassphraseTooShort
from leap.soledad.client.shared_db import SoledadSharedDatabase
-from leap.soledad.client.target import SoledadSyncTarget
class AuxMethodsTestCase(BaseSoledadTest):
def test__init_dirs(self):
sol = self._soledad_instance(prefix='_init_dirs')
- sol._init_dirs()
local_db_dir = os.path.dirname(sol.local_db_path)
- secrets_path = os.path.dirname(sol.secrets_path)
+ secrets_path = os.path.dirname(sol.secrets.secrets_path)
self.assertTrue(os.path.isdir(local_db_dir))
self.assertTrue(os.path.isdir(secrets_path))
- sol.close()
- def test__init_db(self):
- sol = self._soledad_instance()
- sol._init_dirs()
- sol._crypto = SoledadCrypto(sol)
- #self._soledad._gpg.import_keys(PUBLIC_KEY)
- if not sol._has_secret():
- sol._gen_secret()
- sol._load_secrets()
- sol._init_db()
- from leap.soledad.client.sqlcipher import SQLCipherDatabase
- self.assertIsInstance(sol._db, SQLCipherDatabase)
+ def _close_soledad(results):
+ sol.close()
+
+ d = sol.create_doc({})
+ d.addCallback(_close_soledad)
+ return d
+
+ def test__init_u1db_sqlcipher_backend(self):
+ sol = self._soledad_instance(prefix='_init_db')
+ self.assertIsInstance(sol._dbpool, U1DBConnectionPool)
+ self.assertTrue(os.path.isfile(sol.local_db_path))
sol.close()
- def test__init_config_defaults(self):
+ def test__init_config_with_defaults(self):
"""
Test if configuration defaults point to the correct place.
"""
@@ -69,17 +66,15 @@ class AuxMethodsTestCase(BaseSoledadTest):
def __init__(self):
pass
- # instantiate without initializing so we just test _init_config()
+ # instantiate without initializing so we just test
+ # _init_config_with_defaults()
sol = SoledadMock()
- Soledad._init_config(sol, None, None, '')
- # assert value of secrets_path
- self.assertEquals(
- os.path.join(
- sol.DEFAULT_PREFIX, Soledad.STORAGE_SECRETS_FILE_NAME),
- sol.secrets_path)
+ sol._passphrase = u''
+ sol._server_url = ''
+ sol._init_config_with_defaults()
# assert value of local_db_path
self.assertEquals(
- os.path.join(sol.DEFAULT_PREFIX, 'soledad.u1db'),
+ os.path.join(sol.default_prefix, 'soledad.u1db'),
sol.local_db_path)
def test__init_config_from_params(self):
@@ -95,43 +90,56 @@ class AuxMethodsTestCase(BaseSoledadTest):
cert_file=None)
self.assertEqual(
os.path.join(self.tempdir, 'value_3'),
- sol.secrets_path)
+ sol.secrets.secrets_path)
self.assertEqual(
os.path.join(self.tempdir, 'value_2'),
sol.local_db_path)
- self.assertEqual('value_1', sol.server_url)
+ self.assertEqual('value_1', sol._server_url)
sol.close()
def test_change_passphrase(self):
"""
Test if passphrase can be changed.
"""
+ prefix = '_change_passphrase'
sol = self._soledad_instance(
'leap@leap.se',
passphrase=u'123',
- prefix=self.rand_prefix,
+ prefix=prefix,
)
- doc = sol.create_doc({'simple': 'doc'})
- doc_id = doc.doc_id
- # change the passphrase
- sol.change_passphrase(u'654321')
- sol.close()
-
- self.assertRaises(
- WrongMac,
- self._soledad_instance, 'leap@leap.se',
- passphrase=u'123',
- prefix=self.rand_prefix)
-
- # use new passphrase and retrieve doc
- sol2 = self._soledad_instance(
- 'leap@leap.se',
- passphrase=u'654321',
- prefix=self.rand_prefix)
- doc2 = sol2.get_doc(doc_id)
- self.assertEqual(doc, doc2)
- sol2.close()
+ def _change_passphrase(doc1):
+ self._doc1 = doc1
+ sol.change_passphrase(u'654321')
+ sol.close()
+
+ def _assert_wrong_password_raises(results):
+ self.assertRaises(
+ WrongMacError,
+ self._soledad_instance, 'leap@leap.se',
+ passphrase=u'123',
+ prefix=prefix)
+
+ def _instantiate_with_new_passphrase(results):
+ sol2 = self._soledad_instance(
+ 'leap@leap.se',
+ passphrase=u'654321',
+ prefix=prefix)
+ self._sol2 = sol2
+ return sol2.get_doc(self._doc1.doc_id)
+
+ def _assert_docs_are_equal(doc2):
+ self.assertEqual(self._doc1, doc2)
+ self._sol2.close()
+
+ d = sol.create_doc({'simple': 'doc'})
+ d.addCallback(_change_passphrase)
+ d.addCallback(_assert_wrong_password_raises)
+ d.addCallback(_instantiate_with_new_passphrase)
+ d.addCallback(_assert_docs_are_equal)
+ d.addCallback(lambda _: sol.close())
+
+ return d
def test_change_passphrase_with_short_passphrase_raises(self):
"""
@@ -152,7 +160,7 @@ class AuxMethodsTestCase(BaseSoledadTest):
Assert passphrase getter works fine.
"""
sol = self._soledad_instance()
- self.assertEqual('123', sol.passphrase)
+ self.assertEqual('123', sol._passphrase)
sol.close()
@@ -174,10 +182,10 @@ class SoledadSharedDBTestCase(BaseSoledadTest):
"""
Ensure the shared db is queried with the correct doc_id.
"""
- doc_id = self._soledad._shared_db_doc_id()
- self._soledad._get_secrets_from_shared_db()
+ doc_id = self._soledad.secrets._shared_db_doc_id()
+ self._soledad.secrets._get_secrets_from_shared_db()
self.assertTrue(
- self._soledad._shared_db().get_doc.assert_called_with(
+ self._soledad.shared_db.get_doc.assert_called_with(
doc_id) is None,
'Wrong doc_id when fetching recovery document.')
@@ -185,14 +193,14 @@ class SoledadSharedDBTestCase(BaseSoledadTest):
"""
Ensure recovery document is put into shared recover db.
"""
- doc_id = self._soledad._shared_db_doc_id()
- self._soledad._put_secrets_in_shared_db()
+ doc_id = self._soledad.secrets._shared_db_doc_id()
+ self._soledad.secrets._put_secrets_in_shared_db()
self.assertTrue(
- self._soledad._shared_db().get_doc.assert_called_with(
+ self._soledad.shared_db.get_doc.assert_called_with(
doc_id) is None,
'Wrong doc_id when fetching recovery document.')
self.assertTrue(
- self._soledad._shared_db.put_doc.assert_called_with(
+ self._soledad.shared_db.put_doc.assert_called_with(
self._doc_put) is None,
'Wrong document when putting recovery document.')
self.assertTrue(
@@ -210,6 +218,7 @@ class SoledadSignalingTestCase(BaseSoledadTest):
def setUp(self):
# mock signaling
soledad.client.signal = Mock()
+ soledad.client.secrets.events.emit = Mock()
# run parent's setUp
BaseSoledadTest.setUp(self)
@@ -231,63 +240,63 @@ class SoledadSignalingTestCase(BaseSoledadTest):
- downloading keys / done downloading keys.
- uploading keys / done uploading keys.
"""
- soledad.client.signal.reset_mock()
+ soledad.client.secrets.events.emit.reset_mock()
# get a fresh instance so it emits all bootstrap signals
sol = self._soledad_instance(
secrets_path='alternative_stage3.json',
local_db_path='alternative_stage3.u1db')
# reverse call order so we can verify in the order the signals were
# expected
- soledad.client.signal.mock_calls.reverse()
- soledad.client.signal.call_args = \
- soledad.client.signal.call_args_list[0]
- soledad.client.signal.call_args_list.reverse()
+ soledad.client.secrets.events.emit.mock_calls.reverse()
+ soledad.client.secrets.events.emit.call_args = \
+ soledad.client.secrets.events.emit.call_args_list[0]
+ soledad.client.secrets.events.emit.call_args_list.reverse()
# downloading keys signals
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_DOWNLOADING_KEYS,
+ soledad.client.secrets.events.emit.assert_called_with(
+ catalog.SOLEDAD_DOWNLOADING_KEYS,
ADDRESS,
)
- self._pop_mock_call(soledad.client.signal)
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_DONE_DOWNLOADING_KEYS,
+ self._pop_mock_call(soledad.client.secrets.events.emit)
+ soledad.client.secrets.events.emit.assert_called_with(
+ catalog.SOLEDAD_DONE_DOWNLOADING_KEYS,
ADDRESS,
)
# creating keys signals
- self._pop_mock_call(soledad.client.signal)
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_CREATING_KEYS,
+ self._pop_mock_call(soledad.client.secrets.events.emit)
+ soledad.client.secrets.events.emit.assert_called_with(
+ catalog.SOLEDAD_CREATING_KEYS,
ADDRESS,
)
- self._pop_mock_call(soledad.client.signal)
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_DONE_CREATING_KEYS,
+ self._pop_mock_call(soledad.client.secrets.events.emit)
+ soledad.client.secrets.events.emit.assert_called_with(
+ catalog.SOLEDAD_DONE_CREATING_KEYS,
ADDRESS,
)
# downloading once more (inside _put_keys_in_shared_db)
- self._pop_mock_call(soledad.client.signal)
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_DOWNLOADING_KEYS,
+ self._pop_mock_call(soledad.client.secrets.events.emit)
+ soledad.client.secrets.events.emit.assert_called_with(
+ catalog.SOLEDAD_DOWNLOADING_KEYS,
ADDRESS,
)
- self._pop_mock_call(soledad.client.signal)
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_DONE_DOWNLOADING_KEYS,
+ self._pop_mock_call(soledad.client.secrets.events.emit)
+ soledad.client.secrets.events.emit.assert_called_with(
+ catalog.SOLEDAD_DONE_DOWNLOADING_KEYS,
ADDRESS,
)
# uploading keys signals
- self._pop_mock_call(soledad.client.signal)
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_UPLOADING_KEYS,
+ self._pop_mock_call(soledad.client.secrets.events.emit)
+ soledad.client.secrets.events.emit.assert_called_with(
+ catalog.SOLEDAD_UPLOADING_KEYS,
ADDRESS,
)
- self._pop_mock_call(soledad.client.signal)
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_DONE_UPLOADING_KEYS,
+ self._pop_mock_call(soledad.client.secrets.events.emit)
+ soledad.client.secrets.events.emit.assert_called_with(
+ catalog.SOLEDAD_DONE_UPLOADING_KEYS,
ADDRESS,
)
# assert db was locked and unlocked
- sol._shared_db.lock.assert_called_with()
- sol._shared_db.unlock.assert_called_with('atoken')
+ sol.shared_db.lock.assert_called_with()
+ sol.shared_db.unlock.assert_called_with('atoken')
sol.close()
def test_stage2_bootstrap_signals(self):
@@ -298,41 +307,31 @@ class SoledadSignalingTestCase(BaseSoledadTest):
# get existing instance so we have access to keys
sol = self._soledad_instance()
# create a document with secrets
- doc = SoledadDocument(doc_id=sol._shared_db_doc_id())
- doc.content = sol.export_recovery_document()
-
- class Stage2MockSharedDB(object):
-
- get_doc = Mock(return_value=doc)
- put_doc = Mock()
- lock = Mock(return_value=('atoken', 300))
- unlock = Mock()
-
- def __call__(self):
- return self
-
+ doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id())
+ doc.content = sol.secrets._export_recovery_document()
sol.close()
# reset mock
- soledad.client.signal.reset_mock()
+ soledad.client.secrets.events.emit.reset_mock()
# get a fresh instance so it emits all bootstrap signals
+ shared_db = self.get_default_shared_mock(get_doc_return_value=doc)
sol = self._soledad_instance(
secrets_path='alternative_stage2.json',
local_db_path='alternative_stage2.u1db',
- shared_db_class=Stage2MockSharedDB)
+ shared_db_class=shared_db)
# reverse call order so we can verify in the order the signals were
# expected
- soledad.client.signal.mock_calls.reverse()
- soledad.client.signal.call_args = \
- soledad.client.signal.call_args_list[0]
- soledad.client.signal.call_args_list.reverse()
+ soledad.client.secrets.events.emit.mock_calls.reverse()
+ soledad.client.secrets.events.emit.call_args = \
+ soledad.client.secrets.events.emit.call_args_list[0]
+ soledad.client.secrets.events.emit.call_args_list.reverse()
# assert download keys signals
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_DOWNLOADING_KEYS,
+ soledad.client.secrets.events.emit.assert_called_with(
+ catalog.SOLEDAD_DOWNLOADING_KEYS,
ADDRESS,
)
- self._pop_mock_call(soledad.client.signal)
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_DONE_DOWNLOADING_KEYS,
+ self._pop_mock_call(soledad.client.secrets.events.emit)
+ soledad.client.secrets.events.emit.assert_called_with(
+ catalog.SOLEDAD_DONE_DOWNLOADING_KEYS,
ADDRESS,
)
sol.close()
@@ -356,33 +355,17 @@ class SoledadSignalingTestCase(BaseSoledadTest):
sol = self._soledad_instance()
# mock the actual db sync so soledad does not try to connect to the
# server
- sol._db.sync = Mock()
- # do the sync
- sol.sync()
- # assert the signal has been emitted
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_DONE_DATA_SYNC,
- ADDRESS,
- )
- sol.close()
-
- def test_need_sync_signals(self):
- """
- Test Soledad emits SOLEDAD_CREATING_KEYS signal.
- """
- soledad.client.signal.reset_mock()
- sol = self._soledad_instance()
- # mock the sync target
- old_get_sync_info = SoledadSyncTarget.get_sync_info
- SoledadSyncTarget.get_sync_info = Mock(return_value=[0, 0, 0, 0, 2])
- # mock our generation so soledad thinks there's new data to sync
- sol._db._get_generation = Mock(return_value=1)
- # check for new data to sync
- sol.need_sync('http://provider/userdb')
- # assert the signal has been emitted
- soledad.client.signal.assert_called_with(
- proto.SOLEDAD_NEW_DATA_TO_SYNC,
- ADDRESS,
- )
- SoledadSyncTarget.get_sync_info = old_get_sync_info
- sol.close()
+ sol._dbsyncer.sync = Mock()
+
+ def _assert_done_data_sync_signal_emitted(results):
+ # assert the signal has been emitted
+ soledad.client.signal.assert_called_with(
+ catalog.SOLEDAD_DONE_DATA_SYNC,
+ ADDRESS,
+ )
+ sol.close()
+
+ # do the sync and assert signal was emitted
+ d = sol.sync()
+ d.addCallback(_assert_done_data_sync_signal_emitted)
+ return d
diff --git a/common/src/leap/soledad/common/tests/test_soledad_app.py b/common/src/leap/soledad/common/tests/test_soledad_app.py
new file mode 100644
index 00000000..6efae1d6
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_soledad_app.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+# test_soledad_app.py
+# Copyright (C) 2014 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Test ObjectStore and Couch backend bits.
+"""
+
+
+from testscenarios import TestWithScenarios
+
+from leap.soledad.common.tests.util import BaseSoledadTest
+from leap.soledad.common.tests.util import make_soledad_document_for_test
+from leap.soledad.common.tests.util import make_soledad_app
+from leap.soledad.common.tests.util import make_token_soledad_app
+from leap.soledad.common.tests.util import make_token_http_database_for_test
+from leap.soledad.common.tests.util import copy_token_http_database_for_test
+from leap.soledad.common.tests.u1db_tests import test_backends
+
+
+#-----------------------------------------------------------------------------
+# The following tests come from `u1db.tests.test_backends`.
+#-----------------------------------------------------------------------------
+
+LEAP_SCENARIOS = [
+ ('http', {
+ 'make_database_for_test': test_backends.make_http_database_for_test,
+ 'copy_database_for_test': test_backends.copy_http_database_for_test,
+ 'make_document_for_test': make_soledad_document_for_test,
+ 'make_app_with_state': make_soledad_app}),
+]
+
+
+class SoledadTests(
+ TestWithScenarios, test_backends.AllDatabaseTests, BaseSoledadTest):
+
+ scenarios = LEAP_SCENARIOS + [
+ ('token_http', {'make_database_for_test':
+ make_token_http_database_for_test,
+ 'copy_database_for_test':
+ copy_token_http_database_for_test,
+ 'make_document_for_test': make_soledad_document_for_test,
+ 'make_app_with_state': make_token_soledad_app,
+ })
+ ]
diff --git a/common/src/leap/soledad/common/tests/test_soledad_doc.py b/common/src/leap/soledad/common/tests/test_soledad_doc.py
index 0952de6d..4a67f80a 100644
--- a/common/src/leap/soledad/common/tests/test_soledad_doc.py
+++ b/common/src/leap/soledad/common/tests/test_soledad_doc.py
@@ -17,28 +17,30 @@
"""
Test Leap backend bits: soledad docs
"""
-from leap.soledad.common.tests import BaseSoledadTest
+from testscenarios import TestWithScenarios
+
from leap.soledad.common.tests.u1db_tests import test_document
-from leap.soledad.common.tests import u1db_tests as tests
-from leap.soledad.common.tests import test_sync_target as st
+from leap.soledad.common.tests.util import BaseSoledadTest
+from leap.soledad.common.tests.util import make_soledad_document_for_test
+
#-----------------------------------------------------------------------------
# The following tests come from `u1db.tests.test_document`.
#-----------------------------------------------------------------------------
-
-class TestSoledadDocument(test_document.TestDocument, BaseSoledadTest):
+class TestSoledadDocument(
+ TestWithScenarios,
+ test_document.TestDocument, BaseSoledadTest):
scenarios = ([(
'leap', {
- 'make_document_for_test': st.make_leap_document_for_test})])
+ 'make_document_for_test': make_soledad_document_for_test})])
-class TestSoledadPyDocument(test_document.TestPyDocument, BaseSoledadTest):
+class TestSoledadPyDocument(
+ TestWithScenarios,
+ test_document.TestPyDocument, BaseSoledadTest):
scenarios = ([(
'leap', {
- 'make_document_for_test': st.make_leap_document_for_test})])
-
-
-load_tests = tests.load_with_scenarios
+ 'make_document_for_test': make_soledad_document_for_test})])
diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py
index 595966ec..ceb095b8 100644
--- a/common/src/leap/soledad/common/tests/test_sqlcipher.py
+++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py
@@ -19,53 +19,49 @@ Test sqlcipher backend internals.
"""
import os
import time
-import simplejson as json
import threading
-
+import tempfile
+import shutil
from pysqlcipher import dbapi2
-from StringIO import StringIO
-from urlparse import urljoin
-
+from testscenarios import TestWithScenarios
# u1db stuff.
from u1db import (
errors,
query_parser,
- sync,
- vectorclock,
)
from u1db.backends.sqlite_backend import SQLitePartialExpandDatabase
# soledad stuff.
+from leap.soledad.common import soledad_assert
from leap.soledad.common.document import SoledadDocument
from leap.soledad.client.sqlcipher import (
SQLCipherDatabase,
+ SQLCipherOptions,
DatabaseIsNotEncrypted,
- open as u1db_open,
)
-from leap.soledad.client.target import SoledadSyncTarget
-from leap.soledad.common.crypto import ENC_SCHEME_KEY
-from leap.soledad.client.crypto import decrypt_doc_dict
# u1db tests stuff.
-from leap.common.testing.basetest import BaseLeapTest
-from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest
+from leap.soledad.common.tests import u1db_tests as tests
from leap.soledad.common.tests.u1db_tests import test_sqlite_backend
from leap.soledad.common.tests.u1db_tests import test_backends
from leap.soledad.common.tests.u1db_tests import test_open
-from leap.soledad.common.tests.u1db_tests import test_sync
from leap.soledad.common.tests.util import (
make_sqlcipher_database_for_test,
copy_sqlcipher_database_for_test,
- make_soledad_app,
- SoledadWithCouchServerMixin,
PASSWORD,
+ BaseSoledadTest,
)
+def sqlcipher_open(path, passphrase, create=True, document_factory=None):
+ return SQLCipherDatabase(
+ SQLCipherOptions(path, passphrase, create=create))
+
+
#-----------------------------------------------------------------------------
# The following tests come from `u1db.tests.test_common_backend`.
#-----------------------------------------------------------------------------
@@ -73,12 +69,13 @@ from leap.soledad.common.tests.util import (
class TestSQLCipherBackendImpl(tests.TestCase):
def test__allocate_doc_id(self):
- db = SQLCipherDatabase(':memory:', PASSWORD)
+ db = sqlcipher_open(':memory:', PASSWORD)
doc_id1 = db._allocate_doc_id()
self.assertTrue(doc_id1.startswith('D-'))
self.assertEqual(34, len(doc_id1))
int(doc_id1[len('D-'):], 16)
self.assertNotEqual(doc_id1, db._allocate_doc_id())
+ db.close()
#-----------------------------------------------------------------------------
@@ -96,43 +93,46 @@ SQLCIPHER_SCENARIOS = [
]
-class SQLCipherTests(test_backends.AllDatabaseTests):
+class SQLCipherTests(TestWithScenarios, test_backends.AllDatabaseTests):
scenarios = SQLCIPHER_SCENARIOS
-class SQLCipherDatabaseTests(test_backends.LocalDatabaseTests):
+class SQLCipherDatabaseTests(TestWithScenarios, test_backends.LocalDatabaseTests):
scenarios = SQLCIPHER_SCENARIOS
class SQLCipherValidateGenNTransIdTests(
+ TestWithScenarios,
test_backends.LocalDatabaseValidateGenNTransIdTests):
scenarios = SQLCIPHER_SCENARIOS
class SQLCipherValidateSourceGenTests(
+ TestWithScenarios,
test_backends.LocalDatabaseValidateSourceGenTests):
scenarios = SQLCIPHER_SCENARIOS
class SQLCipherWithConflictsTests(
+ TestWithScenarios,
test_backends.LocalDatabaseWithConflictsTests):
scenarios = SQLCIPHER_SCENARIOS
-class SQLCipherIndexTests(test_backends.DatabaseIndexTests):
+class SQLCipherIndexTests(
+ TestWithScenarios, test_backends.DatabaseIndexTests):
scenarios = SQLCIPHER_SCENARIOS
-load_tests = tests.load_with_scenarios
-
-
#-----------------------------------------------------------------------------
# The following tests come from `u1db.tests.test_sqlite_backend`.
#-----------------------------------------------------------------------------
-class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase):
+class TestSQLCipherDatabase(TestWithScenarios, test_sqlite_backend.TestSQLiteDatabase):
def test_atomic_initialize(self):
+ # This test was modified to ensure that db2.close() is called within
+ # the thread that created the database.
tmpdir = self.createTempDir()
dbname = os.path.join(tmpdir, 'atomic.db')
@@ -144,7 +144,9 @@ class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase):
def __init__(self, dbname, ntry):
self._try = ntry
self._is_initialized_invocations = 0
- SQLCipherDatabase.__init__(self, dbname, PASSWORD)
+ SQLCipherDatabase.__init__(
+ self,
+ SQLCipherOptions(dbname, PASSWORD))
def _is_initialized(self, c):
res = \
@@ -157,23 +159,25 @@ class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase):
time.sleep(0.05)
return res
- outcome2 = []
+ class SecondTry(threading.Thread):
+
+ outcome2 = []
- def second_try():
- try:
- db2 = SQLCipherDatabaseTesting(dbname, 2)
- except Exception, e:
- outcome2.append(e)
- else:
- outcome2.append(db2)
+ def run(self):
+ try:
+ db2 = SQLCipherDatabaseTesting(dbname, 2)
+ except Exception, e:
+ SecondTry.outcome2.append(e)
+ else:
+ SecondTry.outcome2.append(db2)
- t2 = threading.Thread(target=second_try)
+ t2 = SecondTry()
db1 = SQLCipherDatabaseTesting(dbname, 1)
t2.join()
- self.assertIsInstance(outcome2[0], SQLCipherDatabaseTesting)
- db2 = outcome2[0]
- self.assertTrue(db2._is_initialized(db1._get_sqlite_handle().cursor()))
+ self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting)
+ self.assertTrue(db1._is_initialized(db1._get_sqlite_handle().cursor()))
+ db1.close()
class TestAlternativeDocument(SoledadDocument):
@@ -189,23 +193,23 @@ class TestSQLCipherPartialExpandDatabase(
def setUp(self):
test_sqlite_backend.TestSQLitePartialExpandDatabase.setUp(self)
- self.db = SQLCipherDatabase(':memory:', PASSWORD)
- self.db._set_replica_uid('test')
+ self.db = sqlcipher_open(':memory:', PASSWORD)
+
+ def tearDown(self):
+ self.db.close()
+ test_sqlite_backend.TestSQLitePartialExpandDatabase.tearDown(self)
def test_default_replica_uid(self):
- self.db = SQLCipherDatabase(':memory:', PASSWORD)
self.assertIsNot(None, self.db._replica_uid)
self.assertEqual(32, len(self.db._replica_uid))
int(self.db._replica_uid, 16)
def test__parse_index(self):
- self.db = SQLCipherDatabase(':memory:', PASSWORD)
g = self.db._parse_index_definition('fieldname')
self.assertIsInstance(g, query_parser.ExtractField)
self.assertEqual(['fieldname'], g.field)
def test__update_indexes(self):
- self.db = SQLCipherDatabase(':memory:', PASSWORD)
g = self.db._parse_index_definition('fieldname')
c = self.db._get_sqlite_handle().cursor()
self.db._update_indexes('doc-id', {'fieldname': 'val'},
@@ -216,7 +220,6 @@ class TestSQLCipherPartialExpandDatabase(
def test__set_replica_uid(self):
# Start from scratch, so that replica_uid isn't set.
- self.db = SQLCipherDatabase(':memory:', PASSWORD)
self.assertIsNot(None, self.db._real_replica_uid)
self.assertIsNot(None, self.db._replica_uid)
self.db._set_replica_uid('foo')
@@ -229,98 +232,67 @@ class TestSQLCipherPartialExpandDatabase(
self.assertEqual('foo', self.db._replica_uid)
def test__open_database(self):
- temp_dir = self.createTempDir(prefix='u1db-test-')
- path = temp_dir + '/test.sqlite'
- SQLCipherDatabase(path, PASSWORD)
- db2 = SQLCipherDatabase._open_database(path, PASSWORD)
- self.assertIsInstance(db2, SQLCipherDatabase)
+ # SQLCipherDatabase has no _open_database() method, so we just pass
+ # (and test for the same funcionality on test_open_database_existing()
+ # below).
+ pass
def test__open_database_with_factory(self):
- temp_dir = self.createTempDir(prefix='u1db-test-')
- path = temp_dir + '/test.sqlite'
- SQLCipherDatabase(path, PASSWORD)
- db2 = SQLCipherDatabase._open_database(
- path, PASSWORD,
- document_factory=TestAlternativeDocument)
- doc = db2.create_doc({})
- self.assertTrue(isinstance(doc, SoledadDocument))
+ # SQLCipherDatabase has no _open_database() method.
+ pass
def test__open_database_non_existent(self):
temp_dir = self.createTempDir(prefix='u1db-test-')
path = temp_dir + '/non-existent.sqlite'
self.assertRaises(errors.DatabaseDoesNotExist,
- SQLCipherDatabase._open_database,
- path, PASSWORD)
+ sqlcipher_open,
+ path, PASSWORD, create=False)
def test__open_database_during_init(self):
- temp_dir = self.createTempDir(prefix='u1db-test-')
- path = temp_dir + '/initialised.db'
- db = SQLCipherDatabase.__new__(
- SQLCipherDatabase)
- db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed
- db._syncers = {}
- c = db._db_handle.cursor()
- c.execute('PRAGMA key="%s"' % PASSWORD)
- self.addCleanup(db.close)
- observed = []
-
- class SQLiteDatabaseTesting(SQLCipherDatabase):
- WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL = 0.1
-
- @classmethod
- def _which_index_storage(cls, c):
- res = SQLCipherDatabase._which_index_storage(c)
- db._ensure_schema() # init db
- observed.append(res[0])
- return res
-
- db2 = SQLiteDatabaseTesting._open_database(path, PASSWORD)
- self.addCleanup(db2.close)
- self.assertIsInstance(db2, SQLCipherDatabase)
- self.assertEqual(
- [None,
- SQLCipherDatabase._index_storage_value],
- observed)
+ # The purpose of this test is to ensure that _open_database() parallel
+ # db initialization behaviour is correct. As SQLCipherDatabase does
+ # not have an _open_database() method, we just do not implement this
+ # test.
+ pass
def test__open_database_invalid(self):
- class SQLiteDatabaseTesting(SQLCipherDatabase):
- WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL = 0.1
+ # This test was modified to ensure that an empty database file will
+ # raise a DatabaseIsNotEncrypted exception instead of a
+ # dbapi2.OperationalError exception.
temp_dir = self.createTempDir(prefix='u1db-test-')
path1 = temp_dir + '/invalid1.db'
with open(path1, 'wb') as f:
f.write("")
- self.assertRaises(dbapi2.OperationalError,
- SQLiteDatabaseTesting._open_database, path1,
+ self.assertRaises(DatabaseIsNotEncrypted,
+ sqlcipher_open, path1,
PASSWORD)
with open(path1, 'wb') as f:
f.write("invalid")
self.assertRaises(dbapi2.DatabaseError,
- SQLiteDatabaseTesting._open_database, path1,
+ sqlcipher_open, path1,
PASSWORD)
def test_open_database_existing(self):
- temp_dir = self.createTempDir(prefix='u1db-test-')
- path = temp_dir + '/existing.sqlite'
- SQLCipherDatabase(path, PASSWORD)
- db2 = SQLCipherDatabase.open_database(path, PASSWORD, create=False)
- self.assertIsInstance(db2, SQLCipherDatabase)
+ # In the context of SQLCipherDatabase, where no _open_database()
+ # method exists and thus there's no call to _which_index_storage(),
+ # this test tests for the same functionality as
+ # test_open_database_create() below. So, we just pass.
+ pass
def test_open_database_with_factory(self):
- temp_dir = self.createTempDir(prefix='u1db-test-')
- path = temp_dir + '/existing.sqlite'
- SQLCipherDatabase(path, PASSWORD)
- db2 = SQLCipherDatabase.open_database(
- path, PASSWORD, create=False,
- document_factory=TestAlternativeDocument)
- doc = db2.create_doc({})
- self.assertTrue(isinstance(doc, SoledadDocument))
+ # SQLCipherDatabase's constructor has no factory parameter.
+ pass
def test_open_database_create(self):
+ # SQLCipherDatabas has no open_database() method, so we just test for
+ # the actual database constructor effects.
temp_dir = self.createTempDir(prefix='u1db-test-')
path = temp_dir + '/new.sqlite'
- SQLCipherDatabase.open_database(path, PASSWORD, create=True)
- db2 = SQLCipherDatabase.open_database(path, PASSWORD, create=False)
+ db1 = sqlcipher_open(path, PASSWORD, create=True)
+ db2 = sqlcipher_open(path, PASSWORD, create=False)
self.assertIsInstance(db2, SQLCipherDatabase)
+ db1.close()
+ db2.close()
def test_create_database_initializes_schema(self):
# This test had to be cloned because our implementation of SQLCipher
@@ -331,7 +303,8 @@ class TestSQLCipherPartialExpandDatabase(
c = raw_db.cursor()
c.execute("SELECT * FROM u1db_config")
config = dict([(r[0], r[1]) for r in c.fetchall()])
- self.assertEqual({'sql_schema': '0', 'replica_uid': 'test',
+ replica_uid = self.db._replica_uid
+ self.assertEqual({'sql_schema': '0', 'replica_uid': replica_uid,
'index_storage': 'expand referenced encrypted'},
config)
@@ -353,394 +326,52 @@ class TestSQLCipherPartialExpandDatabase(
# The following tests come from `u1db.tests.test_open`.
#-----------------------------------------------------------------------------
+
class SQLCipherOpen(test_open.TestU1DBOpen):
def test_open_no_create(self):
self.assertRaises(errors.DatabaseDoesNotExist,
- u1db_open, self.db_path,
- password=PASSWORD,
+ sqlcipher_open, self.db_path,
+ PASSWORD,
create=False)
self.assertFalse(os.path.exists(self.db_path))
def test_open_create(self):
- db = u1db_open(self.db_path, password=PASSWORD, create=True)
+ db = sqlcipher_open(self.db_path, PASSWORD, create=True)
self.addCleanup(db.close)
self.assertTrue(os.path.exists(self.db_path))
self.assertIsInstance(db, SQLCipherDatabase)
def test_open_with_factory(self):
- db = u1db_open(self.db_path, password=PASSWORD, create=True,
+ db = sqlcipher_open(self.db_path, PASSWORD, create=True,
document_factory=TestAlternativeDocument)
self.addCleanup(db.close)
doc = db.create_doc({})
self.assertTrue(isinstance(doc, SoledadDocument))
def test_open_existing(self):
- db = SQLCipherDatabase(self.db_path, PASSWORD)
+ db = sqlcipher_open(self.db_path, PASSWORD)
self.addCleanup(db.close)
doc = db.create_doc_from_json(tests.simple_doc)
# Even though create=True, we shouldn't wipe the db
- db2 = u1db_open(self.db_path, password=PASSWORD, create=True)
+ db2 = sqlcipher_open(self.db_path, PASSWORD, create=True)
self.addCleanup(db2.close)
doc2 = db2.get_doc(doc.doc_id)
self.assertEqual(doc, doc2)
def test_open_existing_no_create(self):
- db = SQLCipherDatabase(self.db_path, PASSWORD)
+ db = sqlcipher_open(self.db_path, PASSWORD)
self.addCleanup(db.close)
- db2 = u1db_open(self.db_path, password=PASSWORD, create=False)
+ db2 = sqlcipher_open(self.db_path, PASSWORD, create=False)
self.addCleanup(db2.close)
self.assertIsInstance(db2, SQLCipherDatabase)
#-----------------------------------------------------------------------------
-# The following tests come from `u1db.tests.test_sync`.
-#-----------------------------------------------------------------------------
-
-sync_scenarios = []
-for name, scenario in SQLCIPHER_SCENARIOS:
- scenario = dict(scenario)
- scenario['do_sync'] = test_sync.sync_via_synchronizer
- sync_scenarios.append((name, scenario))
- scenario = dict(scenario)
-
-
-def sync_via_synchronizer_and_leap(test, db_source, db_target,
- trace_hook=None, trace_hook_shallow=None):
- if trace_hook:
- test.skipTest("full trace hook unsupported over http")
- path = test._http_at[db_target]
- target = SoledadSyncTarget.connect(
- test.getURL(path), test._soledad._crypto)
- target.set_token_credentials('user-uuid', 'auth-token')
- if trace_hook_shallow:
- target._set_trace_hook_shallow(trace_hook_shallow)
- return sync.Synchronizer(db_source, target).sync()
-
-
-sync_scenarios.append(('pyleap', {
- 'make_database_for_test': test_sync.make_database_for_http_test,
- 'copy_database_for_test': test_sync.copy_database_for_http_test,
- 'make_document_for_test': make_document_for_test,
- 'make_app_with_state': tests.test_remote_sync_target.make_http_app,
- 'do_sync': test_sync.sync_via_synchronizer,
-}))
-
-
-class SQLCipherDatabaseSyncTests(
- test_sync.DatabaseSyncTests, BaseSoledadTest):
- """
- Test for succesfull sync between SQLCipher and LeapBackend.
-
- Some of the tests in this class had to be adapted because the remote
- backend always receive encrypted content, and so it can not rely on
- document's content comparison to try to autoresolve conflicts.
- """
-
- scenarios = sync_scenarios
-
- def setUp(self):
- test_sync.DatabaseSyncTests.setUp(self)
-
- def tearDown(self):
- test_sync.DatabaseSyncTests.tearDown(self)
-
- def test_sync_autoresolves(self):
- """
- Test for sync autoresolve remote.
-
- This test was adapted because the remote database receives encrypted
- content and so it can't compare documents contents to autoresolve.
- """
- # The remote database can't autoresolve conflicts based on magic
- # content convergence, so we modify this test to leave the possibility
- # of the remode document ending up in conflicted state.
- self.db1 = self.create_database('test1', 'source')
- self.db2 = self.create_database('test2', 'target')
- doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc')
- rev1 = doc1.rev
- doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc')
- rev2 = doc2.rev
- self.sync(self.db1, self.db2)
- doc = self.db1.get_doc('doc')
- self.assertFalse(doc.has_conflicts)
- # if remote content is in conflicted state, then document revisions
- # will be different.
- #self.assertEqual(doc.rev, self.db2.get_doc('doc').rev)
- v = vectorclock.VectorClockRev(doc.rev)
- self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1)))
- self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2)))
-
- def test_sync_autoresolves_moar(self):
- """
- Test for sync autoresolve local.
-
- This test was adapted to decrypt remote content before assert.
- """
- # here we test that when a database that has a conflicted document is
- # the source of a sync, and the target database has a revision of the
- # conflicted document that is newer than the source database's, and
- # that target's database's document's content is the same as the
- # source's document's conflict's, the source's document's conflict gets
- # autoresolved, and the source's document's revision bumped.
- #
- # idea is as follows:
- # A B
- # a1 -
- # `------->
- # a1 a1
- # v v
- # a2 a1b1
- # `------->
- # a1b1+a2 a1b1
- # v
- # a1b1+a2 a1b2 (a1b2 has same content as a2)
- # `------->
- # a3b2 a1b2 (autoresolved)
- # `------->
- # a3b2 a3b2
- self.db1 = self.create_database('test1', 'source')
- self.db2 = self.create_database('test2', 'target')
- self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc')
- self.sync(self.db1, self.db2)
- for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]:
- doc = db.get_doc('doc')
- doc.set_json(content)
- db.put_doc(doc)
- self.sync(self.db1, self.db2)
- # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict
- doc = self.db1.get_doc('doc')
- rev1 = doc.rev
- self.assertTrue(doc.has_conflicts)
- # set db2 to have a doc of {} (same as db1 before the conflict)
- doc = self.db2.get_doc('doc')
- doc.set_json('{}')
- self.db2.put_doc(doc)
- rev2 = doc.rev
- # sync it across
- self.sync(self.db1, self.db2)
- # tadaa!
- doc = self.db1.get_doc('doc')
- self.assertFalse(doc.has_conflicts)
- vec1 = vectorclock.VectorClockRev(rev1)
- vec2 = vectorclock.VectorClockRev(rev2)
- vec3 = vectorclock.VectorClockRev(doc.rev)
- self.assertTrue(vec3.is_newer(vec1))
- self.assertTrue(vec3.is_newer(vec2))
- # because the conflict is on the source, sync it another time
- self.sync(self.db1, self.db2)
- # make sure db2 now has the exact same thing
- doc1 = self.db1.get_doc('doc')
- self.assertGetEncryptedDoc(
- self.db2,
- doc1.doc_id, doc1.rev, doc1.get_json(), False)
-
- def test_sync_autoresolves_moar_backwards(self):
- # here we would test that when a database that has a conflicted
- # document is the target of a sync, and the source database has a
- # revision of the conflicted document that is newer than the target
- # database's, and that source's database's document's content is the
- # same as the target's document's conflict's, the target's document's
- # conflict gets autoresolved, and the document's revision bumped.
- #
- # Despite that, in Soledad we suppose that the server never syncs, so
- # it never has conflicted documents. Also, if it had, convergence
- # would not be possible by checking document's contents because they
- # would be encrypted in server.
- #
- # Therefore we suppress this test.
- pass
-
- def test_sync_autoresolves_moar_backwards_three(self):
- # here we would test that when a database that has a conflicted
- # document is the target of a sync, and the source database has a
- # revision of the conflicted document that is newer than the target
- # database's, and that source's database's document's content is the
- # same as the target's document's conflict's, the target's document's
- # conflict gets autoresolved, and the document's revision bumped.
- #
- # We use the same reasoning from the last test to suppress this one.
- pass
-
- def test_sync_propagates_resolution(self):
- """
- Test if synchronization propagates resolution.
-
- This test was adapted to decrypt remote content before assert.
- """
- self.db1 = self.create_database('test1', 'both')
- self.db2 = self.create_database('test2', 'both')
- doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc')
- db3 = self.create_database('test3', 'both')
- self.sync(self.db2, self.db1)
- self.assertEqual(
- self.db1._get_generation_info(),
- self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid))
- self.assertEqual(
- self.db2._get_generation_info(),
- self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid))
- self.sync(db3, self.db1)
- # update on 2
- doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}')
- self.db2.put_doc(doc2)
- self.sync(self.db2, db3)
- self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev)
- # update on 1
- doc1.set_json('{"a": 3}')
- self.db1.put_doc(doc1)
- # conflicts
- self.sync(self.db2, self.db1)
- self.sync(db3, self.db1)
- self.assertTrue(self.db2.get_doc('the-doc').has_conflicts)
- self.assertTrue(db3.get_doc('the-doc').has_conflicts)
- # resolve
- conflicts = self.db2.get_doc_conflicts('the-doc')
- doc4 = self.make_document('the-doc', None, '{"a": 4}')
- revs = [doc.rev for doc in conflicts]
- self.db2.resolve_doc(doc4, revs)
- doc2 = self.db2.get_doc('the-doc')
- self.assertEqual(doc4.get_json(), doc2.get_json())
- self.assertFalse(doc2.has_conflicts)
- self.sync(self.db2, db3)
- doc3 = db3.get_doc('the-doc')
- if ENC_SCHEME_KEY in doc3.content:
- _crypto = self._soledad._crypto
- key = _crypto.doc_passphrase(doc3.doc_id)
- secret = _crypto.secret
- doc3.set_json(decrypt_doc_dict(
- doc3.content,
- doc3.doc_id, doc3.rev, key, secret))
- self.assertEqual(doc4.get_json(), doc3.get_json())
- self.assertFalse(doc3.has_conflicts)
-
- def test_sync_puts_changes(self):
- """
- Test if sync puts changes in remote replica.
-
- This test was adapted to decrypt remote content before assert.
- """
- self.db1 = self.create_database('test1', 'source')
- self.db2 = self.create_database('test2', 'target')
- doc = self.db1.create_doc_from_json(tests.simple_doc)
- self.assertEqual(1, self.sync(self.db1, self.db2))
- self.assertGetEncryptedDoc(
- self.db2, doc.doc_id, doc.rev, tests.simple_doc, False)
- self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0])
- self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0])
- self.assertLastExchangeLog(
- self.db2,
- {'receive': {'docs': [(doc.doc_id, doc.rev)],
- 'source_uid': 'test1',
- 'source_gen': 1, 'last_known_gen': 0},
- 'return': {'docs': [], 'last_gen': 1}})
-
-
-def _make_local_db_and_token_http_target(test, path='test'):
- test.startServer()
- db = test.request_state._create_database(os.path.basename(path))
- st = SoledadSyncTarget.connect(
- test.getURL(path), crypto=test._soledad._crypto)
- st.set_token_credentials('user-uuid', 'auth-token')
- return db, st
-
-
-target_scenarios = [
- ('leap', {
- 'create_db_and_target': _make_local_db_and_token_http_target,
-# 'make_app_with_state': tests.test_remote_sync_target.make_http_app,
- 'make_app_with_state': make_soledad_app,
- 'do_sync': test_sync.sync_via_synchronizer}),
-]
-
-
-class SQLCipherSyncTargetTests(
- SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests):
-
- scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS,
- target_scenarios))
-
- whitebox = False
-
- def setUp(self):
- self.main_test_class = test_sync.DatabaseSyncTargetTests
- SoledadWithCouchServerMixin.setUp(self)
-
- def test_sync_exchange(self):
- """
- Modified to account for possibly receiving encrypted documents from
- sever-side.
- """
- docs_by_gen = [
- (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10,
- 'T-sid')]
- new_gen, trans_id = self.st.sync_exchange(
- docs_by_gen, 'replica', last_known_generation=0,
- last_known_trans_id=None, return_doc_cb=self.receive_doc)
- self.assertGetEncryptedDoc(
- self.db, 'doc-id', 'replica:1', tests.simple_doc, False)
- self.assertTransactionLog(['doc-id'], self.db)
- last_trans_id = self.getLastTransId(self.db)
- self.assertEqual(([], 1, last_trans_id),
- (self.other_changes, new_gen, last_trans_id))
- self.assertEqual(10, self.st.get_sync_info('replica')[3])
-
- def test_sync_exchange_push_many(self):
- """
- Modified to account for possibly receiving encrypted documents from
- sever-side.
- """
- docs_by_gen = [
- (self.make_document(
- 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'),
- (self.make_document('doc-id2', 'replica:1', tests.nested_doc), 11,
- 'T-2')]
- new_gen, trans_id = self.st.sync_exchange(
- docs_by_gen, 'replica', last_known_generation=0,
- last_known_trans_id=None, return_doc_cb=self.receive_doc)
- self.assertGetEncryptedDoc(
- self.db, 'doc-id', 'replica:1', tests.simple_doc, False)
- self.assertGetEncryptedDoc(
- self.db, 'doc-id2', 'replica:1', tests.nested_doc, False)
- self.assertTransactionLog(['doc-id', 'doc-id2'], self.db)
- last_trans_id = self.getLastTransId(self.db)
- self.assertEqual(([], 2, last_trans_id),
- (self.other_changes, new_gen, trans_id))
- self.assertEqual(11, self.st.get_sync_info('replica')[3])
-
- def test_sync_exchange_returns_many_new_docs(self):
- """
- Modified to account for JSON serialization differences.
- """
- doc = self.db.create_doc_from_json(tests.simple_doc)
- doc2 = self.db.create_doc_from_json(tests.nested_doc)
- self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)
- new_gen, _ = self.st.sync_exchange(
- [], 'other-replica', last_known_generation=0,
- last_known_trans_id=None, return_doc_cb=self.receive_doc)
- self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)
- self.assertEqual(2, new_gen)
- self.assertEqual(
- [(doc.doc_id, doc.rev, 1),
- (doc2.doc_id, doc2.rev, 2)],
- [c[:2] + c[3:4] for c in self.other_changes])
- self.assertEqual(
- json.dumps(tests.simple_doc),
- json.dumps(self.other_changes[0][2]))
- self.assertEqual(
- json.loads(tests.nested_doc),
- json.loads(self.other_changes[1][2]))
- if self.whitebox:
- self.assertEqual(
- self.db._last_exchange_log['return'],
- {'last_gen': 2, 'docs':
- [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]})
-
-
-#-----------------------------------------------------------------------------
# Tests for actual encryption of the database
#-----------------------------------------------------------------------------
-class SQLCipherEncryptionTest(BaseLeapTest):
+class SQLCipherEncryptionTest(BaseSoledadTest):
"""
Tests to guarantee SQLCipher is indeed encrypting data when storing.
"""
@@ -751,17 +382,43 @@ class SQLCipherEncryptionTest(BaseLeapTest):
os.unlink(dbfile)
def setUp(self):
+ # the following come from BaseLeapTest.setUpClass, because
+ # twisted.trial doesn't support such class methods for setting up
+ # test classes.
+ self.old_path = os.environ['PATH']
+ self.old_home = os.environ['HOME']
+ self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
+ self.home = self.tempdir
+ bin_tdir = os.path.join(
+ self.tempdir,
+ 'bin')
+ os.environ["PATH"] = bin_tdir
+ os.environ["HOME"] = self.tempdir
+ # this is our own stuff
self.DB_FILE = os.path.join(self.tempdir, 'test.db')
self._delete_dbfiles()
def tearDown(self):
self._delete_dbfiles()
+ # the following come from BaseLeapTest.tearDownClass, because
+ # twisted.trial doesn't support such class methods for tearing down
+ # test classes.
+ os.environ["PATH"] = self.old_path
+ os.environ["HOME"] = self.old_home
+ # safety check! please do not wipe my home...
+ # XXX needs to adapt to non-linuces
+ soledad_assert(
+ self.tempdir.startswith('/tmp/leap_tests-') or
+ self.tempdir.startswith('/var/folder'),
+ "beware! tried to remove a dir which does not "
+ "live in temporal folder!")
+ shutil.rmtree(self.tempdir)
def test_try_to_open_encrypted_db_with_sqlite_backend(self):
"""
SQLite backend should not succeed to open SQLCipher databases.
"""
- db = SQLCipherDatabase(self.DB_FILE, PASSWORD)
+ db = sqlcipher_open(self.DB_FILE, PASSWORD)
doc = db.create_doc_from_json(tests.simple_doc)
db.close()
try:
@@ -774,10 +431,11 @@ class SQLCipherEncryptionTest(BaseLeapTest):
# at this point we know that the regular U1DB sqlcipher backend
# did not succeed on opening the database, so it was indeed
# encrypted.
- db = SQLCipherDatabase(self.DB_FILE, PASSWORD)
+ db = sqlcipher_open(self.DB_FILE, PASSWORD)
doc = db.get_doc(doc.doc_id)
self.assertEqual(tests.simple_doc, doc.get_json(),
'decrypted content mismatch')
+ db.close()
def test_try_to_open_raw_db_with_sqlcipher_backend(self):
"""
@@ -790,12 +448,10 @@ class SQLCipherEncryptionTest(BaseLeapTest):
try:
# trying to open the a non-encrypted database with sqlcipher
# backend should raise a DatabaseIsNotEncrypted exception.
- SQLCipherDatabase(self.DB_FILE, PASSWORD)
+ db = sqlcipher_open(self.DB_FILE, PASSWORD)
+ db.close()
raise dbapi2.DatabaseError(
"SQLCipher backend should not be able to open non-encrypted "
"dbs.")
except DatabaseIsNotEncrypted:
pass
-
-
-load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py
new file mode 100644
index 00000000..83c3449e
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py
@@ -0,0 +1,398 @@
+# -*- coding: utf-8 -*-
+# test_sqlcipher.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Test sqlcipher backend sync.
+"""
+
+
+import os
+import simplejson as json
+from u1db import (
+ sync,
+ vectorclock,
+)
+
+from testscenarios import TestWithScenarios
+
+from leap.soledad.common.crypto import ENC_SCHEME_KEY
+from leap.soledad.client.target import SoledadSyncTarget
+from leap.soledad.client.crypto import decrypt_doc_dict
+from leap.soledad.client.sqlcipher import (
+ SQLCipherDatabase,
+)
+
+from leap.soledad.common.tests import u1db_tests as tests
+from leap.soledad.common.tests.u1db_tests import test_sync
+from leap.soledad.common.tests.test_sqlcipher import SQLCIPHER_SCENARIOS
+from leap.soledad.common.tests.util import (
+ make_soledad_app,
+ BaseSoledadTest,
+ SoledadWithCouchServerMixin,
+)
+
+
+#-----------------------------------------------------------------------------
+# The following tests come from `u1db.tests.test_sync`.
+#-----------------------------------------------------------------------------
+
+def sync_via_synchronizer_and_soledad(test, db_source, db_target,
+ trace_hook=None, trace_hook_shallow=None):
+ if trace_hook:
+ test.skipTest("full trace hook unsupported over http")
+ path = test._http_at[db_target]
+ target = SoledadSyncTarget.connect(
+ test.getURL(path), test._soledad._crypto)
+ target.set_token_credentials('user-uuid', 'auth-token')
+ if trace_hook_shallow:
+ target._set_trace_hook_shallow(trace_hook_shallow)
+ return sync.Synchronizer(db_source, target).sync()
+
+
+sync_scenarios = []
+for name, scenario in SQLCIPHER_SCENARIOS:
+ scenario['do_sync'] = test_sync.sync_via_synchronizer
+ sync_scenarios.append((name, scenario))
+
+
+class SQLCipherDatabaseSyncTests(
+ TestWithScenarios,
+ test_sync.DatabaseSyncTests,
+ BaseSoledadTest):
+ """
+ Test for succesfull sync between SQLCipher and LeapBackend.
+
+ Some of the tests in this class had to be adapted because the remote
+ backend always receive encrypted content, and so it can not rely on
+ document's content comparison to try to autoresolve conflicts.
+ """
+
+ scenarios = sync_scenarios
+
+ #def setUp(self):
+ # test_sync.DatabaseSyncTests.setUp(self)
+
+ def tearDown(self):
+ test_sync.DatabaseSyncTests.tearDown(self)
+ if hasattr(self, 'db1') and isinstance(self.db1, SQLCipherDatabase):
+ self.db1.close()
+ if hasattr(self, 'db1_copy') \
+ and isinstance(self.db1_copy, SQLCipherDatabase):
+ self.db1_copy.close()
+ if hasattr(self, 'db2') \
+ and isinstance(self.db2, SQLCipherDatabase):
+ self.db2.close()
+ if hasattr(self, 'db2_copy') \
+ and isinstance(self.db2_copy, SQLCipherDatabase):
+ self.db2_copy.close()
+ if hasattr(self, 'db3') \
+ and isinstance(self.db3, SQLCipherDatabase):
+ self.db3.close()
+
+ def test_sync_autoresolves(self):
+ """
+ Test for sync autoresolve remote.
+
+ This test was adapted because the remote database receives encrypted
+ content and so it can't compare documents contents to autoresolve.
+ """
+ # The remote database can't autoresolve conflicts based on magic
+ # content convergence, so we modify this test to leave the possibility
+ # of the remode document ending up in conflicted state.
+ self.db1 = self.create_database('test1', 'source')
+ self.db2 = self.create_database('test2', 'target')
+ doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc')
+ rev1 = doc1.rev
+ doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc')
+ rev2 = doc2.rev
+ self.sync(self.db1, self.db2)
+ doc = self.db1.get_doc('doc')
+ self.assertFalse(doc.has_conflicts)
+ # if remote content is in conflicted state, then document revisions
+ # will be different.
+ #self.assertEqual(doc.rev, self.db2.get_doc('doc').rev)
+ v = vectorclock.VectorClockRev(doc.rev)
+ self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1)))
+ self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2)))
+
+ def test_sync_autoresolves_moar(self):
+ """
+ Test for sync autoresolve local.
+
+ This test was adapted to decrypt remote content before assert.
+ """
+ # here we test that when a database that has a conflicted document is
+ # the source of a sync, and the target database has a revision of the
+ # conflicted document that is newer than the source database's, and
+ # that target's database's document's content is the same as the
+ # source's document's conflict's, the source's document's conflict gets
+ # autoresolved, and the source's document's revision bumped.
+ #
+ # idea is as follows:
+ # A B
+ # a1 -
+ # `------->
+ # a1 a1
+ # v v
+ # a2 a1b1
+ # `------->
+ # a1b1+a2 a1b1
+ # v
+ # a1b1+a2 a1b2 (a1b2 has same content as a2)
+ # `------->
+ # a3b2 a1b2 (autoresolved)
+ # `------->
+ # a3b2 a3b2
+ self.db1 = self.create_database('test1', 'source')
+ self.db2 = self.create_database('test2', 'target')
+ self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc')
+ self.sync(self.db1, self.db2)
+ for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]:
+ doc = db.get_doc('doc')
+ doc.set_json(content)
+ db.put_doc(doc)
+ self.sync(self.db1, self.db2)
+ # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict
+ doc = self.db1.get_doc('doc')
+ rev1 = doc.rev
+ self.assertTrue(doc.has_conflicts)
+ # set db2 to have a doc of {} (same as db1 before the conflict)
+ doc = self.db2.get_doc('doc')
+ doc.set_json('{}')
+ self.db2.put_doc(doc)
+ rev2 = doc.rev
+ # sync it across
+ self.sync(self.db1, self.db2)
+ # tadaa!
+ doc = self.db1.get_doc('doc')
+ self.assertFalse(doc.has_conflicts)
+ vec1 = vectorclock.VectorClockRev(rev1)
+ vec2 = vectorclock.VectorClockRev(rev2)
+ vec3 = vectorclock.VectorClockRev(doc.rev)
+ self.assertTrue(vec3.is_newer(vec1))
+ self.assertTrue(vec3.is_newer(vec2))
+ # because the conflict is on the source, sync it another time
+ self.sync(self.db1, self.db2)
+ # make sure db2 now has the exact same thing
+ doc1 = self.db1.get_doc('doc')
+ self.assertGetEncryptedDoc(
+ self.db2,
+ doc1.doc_id, doc1.rev, doc1.get_json(), False)
+
+ def test_sync_autoresolves_moar_backwards(self):
+ # here we would test that when a database that has a conflicted
+ # document is the target of a sync, and the source database has a
+ # revision of the conflicted document that is newer than the target
+ # database's, and that source's database's document's content is the
+ # same as the target's document's conflict's, the target's document's
+ # conflict gets autoresolved, and the document's revision bumped.
+ #
+ # Despite that, in Soledad we suppose that the server never syncs, so
+ # it never has conflicted documents. Also, if it had, convergence
+ # would not be possible by checking document's contents because they
+ # would be encrypted in server.
+ #
+ # Therefore we suppress this test.
+ pass
+
+ def test_sync_autoresolves_moar_backwards_three(self):
+ # here we would test that when a database that has a conflicted
+ # document is the target of a sync, and the source database has a
+ # revision of the conflicted document that is newer than the target
+ # database's, and that source's database's document's content is the
+ # same as the target's document's conflict's, the target's document's
+ # conflict gets autoresolved, and the document's revision bumped.
+ #
+ # We use the same reasoning from the last test to suppress this one.
+ pass
+
+ def test_sync_propagates_resolution(self):
+ """
+ Test if synchronization propagates resolution.
+
+ This test was adapted to decrypt remote content before assert.
+ """
+ self.db1 = self.create_database('test1', 'both')
+ self.db2 = self.create_database('test2', 'both')
+ doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc')
+ db3 = self.create_database('test3', 'both')
+ self.sync(self.db2, self.db1)
+ self.assertEqual(
+ self.db1._get_generation_info(),
+ self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid))
+ self.assertEqual(
+ self.db2._get_generation_info(),
+ self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid))
+ self.sync(db3, self.db1)
+ # update on 2
+ doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}')
+ self.db2.put_doc(doc2)
+ self.sync(self.db2, db3)
+ self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev)
+ # update on 1
+ doc1.set_json('{"a": 3}')
+ self.db1.put_doc(doc1)
+ # conflicts
+ self.sync(self.db2, self.db1)
+ self.sync(db3, self.db1)
+ self.assertTrue(self.db2.get_doc('the-doc').has_conflicts)
+ self.assertTrue(db3.get_doc('the-doc').has_conflicts)
+ # resolve
+ conflicts = self.db2.get_doc_conflicts('the-doc')
+ doc4 = self.make_document('the-doc', None, '{"a": 4}')
+ revs = [doc.rev for doc in conflicts]
+ self.db2.resolve_doc(doc4, revs)
+ doc2 = self.db2.get_doc('the-doc')
+ self.assertEqual(doc4.get_json(), doc2.get_json())
+ self.assertFalse(doc2.has_conflicts)
+ self.sync(self.db2, db3)
+ doc3 = db3.get_doc('the-doc')
+ if ENC_SCHEME_KEY in doc3.content:
+ _crypto = self._soledad._crypto
+ key = _crypto.doc_passphrase(doc3.doc_id)
+ secret = _crypto.secret
+ doc3.set_json(decrypt_doc_dict(
+ doc3.content,
+ doc3.doc_id, doc3.rev, key, secret))
+ self.assertEqual(doc4.get_json(), doc3.get_json())
+ self.assertFalse(doc3.has_conflicts)
+ self.db1.close()
+ self.db2.close()
+ db3.close()
+
+ def test_sync_puts_changes(self):
+ """
+ Test if sync puts changes in remote replica.
+
+ This test was adapted to decrypt remote content before assert.
+ """
+ self.db1 = self.create_database('test1', 'source')
+ self.db2 = self.create_database('test2', 'target')
+ doc = self.db1.create_doc_from_json(tests.simple_doc)
+ self.assertEqual(1, self.sync(self.db1, self.db2))
+ self.assertGetEncryptedDoc(
+ self.db2, doc.doc_id, doc.rev, tests.simple_doc, False)
+ self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0])
+ self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0])
+ self.assertLastExchangeLog(
+ self.db2,
+ {'receive': {'docs': [(doc.doc_id, doc.rev)],
+ 'source_uid': 'test1',
+ 'source_gen': 1, 'last_known_gen': 0},
+ 'return': {'docs': [], 'last_gen': 1}})
+
+
+def _make_local_db_and_token_http_target(test, path='test'):
+ test.startServer()
+ db = test.request_state._create_database(os.path.basename(path))
+ st = SoledadSyncTarget.connect(
+ test.getURL(path), crypto=test._soledad._crypto)
+ st.set_token_credentials('user-uuid', 'auth-token')
+ return db, st
+
+
+target_scenarios = [
+ ('leap', {
+ 'create_db_and_target': _make_local_db_and_token_http_target,
+# 'make_app_with_state': tests.test_remote_sync_target.make_http_app,
+ 'make_app_with_state': make_soledad_app,
+ 'do_sync': sync_via_synchronizer_and_soledad}),
+]
+
+
+class SQLCipherSyncTargetTests(
+ TestWithScenarios,
+ SoledadWithCouchServerMixin,
+ test_sync.DatabaseSyncTargetTests):
+
+ scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS,
+ target_scenarios))
+
+ whitebox = False
+
+ def setUp(self):
+ self.main_test_class = test_sync.DatabaseSyncTargetTests
+ SoledadWithCouchServerMixin.setUp(self)
+
+ def test_sync_exchange(self):
+ """
+ Modified to account for possibly receiving encrypted documents from
+ sever-side.
+ """
+ docs_by_gen = [
+ (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10,
+ 'T-sid')]
+ new_gen, trans_id = self.st.sync_exchange(
+ docs_by_gen, 'replica', last_known_generation=0,
+ last_known_trans_id=None, return_doc_cb=self.receive_doc)
+ self.assertGetEncryptedDoc(
+ self.db, 'doc-id', 'replica:1', tests.simple_doc, False)
+ self.assertTransactionLog(['doc-id'], self.db)
+ last_trans_id = self.getLastTransId(self.db)
+ self.assertEqual(([], 1, last_trans_id),
+ (self.other_changes, new_gen, last_trans_id))
+ self.assertEqual(10, self.st.get_sync_info('replica')[3])
+
+ def test_sync_exchange_push_many(self):
+ """
+ Modified to account for possibly receiving encrypted documents from
+ sever-side.
+ """
+ docs_by_gen = [
+ (self.make_document(
+ 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'),
+ (self.make_document('doc-id2', 'replica:1', tests.nested_doc), 11,
+ 'T-2')]
+ new_gen, trans_id = self.st.sync_exchange(
+ docs_by_gen, 'replica', last_known_generation=0,
+ last_known_trans_id=None, return_doc_cb=self.receive_doc)
+ self.assertGetEncryptedDoc(
+ self.db, 'doc-id', 'replica:1', tests.simple_doc, False)
+ self.assertGetEncryptedDoc(
+ self.db, 'doc-id2', 'replica:1', tests.nested_doc, False)
+ self.assertTransactionLog(['doc-id', 'doc-id2'], self.db)
+ last_trans_id = self.getLastTransId(self.db)
+ self.assertEqual(([], 2, last_trans_id),
+ (self.other_changes, new_gen, trans_id))
+ self.assertEqual(11, self.st.get_sync_info('replica')[3])
+
+ def test_sync_exchange_returns_many_new_docs(self):
+ """
+ Modified to account for JSON serialization differences.
+ """
+ doc = self.db.create_doc_from_json(tests.simple_doc)
+ doc2 = self.db.create_doc_from_json(tests.nested_doc)
+ self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)
+ new_gen, _ = self.st.sync_exchange(
+ [], 'other-replica', last_known_generation=0,
+ last_known_trans_id=None, return_doc_cb=self.receive_doc)
+ self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)
+ self.assertEqual(2, new_gen)
+ self.assertEqual(
+ [(doc.doc_id, doc.rev, 1),
+ (doc2.doc_id, doc2.rev, 2)],
+ [c[:2] + c[3:4] for c in self.other_changes])
+ self.assertEqual(
+ json.dumps(tests.simple_doc),
+ json.dumps(self.other_changes[0][2]))
+ self.assertEqual(
+ json.loads(tests.nested_doc),
+ json.loads(self.other_changes[1][2]))
+ if self.whitebox:
+ self.assertEqual(
+ self.db._last_exchange_log['return'],
+ {'last_gen': 2, 'docs':
+ [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]})
diff --git a/common/src/leap/soledad/common/tests/test_sync.py b/common/src/leap/soledad/common/tests/test_sync.py
index 0433fac9..92c601dc 100644
--- a/common/src/leap/soledad/common/tests/test_sync.py
+++ b/common/src/leap/soledad/common/tests/test_sync.py
@@ -16,43 +16,35 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-import mock
-import os
import json
import tempfile
import threading
import time
+
from urlparse import urljoin
+from twisted.internet import defer
+
+from testscenarios import TestWithScenarios
from leap.soledad.common import couch
+from leap.soledad.client import target
+from leap.soledad.client import sync
+from leap.soledad.server import SoledadApp
-from leap.soledad.common.tests import BaseSoledadTest
-from leap.soledad.common.tests import test_sync_target
from leap.soledad.common.tests import u1db_tests as tests
-from leap.soledad.common.tests.u1db_tests import (
- TestCaseWithServer,
- simple_doc,
- test_backends,
- test_sync
-)
-from leap.soledad.common.tests.test_couch import CouchDBTestCase
-from leap.soledad.common.tests.test_target_soledad import (
- make_token_soledad_app,
- make_leap_document_for_test,
-)
-from leap.soledad.common.tests.test_sync_target import token_leap_sync_target
-from leap.soledad.client import (
- Soledad,
- target,
-)
+from leap.soledad.common.tests.u1db_tests import TestCaseWithServer
+from leap.soledad.common.tests.u1db_tests import simple_doc
+from leap.soledad.common.tests.u1db_tests import test_sync
+from leap.soledad.common.tests.util import make_token_soledad_app
+from leap.soledad.common.tests.util import make_soledad_document_for_test
+from leap.soledad.common.tests.util import token_soledad_sync_target
+from leap.soledad.common.tests.util import BaseSoledadTest
from leap.soledad.common.tests.util import SoledadWithCouchServerMixin
-from leap.soledad.client.sync import SoledadSynchronizer
-from leap.soledad.server import SoledadApp
-
+from leap.soledad.common.tests.test_couch import CouchDBTestCase
class InterruptableSyncTestCase(
- CouchDBTestCase, TestCaseWithServer):
+ BaseSoledadTest, CouchDBTestCase, TestCaseWithServer):
"""
Tests for encrypted sync using Soledad server backed by a couch database.
"""
@@ -61,51 +53,12 @@ class InterruptableSyncTestCase(
def make_app_with_state(state):
return make_token_soledad_app(state)
- make_document_for_test = make_leap_document_for_test
-
- sync_target = token_leap_sync_target
+ make_document_for_test = make_soledad_document_for_test
- def _soledad_instance(self, user='user-uuid', passphrase=u'123',
- prefix='',
- secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME,
- local_db_path='soledad.u1db', server_url='',
- cert_file=None, auth_token=None, secret_id=None):
- """
- Instantiate Soledad.
- """
-
- # this callback ensures we save a document which is sent to the shared
- # db.
- def _put_doc_side_effect(doc):
- self._doc_put = doc
-
- # we need a mocked shared db or else Soledad will try to access the
- # network to find if there are uploaded secrets.
- class MockSharedDB(object):
-
- get_doc = mock.Mock(return_value=None)
- put_doc = mock.Mock(side_effect=_put_doc_side_effect)
- lock = mock.Mock(return_value=('atoken', 300))
- unlock = mock.Mock()
-
- def __call__(self):
- return self
-
- Soledad._shared_db = MockSharedDB()
- return Soledad(
- user,
- passphrase,
- secrets_path=os.path.join(self.tempdir, prefix, secrets_path),
- local_db_path=os.path.join(
- self.tempdir, prefix, local_db_path),
- server_url=server_url,
- cert_file=cert_file,
- auth_token=auth_token,
- secret_id=secret_id)
+ sync_target = token_soledad_sync_target
def make_app(self):
- self.request_state = couch.CouchServerState(
- self._couch_url, 'shared', 'tokens')
+ self.request_state = couch.CouchServerState(self._couch_url)
return self.make_app_with_state(self.request_state)
def setUp(self):
@@ -135,7 +88,8 @@ class InterruptableSyncTestCase(
def run(self):
while db._get_generation() < 2:
- time.sleep(1)
+ #print "WAITING %d" % db._get_generation()
+ time.sleep(0.1)
self._soledad.stop_sync()
time.sleep(1)
@@ -143,16 +97,7 @@ class InterruptableSyncTestCase(
self.startServer()
# instantiate soledad and create a document
- sol = self._soledad_instance(
- # token is verified in test_target.make_token_soledad_app
- auth_token='auth-token'
- )
- _, doclist = sol.get_all_docs()
- self.assertEqual([], doclist)
-
- # create many small files
- for i in range(0, number_of_docs):
- sol.create_doc(json.loads(simple_doc))
+ sol = self._soledad_instance(user='user-uuid', server_url=self.getURL())
# ensure remote db exists before syncing
db = couch.CouchDatabase.open_database(
@@ -164,21 +109,35 @@ class InterruptableSyncTestCase(
t = _SyncInterruptor(sol, db)
t.start()
- # sync with server
- sol._server_url = self.getURL()
- sol.sync() # this will be interrupted when couch db gen >= 2
- t.join()
+ d = sol.get_all_docs()
+ d.addCallback(lambda results: self.assertEqual([], results[1]))
- # recover the sync process
- sol.sync()
+ def _create_docs(results):
+ # create many small files
+ deferreds = []
+ for i in range(0, number_of_docs):
+ deferreds.append(sol.create_doc(json.loads(simple_doc)))
+ return defer.DeferredList(deferreds)
- gen, doclist = db.get_all_docs()
- self.assertEqual(number_of_docs, len(doclist))
-
- # delete remote database
- db.delete_database()
- db.close()
- sol.close()
+ # sync with server
+ d.addCallback(_create_docs)
+ d.addCallback(lambda _: sol.get_all_docs())
+ d.addCallback(lambda results: self.assertEqual(number_of_docs, len(results[1])))
+ d.addCallback(lambda _: sol.sync())
+ d.addCallback(lambda _: t.join())
+ d.addCallback(lambda _: db.get_all_docs())
+ d.addCallback(lambda results: self.assertNotEqual(number_of_docs, len(results[1])))
+ d.addCallback(lambda _: sol.sync())
+ d.addCallback(lambda _: db.get_all_docs())
+ d.addCallback(lambda results: self.assertEqual(number_of_docs, len(results[1])))
+
+ def _tear_down(results):
+ db.delete_database()
+ db.close()
+ sol.close()
+
+ d.addCallback(_tear_down)
+ return d
def make_soledad_app(state):
@@ -186,6 +145,7 @@ def make_soledad_app(state):
class TestSoledadDbSync(
+ TestWithScenarios,
SoledadWithCouchServerMixin,
test_sync.TestDbSync):
"""
@@ -198,7 +158,7 @@ class TestSoledadDbSync(
'make_database_for_test': tests.make_memory_database_for_test,
}),
('py-token-http', {
- 'make_app_with_state': test_sync_target.make_token_soledad_app,
+ 'make_app_with_state': make_token_soledad_app,
'make_database_for_test': tests.make_memory_database_for_test,
'token': True
}),
@@ -211,10 +171,11 @@ class TestSoledadDbSync(
"""
Need to explicitely invoke inicialization on all bases.
"""
- tests.TestCaseWithServer.setUp(self)
- self.main_test_class = test_sync.TestDbSync
+ #tests.TestCaseWithServer.setUp(self)
+ #self.main_test_class = test_sync.TestDbSync
SoledadWithCouchServerMixin.setUp(self)
self.startServer()
+ self.db = self.make_database_for_test(self, 'test1')
self.db2 = couch.CouchDatabase.open_database(
urljoin(
'http://localhost:' + str(self.wrapper.port), 'test'),
@@ -227,7 +188,7 @@ class TestSoledadDbSync(
"""
self.db2.delete_database()
SoledadWithCouchServerMixin.tearDown(self)
- tests.TestCaseWithServer.tearDown(self)
+ #tests.TestCaseWithServer.tearDown(self)
def do_sync(self, target_name):
"""
@@ -240,7 +201,7 @@ class TestSoledadDbSync(
'token': 'auth-token',
}})
target_url = self.getURL(target_name)
- return SoledadSynchronizer(
+ return sync.SoledadSynchronizer(
self.db,
target.SoledadSyncTarget(
target_url,
@@ -254,8 +215,10 @@ class TestSoledadDbSync(
Adapted to check for encrypted content.
"""
+
doc1 = self.db.create_doc_from_json(tests.simple_doc)
doc2 = self.db2.create_doc_from_json(tests.nested_doc)
+
local_gen_before_sync = self.do_sync('test')
gen, _, changes = self.db.whats_changed(local_gen_before_sync)
self.assertEqual(1, len(changes))
@@ -287,6 +250,3 @@ class TestSoledadDbSync(
s_gen, _ = db3._get_replica_gen_and_trans_id('test1')
self.assertEqual(1, t_gen)
self.assertEqual(1, s_gen)
-
-
-load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_sync_deferred.py b/common/src/leap/soledad/common/tests/test_sync_deferred.py
index 48e3150f..010f5492 100644
--- a/common/src/leap/soledad/common/tests/test_sync_deferred.py
+++ b/common/src/leap/soledad/common/tests/test_sync_deferred.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# test_sync_deferred.py
# Copyright (C) 2014 LEAP
#
@@ -21,31 +20,31 @@ import time
import os
import random
import string
+import shutil
+
from urlparse import urljoin
-from leap.soledad.common.tests import u1db_tests as tests, ADDRESS
+from leap.soledad.common import couch
+from leap.soledad.client.sqlcipher import (
+ SQLCipherOptions,
+ SQLCipherDatabase,
+ SQLCipherU1DBSync,
+)
+
+from testscenarios import TestWithScenarios
+
+from leap.soledad.common.tests import u1db_tests as tests
from leap.soledad.common.tests.u1db_tests import test_sync
+from leap.soledad.common.tests.util import ADDRESS
+from leap.soledad.common.tests.util import SoledadWithCouchServerMixin
+from leap.soledad.common.tests.util import make_soledad_app
-from leap.soledad.common.document import SoledadDocument
-from leap.soledad.common import couch
-from leap.soledad.client import target
-from leap.soledad.client.sync import SoledadSynchronizer
# Just to make clear how this test is different... :)
DEFER_DECRYPTION = True
WAIT_STEP = 1
MAX_WAIT = 10
-
-from leap.soledad.common.tests import test_sqlcipher as ts
-from leap.soledad.server import SoledadApp
-
-
-from leap.soledad.client.sqlcipher import open as open_sqlcipher
-from leap.soledad.common.tests.util import SoledadWithCouchServerMixin
-from leap.soledad.common.tests.util import make_soledad_app
-
-
DBPASS = "pass"
@@ -57,8 +56,10 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin):
defer_sync_encryption = True
def setUp(self):
+ SoledadWithCouchServerMixin.setUp(self)
# config info
self.db1_file = os.path.join(self.tempdir, "db1.u1db")
+ os.unlink(self.db1_file)
self.db_pass = DBPASS
self.email = ADDRESS
@@ -67,16 +68,21 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin):
# each local db.
self.rand_prefix = ''.join(
map(lambda x: random.choice(string.ascii_letters), range(6)))
- # initialize soledad by hand so we can control keys
- self._soledad = self._soledad_instance(
- prefix=self.rand_prefix, user=self.email)
-
- # open test dbs: db1 will be the local sqlcipher db
- # (which instantiates a syncdb)
- self.db1 = open_sqlcipher(self.db1_file, DBPASS, create=True,
- document_factory=SoledadDocument,
- crypto=self._soledad._crypto,
- defer_encryption=True)
+
+ # open test dbs: db1 will be the local sqlcipher db (which
+ # instantiates a syncdb). We use the self._soledad instance that was
+ # already created on some setUp method.
+ import binascii
+ tohex = binascii.b2a_hex
+ key = tohex(self._soledad.secrets.get_local_storage_key())
+ sync_db_key = tohex(self._soledad.secrets.get_sync_db_key())
+ dbpath = self._soledad._local_db_path
+
+ self.opts = SQLCipherOptions(
+ dbpath, key, is_raw_key=True, create=False,
+ defer_encryption=True, sync_db_key=sync_db_key)
+ self.db1 = SQLCipherDatabase(self.opts)
+
self.db2 = couch.CouchDatabase.open_database(
urljoin(
'http://localhost:' + str(self.wrapper.port), 'test'),
@@ -89,23 +95,8 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin):
self._soledad.close()
# XXX should not access "private" attrs
- for f in [self._soledad._local_db_path,
- self._soledad._secrets_path,
- self.db1._sync_db_path]:
- if os.path.isfile(f):
- os.unlink(f)
-
-
-#SQLCIPHER_SCENARIOS = [
-# ('http', {
-# #'make_app_with_state': test_sync_target.make_token_soledad_app,
-# 'make_app_with_state': make_soledad_app,
-# 'make_database_for_test': ts.make_sqlcipher_database_for_test,
-# 'copy_database_for_test': ts.copy_sqlcipher_database_for_test,
-# 'make_document_for_test': ts.make_document_for_test,
-# 'token': True
-# }),
-#]
+ shutil.rmtree(os.path.dirname(self._soledad._local_db_path))
+ SoledadWithCouchServerMixin.tearDown(self)
class SyncTimeoutError(Exception):
@@ -116,8 +107,9 @@ class SyncTimeoutError(Exception):
class TestSoledadDbSyncDeferredEncDecr(
- BaseSoledadDeferredEncTest,
- test_sync.TestDbSync):
+ TestWithScenarios,
+ test_sync.TestDbSync,
+ BaseSoledadDeferredEncTest):
"""
Test db.sync remote sync shortcut.
Case with deferred encryption and decryption: using the intermediate
@@ -134,13 +126,16 @@ class TestSoledadDbSyncDeferredEncDecr(
oauth = False
token = True
+ def make_app(self):
+ self.request_state = couch.CouchServerState(self._couch_url)
+ return self.make_app_with_state(self.request_state)
+
def setUp(self):
"""
Need to explicitely invoke inicialization on all bases.
"""
- tests.TestCaseWithServer.setUp(self)
- self.main_test_class = test_sync.TestDbSync
BaseSoledadDeferredEncTest.setUp(self)
+ self.server = self.server_thread = None
self.startServer()
self.syncer = None
@@ -148,8 +143,10 @@ class TestSoledadDbSyncDeferredEncDecr(
"""
Need to explicitely invoke destruction on all bases.
"""
+ dbsyncer = getattr(self, 'dbsyncer', None)
+ if dbsyncer:
+ dbsyncer.close()
BaseSoledadDeferredEncTest.tearDown(self)
- tests.TestCaseWithServer.tearDown(self)
def do_sync(self, target_name):
"""
@@ -157,25 +154,20 @@ class TestSoledadDbSyncDeferredEncDecr(
and Token auth.
"""
if self.token:
- extra = dict(creds={'token': {
+ creds={'token': {
'uuid': 'user-uuid',
'token': 'auth-token',
- }})
+ }}
target_url = self.getURL(target_name)
- syncdb = getattr(self.db1, "_sync_db", None)
-
- syncer = SoledadSynchronizer(
- self.db1,
- target.SoledadSyncTarget(
- target_url,
- crypto=self._soledad._crypto,
- sync_db=syncdb,
- **extra))
- # Keep a reference to be able to know when the sync
- # has finished.
- self.syncer = syncer
- return syncer.sync(
- autocreate=True, defer_decryption=DEFER_DECRYPTION)
+
+ # get a u1db syncer
+ crypto = self._soledad._crypto
+ replica_uid = self.db1._replica_uid
+ dbsyncer = SQLCipherU1DBSync(self.opts, crypto, replica_uid,
+ defer_encryption=True)
+ self.dbsyncer = dbsyncer
+ return dbsyncer.sync(target_url, creds=creds,
+ autocreate=True,defer_decryption=DEFER_DECRYPTION)
else:
return test_sync.TestDbSync.do_sync(self, target_name)
@@ -200,28 +192,30 @@ class TestSoledadDbSyncDeferredEncDecr(
"""
doc1 = self.db1.create_doc_from_json(tests.simple_doc)
doc2 = self.db2.create_doc_from_json(tests.nested_doc)
+ d = self.do_sync('test')
- import time
- # need to give time to the encryption to proceed
- # TODO should implement a defer list to subscribe to the all-decrypted
- # event
- time.sleep(2)
+ def _assert_successful_sync(results):
+ import time
+ # need to give time to the encryption to proceed
+ # TODO should implement a defer list to subscribe to the all-decrypted
+ # event
+ time.sleep(2)
+ local_gen_before_sync = results
+ self.wait_for_sync()
- local_gen_before_sync = self.do_sync('test')
- self.wait_for_sync()
+ gen, _, changes = self.db1.whats_changed(local_gen_before_sync)
+ self.assertEqual(1, len(changes))
- gen, _, changes = self.db1.whats_changed(local_gen_before_sync)
- self.assertEqual(1, len(changes))
+ self.assertEqual(doc2.doc_id, changes[0][0])
+ self.assertEqual(1, gen - local_gen_before_sync)
- self.assertEqual(doc2.doc_id, changes[0][0])
- self.assertEqual(1, gen - local_gen_before_sync)
+ self.assertGetEncryptedDoc(
+ self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False)
+ self.assertGetEncryptedDoc(
+ self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False)
- self.assertGetEncryptedDoc(
- self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False)
- self.assertGetEncryptedDoc(
- self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False)
+ d.addCallback(_assert_successful_sync)
+ return d
def test_db_sync_autocreate(self):
pass
-
-load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_sync_target.py b/common/src/leap/soledad/common/tests/test_sync_target.py
index edc4589b..941dc37c 100644
--- a/common/src/leap/soledad/common/tests/test_sync_target.py
+++ b/common/src/leap/soledad/common/tests/test_sync_target.py
@@ -19,101 +19,36 @@ Test Leap backend bits: sync target
"""
import cStringIO
import os
-
+import time
import simplejson as json
import u1db
+import random
+import string
+import shutil
-from uuid import uuid4
-
-from u1db.remote import http_database
-
-from u1db import SyncTarget
-from u1db.sync import Synchronizer
-from u1db.remote import (
- http_client,
- http_database,
- http_target,
-)
-
-from leap.soledad import client
-from leap.soledad.client import (
- target,
- auth,
- crypto,
- VerifiedHTTPSConnection,
- sync,
-)
-from leap.soledad.common.document import SoledadDocument
-from leap.soledad.server.auth import SoledadTokenAuthMiddleware
+from testscenarios import TestWithScenarios
+from urlparse import urljoin
+from leap.soledad.client import target
+from leap.soledad.client import crypto
+from leap.soledad.client.sqlcipher import SQLCipherU1DBSync
+from leap.soledad.client.sqlcipher import SQLCipherOptions
+from leap.soledad.client.sqlcipher import SQLCipherDatabase
+
+from leap.soledad.common import couch
+from leap.soledad.common.document import SoledadDocument
from leap.soledad.common.tests import u1db_tests as tests
-from leap.soledad.common.tests import BaseSoledadTest
-from leap.soledad.common.tests.util import (
- make_sqlcipher_database_for_test,
- make_soledad_app,
- make_token_soledad_app,
- SoledadWithCouchServerMixin,
-)
-from leap.soledad.common.tests.u1db_tests import test_backends
+from leap.soledad.common.tests.util import make_sqlcipher_database_for_test
+from leap.soledad.common.tests.util import make_soledad_app
+from leap.soledad.common.tests.util import make_token_soledad_app
+from leap.soledad.common.tests.util import make_soledad_document_for_test
+from leap.soledad.common.tests.util import token_soledad_sync_target
+from leap.soledad.common.tests.util import BaseSoledadTest
+from leap.soledad.common.tests.util import SoledadWithCouchServerMixin
+from leap.soledad.common.tests.util import ADDRESS
from leap.soledad.common.tests.u1db_tests import test_remote_sync_target
from leap.soledad.common.tests.u1db_tests import test_sync
-from leap.soledad.common.tests.test_couch import (
- CouchDBTestCase,
- CouchDBWrapper,
-)
-
-from leap.soledad.server import SoledadApp
-from leap.soledad.server.auth import SoledadTokenAuthMiddleware
-
-
-#-----------------------------------------------------------------------------
-# The following tests come from `u1db.tests.test_backends`.
-#-----------------------------------------------------------------------------
-
-def make_leap_document_for_test(test, doc_id, rev, content,
- has_conflicts=False):
- return SoledadDocument(
- doc_id, rev, content, has_conflicts=has_conflicts)
-
-
-LEAP_SCENARIOS = [
- ('http', {
- 'make_database_for_test': test_backends.make_http_database_for_test,
- 'copy_database_for_test': test_backends.copy_http_database_for_test,
- 'make_document_for_test': make_leap_document_for_test,
- 'make_app_with_state': make_soledad_app}),
-]
-
-
-def make_token_http_database_for_test(test, replica_uid):
- test.startServer()
- test.request_state._create_database(replica_uid)
-
- class _HTTPDatabaseWithToken(
- http_database.HTTPDatabase, auth.TokenBasedAuth):
-
- def set_token_credentials(self, uuid, token):
- auth.TokenBasedAuth.set_token_credentials(self, uuid, token)
-
- def _sign_request(self, method, url_query, params):
- return auth.TokenBasedAuth._sign_request(
- self, method, url_query, params)
-
- http_db = _HTTPDatabaseWithToken(test.getURL('test'))
- http_db.set_token_credentials('user-uuid', 'auth-token')
- return http_db
-
-
-def copy_token_http_database_for_test(test, db):
- # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS
- # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE
- # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN
- # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR
- # HOUSE.
- http_db = test.request_state._copy_database(db)
- http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token')
- return http_db
#-----------------------------------------------------------------------------
@@ -143,12 +78,6 @@ class TestSoledadParsingSyncStream(
target.
"""
- def setUp(self):
- test_remote_sync_target.TestParsingSyncStream.setUp(self)
-
- def tearDown(self):
- test_remote_sync_target.TestParsingSyncStream.tearDown(self)
-
def test_extra_comma(self):
"""
Test adapted to use encrypted content.
@@ -230,17 +159,6 @@ class TestSoledadParsingSyncStream(
# functions for TestRemoteSyncTargets
#
-def leap_sync_target(test, path):
- return target.SoledadSyncTarget(
- test.getURL(path), crypto=test._soledad._crypto)
-
-
-def token_leap_sync_target(test, path):
- st = leap_sync_target(test, path)
- st.set_token_credentials('user-uuid', 'auth-token')
- return st
-
-
def make_local_db_and_soledad_target(test, path='test'):
test.startServer()
db = test.request_state._create_database(os.path.basename(path))
@@ -256,31 +174,32 @@ def make_local_db_and_token_soledad_target(test):
class TestSoledadSyncTarget(
+ TestWithScenarios,
SoledadWithCouchServerMixin,
test_remote_sync_target.TestRemoteSyncTargets):
scenarios = [
('token_soledad',
{'make_app_with_state': make_token_soledad_app,
- 'make_document_for_test': make_leap_document_for_test,
+ 'make_document_for_test': make_soledad_document_for_test,
'create_db_and_target': make_local_db_and_token_soledad_target,
'make_database_for_test': make_sqlcipher_database_for_test,
- 'sync_target': token_leap_sync_target}),
+ 'sync_target': token_soledad_sync_target}),
]
def setUp(self):
- tests.TestCaseWithServer.setUp(self)
- self.main_test_class = test_remote_sync_target.TestRemoteSyncTargets
+ TestWithScenarios.setUp(self)
SoledadWithCouchServerMixin.setUp(self)
self.startServer()
self.db1 = make_sqlcipher_database_for_test(self, 'test1')
self.db2 = self.request_state._create_database('test2')
def tearDown(self):
+ #db2, _ = self.request_state.ensure_database('test2')
+ self.db2.delete_database()
+ self.db1.close()
SoledadWithCouchServerMixin.tearDown(self)
- tests.TestCaseWithServer.tearDown(self)
- db, _ = self.request_state.ensure_database('test2')
- db.delete_database()
+ TestWithScenarios.tearDown(self)
def test_sync_exchange_send(self):
"""
@@ -288,7 +207,6 @@ class TestSoledadSyncTarget(
This test was adapted to decrypt remote content before assert.
"""
- self.startServer()
db = self.request_state._create_database('test')
remote_target = self.getSyncTarget('test')
other_docs = []
@@ -309,14 +227,9 @@ class TestSoledadSyncTarget(
"""
Test for sync exchange failure and retry.
- This test was adapted to:
- - decrypt remote content before assert.
- - not expect a bounced document because soledad has stateful
- recoverable sync.
+ This test was adapted to decrypt remote content before assert.
"""
- self.startServer()
-
def blackhole_getstderr(inst):
return cStringIO.StringIO()
@@ -352,8 +265,9 @@ class TestSoledadSyncTarget(
doc2 = self.make_document('doc-here2', 'replica:1',
'{"value": "here2"}')
- # we do not expect an HTTPError because soledad sync fails gracefully
- remote_target.sync_exchange(
+ self.assertRaises(
+ u1db.errors.HTTPError,
+ remote_target.sync_exchange,
[(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')],
'replica', last_known_generation=0, last_known_trans_id=None,
return_doc_cb=receive_doc)
@@ -384,7 +298,6 @@ class TestSoledadSyncTarget(
This test was adapted to decrypt remote content before assert.
"""
- self.startServer()
remote_target = self.getSyncTarget('test')
other_docs = []
replica_uid_box = []
@@ -425,7 +338,9 @@ target_scenarios = [
class SoledadDatabaseSyncTargetTests(
- SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests):
+ TestWithScenarios,
+ SoledadWithCouchServerMixin,
+ test_sync.DatabaseSyncTargetTests):
scenarios = (
tests.multiply_scenarios(
@@ -520,8 +435,25 @@ class SoledadDatabaseSyncTargetTests(
[(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]})
+# Just to make clear how this test is different... :)
+DEFER_DECRYPTION = False
+
+WAIT_STEP = 1
+MAX_WAIT = 10
+DBPASS = "pass"
+
+
+class SyncTimeoutError(Exception):
+ """
+ Dummy exception to notify timeout during sync.
+ """
+ pass
+
+
class TestSoledadDbSync(
- SoledadWithCouchServerMixin, test_sync.TestDbSync):
+ TestWithScenarios,
+ SoledadWithCouchServerMixin,
+ test_sync.TestDbSync):
"""Test db.sync remote sync shortcut"""
scenarios = [
@@ -536,9 +468,66 @@ class TestSoledadDbSync(
oauth = False
token = False
+
+ def make_app(self):
+ self.request_state = couch.CouchServerState(self._couch_url)
+ return self.make_app_with_state(self.request_state)
+
def setUp(self):
- self.main_test_class = test_sync.TestDbSync
+ """
+ Need to explicitely invoke inicialization on all bases.
+ """
SoledadWithCouchServerMixin.setUp(self)
+ self.server = self.server_thread = None
+ self.startServer()
+ self.syncer = None
+
+ # config info
+ self.db1_file = os.path.join(self.tempdir, "db1.u1db")
+ os.unlink(self.db1_file)
+ self.db_pass = DBPASS
+ self.email = ADDRESS
+
+ # get a random prefix for each test, so we do not mess with
+ # concurrency during initialization and shutting down of
+ # each local db.
+ self.rand_prefix = ''.join(
+ map(lambda x: random.choice(string.ascii_letters), range(6)))
+
+ # open test dbs: db1 will be the local sqlcipher db (which
+ # instantiates a syncdb). We use the self._soledad instance that was
+ # already created on some setUp method.
+ import binascii
+ tohex = binascii.b2a_hex
+ key = tohex(self._soledad.secrets.get_local_storage_key())
+ sync_db_key = tohex(self._soledad.secrets.get_sync_db_key())
+ dbpath = self._soledad._local_db_path
+
+ self.opts = SQLCipherOptions(
+ dbpath, key, is_raw_key=True, create=False,
+ defer_encryption=True, sync_db_key=sync_db_key)
+ self.db1 = SQLCipherDatabase(self.opts)
+
+ self.db2 = couch.CouchDatabase.open_database(
+ urljoin(
+ 'http://localhost:' + str(self.wrapper.port), 'test'),
+ create=True,
+ ensure_ddocs=True)
+
+ def tearDown(self):
+ """
+ Need to explicitely invoke destruction on all bases.
+ """
+ dbsyncer = getattr(self, 'dbsyncer', None)
+ if dbsyncer:
+ dbsyncer.close()
+ self.db1.close()
+ self.db2.close()
+ self._soledad.close()
+
+ # XXX should not access "private" attrs
+ shutil.rmtree(os.path.dirname(self._soledad._local_db_path))
+ SoledadWithCouchServerMixin.tearDown(self)
def do_sync(self, target_name):
"""
@@ -546,44 +535,71 @@ class TestSoledadDbSync(
and Token auth.
"""
if self.token:
- extra = dict(creds={'token': {
+ creds={'token': {
'uuid': 'user-uuid',
'token': 'auth-token',
- }})
+ }}
target_url = self.getURL(target_name)
- return sync.SoledadSynchronizer(
- self.db,
- target.SoledadSyncTarget(
- target_url,
- crypto=self._soledad._crypto,
- **extra)).sync(autocreate=True,
- defer_decryption=False)
+
+ # get a u1db syncer
+ crypto = self._soledad._crypto
+ replica_uid = self.db1._replica_uid
+ dbsyncer = SQLCipherU1DBSync(self.opts, crypto, replica_uid,
+ defer_encryption=True)
+ self.dbsyncer = dbsyncer
+ return dbsyncer.sync(target_url, creds=creds,
+ autocreate=True,defer_decryption=DEFER_DECRYPTION)
else:
return test_sync.TestDbSync.do_sync(self, target_name)
+ def wait_for_sync(self):
+ """
+ Wait for sync to finish.
+ """
+ wait = 0
+ syncer = self.syncer
+ if syncer is not None:
+ while syncer.syncing:
+ time.sleep(WAIT_STEP)
+ wait += WAIT_STEP
+ if wait >= MAX_WAIT:
+ raise SyncTimeoutError
+
def test_db_sync(self):
"""
Test sync.
Adapted to check for encrypted content.
"""
- doc1 = self.db.create_doc_from_json(tests.simple_doc)
+ doc1 = self.db1.create_doc_from_json(tests.simple_doc)
doc2 = self.db2.create_doc_from_json(tests.nested_doc)
- local_gen_before_sync = self.do_sync('test2')
- gen, _, changes = self.db.whats_changed(local_gen_before_sync)
- self.assertEqual(1, len(changes))
- self.assertEqual(doc2.doc_id, changes[0][0])
- self.assertEqual(1, gen - local_gen_before_sync)
- self.assertGetEncryptedDoc(
- self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False)
- self.assertGetEncryptedDoc(
- self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False)
+ d = self.do_sync('test')
+
+ def _assert_successful_sync(results):
+ import time
+ # need to give time to the encryption to proceed
+ # TODO should implement a defer list to subscribe to the all-decrypted
+ # event
+ time.sleep(2)
+ local_gen_before_sync = results
+ self.wait_for_sync()
+
+ gen, _, changes = self.db1.whats_changed(local_gen_before_sync)
+ self.assertEqual(1, len(changes))
+
+ self.assertEqual(doc2.doc_id, changes[0][0])
+ self.assertEqual(1, gen - local_gen_before_sync)
+
+ self.assertGetEncryptedDoc(
+ self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False)
+ self.assertGetEncryptedDoc(
+ self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False)
+
+ d.addCallback(_assert_successful_sync)
+ return d
def test_db_sync_autocreate(self):
"""
We bypass this test because we never need to autocreate databases.
"""
pass
-
-
-load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_target.py b/common/src/leap/soledad/common/tests/test_target.py
deleted file mode 100644
index 6242099d..00000000
--- a/common/src/leap/soledad/common/tests/test_target.py
+++ /dev/null
@@ -1,794 +0,0 @@
-# -*- coding: utf-8 -*-
-# test_target.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-Test Leap backend bits.
-"""
-
-import u1db
-import os
-import ssl
-import simplejson as json
-import cStringIO
-
-
-from u1db import SyncTarget
-from u1db.sync import Synchronizer
-from u1db.remote import (
- http_client,
- http_database,
- http_target,
-)
-
-from leap.soledad import client
-from leap.soledad.client import (
- target,
- auth,
- VerifiedHTTPSConnection,
- sync,
-)
-from leap.soledad.common.document import SoledadDocument
-from leap.soledad.server.auth import SoledadTokenAuthMiddleware
-
-
-from leap.soledad.common.tests import u1db_tests as tests
-from leap.soledad.common.tests import BaseSoledadTest
-from leap.soledad.common.tests.util import (
- make_sqlcipher_database_for_test,
- make_soledad_app,
- make_token_soledad_app,
- SoledadWithCouchServerMixin,
-)
-from leap.soledad.common.tests.u1db_tests import test_backends
-from leap.soledad.common.tests.u1db_tests import test_http_database
-from leap.soledad.common.tests.u1db_tests import test_http_client
-from leap.soledad.common.tests.u1db_tests import test_document
-from leap.soledad.common.tests.u1db_tests import test_remote_sync_target
-from leap.soledad.common.tests.u1db_tests import test_https
-from leap.soledad.common.tests.u1db_tests import test_sync
-from leap.soledad.common.tests.test_couch import (
- CouchDBTestCase,
- CouchDBWrapper,
-)
-
-
-#-----------------------------------------------------------------------------
-# The following tests come from `u1db.tests.test_backends`.
-#-----------------------------------------------------------------------------
-
-def make_leap_document_for_test(test, doc_id, rev, content,
- has_conflicts=False):
- return SoledadDocument(
- doc_id, rev, content, has_conflicts=has_conflicts)
-
-
-LEAP_SCENARIOS = [
- ('http', {
- 'make_database_for_test': test_backends.make_http_database_for_test,
- 'copy_database_for_test': test_backends.copy_http_database_for_test,
- 'make_document_for_test': make_leap_document_for_test,
- 'make_app_with_state': make_soledad_app}),
-]
-
-
-def make_token_http_database_for_test(test, replica_uid):
- test.startServer()
- test.request_state._create_database(replica_uid)
-
- class _HTTPDatabaseWithToken(
- http_database.HTTPDatabase, auth.TokenBasedAuth):
-
- def set_token_credentials(self, uuid, token):
- auth.TokenBasedAuth.set_token_credentials(self, uuid, token)
-
- def _sign_request(self, method, url_query, params):
- return auth.TokenBasedAuth._sign_request(
- self, method, url_query, params)
-
- http_db = _HTTPDatabaseWithToken(test.getURL('test'))
- http_db.set_token_credentials('user-uuid', 'auth-token')
- return http_db
-
-
-def copy_token_http_database_for_test(test, db):
- # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS
- # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE
- # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN
- # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR
- # HOUSE.
- http_db = test.request_state._copy_database(db)
- http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token')
- return http_db
-
-
-class SoledadTests(test_backends.AllDatabaseTests, BaseSoledadTest):
-
- scenarios = LEAP_SCENARIOS + [
- ('token_http', {'make_database_for_test':
- make_token_http_database_for_test,
- 'copy_database_for_test':
- copy_token_http_database_for_test,
- 'make_document_for_test': make_leap_document_for_test,
- 'make_app_with_state': make_token_soledad_app,
- })
- ]
-
-
-#-----------------------------------------------------------------------------
-# The following tests come from `u1db.tests.test_http_client`.
-#-----------------------------------------------------------------------------
-
-class TestSoledadClientBase(test_http_client.TestHTTPClientBase):
- """
- This class should be used to test Token auth.
- """
-
- def getClientWithToken(self, **kwds):
- self.startServer()
-
- class _HTTPClientWithToken(
- http_client.HTTPClientBase, auth.TokenBasedAuth):
-
- def set_token_credentials(self, uuid, token):
- auth.TokenBasedAuth.set_token_credentials(self, uuid, token)
-
- def _sign_request(self, method, url_query, params):
- return auth.TokenBasedAuth._sign_request(
- self, method, url_query, params)
-
- return _HTTPClientWithToken(self.getURL('dbase'), **kwds)
-
- def test_oauth(self):
- """
- Suppress oauth test (we test for token auth here).
- """
- pass
-
- def test_oauth_ctr_creds(self):
- """
- Suppress oauth test (we test for token auth here).
- """
- pass
-
- def test_oauth_Unauthorized(self):
- """
- Suppress oauth test (we test for token auth here).
- """
- pass
-
- def app(self, environ, start_response):
- res = test_http_client.TestHTTPClientBase.app(
- self, environ, start_response)
- if res is not None:
- return res
- # mime solead application here.
- if '/token' in environ['PATH_INFO']:
- auth = environ.get(SoledadTokenAuthMiddleware.HTTP_AUTH_KEY)
- if not auth:
- start_response("401 Unauthorized",
- [('Content-Type', 'application/json')])
- return [json.dumps({"error": "unauthorized",
- "message": e.message})]
- scheme, encoded = auth.split(None, 1)
- if scheme.lower() != 'token':
- start_response("401 Unauthorized",
- [('Content-Type', 'application/json')])
- return [json.dumps({"error": "unauthorized",
- "message": e.message})]
- uuid, token = encoded.decode('base64').split(':', 1)
- if uuid != 'user-uuid' and token != 'auth-token':
- return unauth_err("Incorrect address or token.")
- start_response("200 OK", [('Content-Type', 'application/json')])
- return [json.dumps([environ['PATH_INFO'], uuid, token])]
-
- def test_token(self):
- """
- Test if token is sent correctly.
- """
- cli = self.getClientWithToken()
- cli.set_token_credentials('user-uuid', 'auth-token')
- res, headers = cli._request('GET', ['doc', 'token'])
- self.assertEqual(
- ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res))
-
- def test_token_ctr_creds(self):
- cli = self.getClientWithToken(creds={'token': {
- 'uuid': 'user-uuid',
- 'token': 'auth-token',
- }})
- res, headers = cli._request('GET', ['doc', 'token'])
- self.assertEqual(
- ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res))
-
-
-#-----------------------------------------------------------------------------
-# The following tests come from `u1db.tests.test_document`.
-#-----------------------------------------------------------------------------
-
-class TestSoledadDocument(test_document.TestDocument, BaseSoledadTest):
-
- scenarios = ([(
- 'leap', {'make_document_for_test': make_leap_document_for_test})])
-
-
-class TestSoledadPyDocument(test_document.TestPyDocument, BaseSoledadTest):
-
- scenarios = ([(
- 'leap', {'make_document_for_test': make_leap_document_for_test})])
-
-
-#-----------------------------------------------------------------------------
-# The following tests come from `u1db.tests.test_remote_sync_target`.
-#-----------------------------------------------------------------------------
-
-class TestSoledadSyncTargetBasics(
- test_remote_sync_target.TestHTTPSyncTargetBasics):
- """
- Some tests had to be copied to this class so we can instantiate our own
- target.
- """
-
- def test_parse_url(self):
- remote_target = target.SoledadSyncTarget('http://127.0.0.1:12345/')
- self.assertEqual('http', remote_target._url.scheme)
- self.assertEqual('127.0.0.1', remote_target._url.hostname)
- self.assertEqual(12345, remote_target._url.port)
- self.assertEqual('/', remote_target._url.path)
-
-
-class TestSoledadParsingSyncStream(
- test_remote_sync_target.TestParsingSyncStream,
- BaseSoledadTest):
- """
- Some tests had to be copied to this class so we can instantiate our own
- target.
- """
-
- def setUp(self):
- test_remote_sync_target.TestParsingSyncStream.setUp(self)
-
- def tearDown(self):
- test_remote_sync_target.TestParsingSyncStream.tearDown(self)
-
- def test_extra_comma(self):
- """
- Test adapted to use encrypted content.
- """
- doc = SoledadDocument('i', rev='r')
- doc.content = {}
- enc_json = target.encrypt_doc(self._soledad._crypto, doc)
- tgt = target.SoledadSyncTarget(
- "http://foo/foo", crypto=self._soledad._crypto)
-
- self.assertRaises(u1db.errors.BrokenSyncStream,
- tgt._parse_sync_stream, "[\r\n{},\r\n]", None)
- self.assertRaises(u1db.errors.BrokenSyncStream,
- tgt._parse_sync_stream,
- '[\r\n{},\r\n{"id": "i", "rev": "r", '
- '"content": %s, "gen": 3, "trans_id": "T-sid"}'
- ',\r\n]' % json.dumps(enc_json),
- lambda doc, gen, trans_id: None)
-
- def test_wrong_start(self):
- tgt = target.SoledadSyncTarget("http://foo/foo")
-
- self.assertRaises(u1db.errors.BrokenSyncStream,
- tgt._parse_sync_stream, "{}\r\n]", None)
-
- self.assertRaises(u1db.errors.BrokenSyncStream,
- tgt._parse_sync_stream, "\r\n{}\r\n]", None)
-
- self.assertRaises(u1db.errors.BrokenSyncStream,
- tgt._parse_sync_stream, "", None)
-
- def test_wrong_end(self):
- tgt = target.SoledadSyncTarget("http://foo/foo")
-
- self.assertRaises(u1db.errors.BrokenSyncStream,
- tgt._parse_sync_stream, "[\r\n{}", None)
-
- self.assertRaises(u1db.errors.BrokenSyncStream,
- tgt._parse_sync_stream, "[\r\n", None)
-
- def test_missing_comma(self):
- tgt = target.SoledadSyncTarget("http://foo/foo")
-
- self.assertRaises(u1db.errors.BrokenSyncStream,
- tgt._parse_sync_stream,
- '[\r\n{}\r\n{"id": "i", "rev": "r", '
- '"content": "c", "gen": 3}\r\n]', None)
-
- def test_no_entries(self):
- tgt = target.SoledadSyncTarget("http://foo/foo")
-
- self.assertRaises(u1db.errors.BrokenSyncStream,
- tgt._parse_sync_stream, "[\r\n]", None)
-
- def test_error_in_stream(self):
- tgt = target.SoledadSyncTarget("http://foo/foo")
-
- self.assertRaises(u1db.errors.Unavailable,
- tgt._parse_sync_stream,
- '[\r\n{"new_generation": 0},'
- '\r\n{"error": "unavailable"}\r\n', None)
-
- self.assertRaises(u1db.errors.Unavailable,
- tgt._parse_sync_stream,
- '[\r\n{"error": "unavailable"}\r\n', None)
-
- self.assertRaises(u1db.errors.BrokenSyncStream,
- tgt._parse_sync_stream,
- '[\r\n{"error": "?"}\r\n', None)
-
-
-#
-# functions for TestRemoteSyncTargets
-#
-
-def leap_sync_target(test, path):
- return target.SoledadSyncTarget(
- test.getURL(path), crypto=test._soledad._crypto)
-
-
-def token_leap_sync_target(test, path):
- st = leap_sync_target(test, path)
- st.set_token_credentials('user-uuid', 'auth-token')
- return st
-
-
-def make_local_db_and_soledad_target(test, path='test'):
- test.startServer()
- db = test.request_state._create_database(os.path.basename(path))
- st = target.SoledadSyncTarget.connect(
- test.getURL(path), crypto=test._soledad._crypto)
- return db, st
-
-
-def make_local_db_and_token_soledad_target(test):
- db, st = make_local_db_and_soledad_target(test, 'test')
- st.set_token_credentials('user-uuid', 'auth-token')
- return db, st
-
-
-class TestSoledadSyncTarget(
- SoledadWithCouchServerMixin,
- test_remote_sync_target.TestRemoteSyncTargets):
-
- scenarios = [
- ('token_soledad',
- {'make_app_with_state': make_token_soledad_app,
- 'make_document_for_test': make_leap_document_for_test,
- 'create_db_and_target': make_local_db_and_token_soledad_target,
- 'make_database_for_test': make_sqlcipher_database_for_test,
- 'sync_target': token_leap_sync_target}),
- ]
-
- def setUp(self):
- tests.TestCaseWithServer.setUp(self)
- self.main_test_class = test_remote_sync_target.TestRemoteSyncTargets
- SoledadWithCouchServerMixin.setUp(self)
- self.startServer()
- self.db1 = make_sqlcipher_database_for_test(self, 'test1')
- self.db2 = self.request_state._create_database('test2')
-
- def tearDown(self):
- SoledadWithCouchServerMixin.tearDown(self)
- tests.TestCaseWithServer.tearDown(self)
- db, _ = self.request_state.ensure_database('test2')
- db.delete_database()
-
- def test_sync_exchange_send(self):
- """
- Test for sync exchanging send of document.
-
- This test was adapted to decrypt remote content before assert.
- """
- self.startServer()
- db = self.request_state._create_database('test')
- remote_target = self.getSyncTarget('test')
- other_docs = []
-
- def receive_doc(doc, gen, trans_id):
- other_docs.append((doc.doc_id, doc.rev, doc.get_json()))
-
- doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}')
- new_gen, trans_id = remote_target.sync_exchange(
- [(doc, 10, 'T-sid')], 'replica', last_known_generation=0,
- last_known_trans_id=None, return_doc_cb=receive_doc)
- self.assertEqual(1, new_gen)
- self.assertGetEncryptedDoc(
- db, 'doc-here', 'replica:1', '{"value": "here"}', False)
-
- def test_sync_exchange_send_failure_and_retry_scenario(self):
- """
- Test for sync exchange failure and retry.
-
- This test was adapted to:
- - decrypt remote content before assert.
- - not expect a bounced document because soledad has stateful
- recoverable sync.
- """
-
- self.startServer()
-
- def blackhole_getstderr(inst):
- return cStringIO.StringIO()
-
- self.patch(self.server.RequestHandlerClass, 'get_stderr',
- blackhole_getstderr)
- db = self.request_state._create_database('test')
- _put_doc_if_newer = db._put_doc_if_newer
- trigger_ids = ['doc-here2']
-
- def bomb_put_doc_if_newer(self, doc, save_conflict,
- replica_uid=None, replica_gen=None,
- replica_trans_id=None, number_of_docs=None,
- doc_idx=None, sync_id=None):
- if doc.doc_id in trigger_ids:
- raise Exception
- return _put_doc_if_newer(doc, save_conflict=save_conflict,
- replica_uid=replica_uid,
- replica_gen=replica_gen,
- replica_trans_id=replica_trans_id,
- number_of_docs=number_of_docs,
- doc_idx=doc_idx,
- sync_id=sync_id)
- from leap.soledad.common.tests.test_couch import IndexedCouchDatabase
- self.patch(
- IndexedCouchDatabase, '_put_doc_if_newer', bomb_put_doc_if_newer)
- remote_target = self.getSyncTarget('test')
- other_changes = []
-
- def receive_doc(doc, gen, trans_id):
- other_changes.append(
- (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id))
-
- doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}')
- doc2 = self.make_document('doc-here2', 'replica:1',
- '{"value": "here2"}')
- # We do not expect an exception here because the sync fails gracefully
- remote_target.sync_exchange(
- [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')],
- 'replica', last_known_generation=0, last_known_trans_id=None,
- return_doc_cb=receive_doc)
- self.assertGetEncryptedDoc(
- db, 'doc-here', 'replica:1', '{"value": "here"}',
- False)
- self.assertEqual(
- (10, 'T-sid'), db._get_replica_gen_and_trans_id('replica'))
- self.assertEqual([], other_changes)
- # retry
- trigger_ids = []
- new_gen, trans_id = remote_target.sync_exchange(
- [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0,
- last_known_trans_id=None, return_doc_cb=receive_doc)
- self.assertGetEncryptedDoc(
- db, 'doc-here2', 'replica:1', '{"value": "here2"}',
- False)
- self.assertEqual(
- (11, 'T-sud'), db._get_replica_gen_and_trans_id('replica'))
- self.assertEqual(2, new_gen)
- self.assertEqual(
- ('doc-here', 'replica:1', '{"value": "here"}', 1),
- other_changes[0][:-1])
-
- def test_sync_exchange_send_ensure_callback(self):
- """
- Test for sync exchange failure and retry.
-
- This test was adapted to decrypt remote content before assert.
- """
- self.startServer()
- remote_target = self.getSyncTarget('test')
- other_docs = []
- replica_uid_box = []
-
- def receive_doc(doc, gen, trans_id):
- other_docs.append((doc.doc_id, doc.rev, doc.get_json()))
-
- def ensure_cb(replica_uid):
- replica_uid_box.append(replica_uid)
-
- doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}')
- new_gen, trans_id = remote_target.sync_exchange(
- [(doc, 10, 'T-sid')], 'replica', last_known_generation=0,
- last_known_trans_id=None, return_doc_cb=receive_doc,
- ensure_callback=ensure_cb)
- self.assertEqual(1, new_gen)
- db = self.request_state.open_database('test')
- self.assertEqual(1, len(replica_uid_box))
- self.assertEqual(db._replica_uid, replica_uid_box[0])
- self.assertGetEncryptedDoc(
- db, 'doc-here', 'replica:1', '{"value": "here"}', False)
-
- def test_sync_exchange_in_stream_error(self):
- # we bypass this test because our sync_exchange process does not
- # return u1db error 503 "unavailable" for now.
- pass
-
-
-#-----------------------------------------------------------------------------
-# The following tests come from `u1db.tests.test_https`.
-#-----------------------------------------------------------------------------
-
-def token_leap_https_sync_target(test, host, path):
- _, port = test.server.server_address
- st = target.SoledadSyncTarget(
- 'https://%s:%d/%s' % (host, port, path),
- crypto=test._soledad._crypto)
- st.set_token_credentials('user-uuid', 'auth-token')
- return st
-
-
-class TestSoledadSyncTargetHttpsSupport(
- test_https.TestHttpSyncTargetHttpsSupport,
- BaseSoledadTest):
-
- scenarios = [
- ('token_soledad_https',
- {'server_def': test_https.https_server_def,
- 'make_app_with_state': make_token_soledad_app,
- 'make_document_for_test': make_leap_document_for_test,
- 'sync_target': token_leap_https_sync_target}),
- ]
-
- def setUp(self):
- # the parent constructor undoes our SSL monkey patch to ensure tests
- # run smoothly with standard u1db.
- test_https.TestHttpSyncTargetHttpsSupport.setUp(self)
- # so here monkey patch again to test our functionality.
- http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection
- client.SOLEDAD_CERT = http_client.CA_CERTS
-
- def test_working(self):
- """
- Test that SSL connections work well.
-
- This test was adapted to patch Soledad's HTTPS connection custom class
- with the intended CA certificates.
- """
- self.startServer()
- db = self.request_state._create_database('test')
- self.patch(client, 'SOLEDAD_CERT', self.cacert_pem)
- remote_target = self.getSyncTarget('localhost', 'test')
- remote_target.record_sync_info('other-id', 2, 'T-id')
- self.assertEqual(
- (2, 'T-id'), db._get_replica_gen_and_trans_id('other-id'))
-
- def test_host_mismatch(self):
- """
- Test that SSL connections to a hostname different than the one in the
- certificate raise CertificateError.
-
- This test was adapted to patch Soledad's HTTPS connection custom class
- with the intended CA certificates.
- """
- self.startServer()
- self.request_state._create_database('test')
- self.patch(client, 'SOLEDAD_CERT', self.cacert_pem)
- remote_target = self.getSyncTarget('127.0.0.1', 'test')
- self.assertRaises(
- http_client.CertificateError, remote_target.record_sync_info,
- 'other-id', 2, 'T-id')
-
-
-#-----------------------------------------------------------------------------
-# The following tests come from `u1db.tests.test_http_database`.
-#-----------------------------------------------------------------------------
-
-class _HTTPDatabase(http_database.HTTPDatabase, auth.TokenBasedAuth):
- """
- Wraps our token auth implementation.
- """
-
- def set_token_credentials(self, uuid, token):
- auth.TokenBasedAuth.set_token_credentials(self, uuid, token)
-
- def _sign_request(self, method, url_query, params):
- return auth.TokenBasedAuth._sign_request(
- self, method, url_query, params)
-
-
-class TestHTTPDatabaseWithCreds(
- test_http_database.TestHTTPDatabaseCtrWithCreds):
-
- def test_get_sync_target_inherits_token_credentials(self):
- # this test was from TestDatabaseSimpleOperations but we put it here
- # for convenience.
- self.db = _HTTPDatabase('dbase')
- self.db.set_token_credentials('user-uuid', 'auth-token')
- st = self.db.get_sync_target()
- self.assertEqual(self.db._creds, st._creds)
-
- def test_ctr_with_creds(self):
- db1 = _HTTPDatabase('http://dbs/db', creds={'token': {
- 'uuid': 'user-uuid',
- 'token': 'auth-token',
- }})
- self.assertIn('token', db1._creds)
-
-
-#-----------------------------------------------------------------------------
-# The following tests come from `u1db.tests.test_sync`.
-#-----------------------------------------------------------------------------
-
-target_scenarios = [
- ('token_leap', {'create_db_and_target':
- make_local_db_and_token_soledad_target,
- 'make_app_with_state': make_soledad_app}),
-]
-
-
-class SoledadDatabaseSyncTargetTests(
- SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests):
-
- scenarios = (
- tests.multiply_scenarios(
- tests.DatabaseBaseTests.scenarios,
- target_scenarios))
-
- whitebox = False
-
- def setUp(self):
- self.main_test_class = test_sync.DatabaseSyncTargetTests
- SoledadWithCouchServerMixin.setUp(self)
-
- def test_sync_exchange(self):
- """
- Test sync exchange.
-
- This test was adapted to decrypt remote content before assert.
- """
- sol, _ = make_local_db_and_soledad_target(self)
- docs_by_gen = [
- (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10,
- 'T-sid')]
- new_gen, trans_id = self.st.sync_exchange(
- docs_by_gen, 'replica', last_known_generation=0,
- last_known_trans_id=None, return_doc_cb=self.receive_doc)
- self.assertGetEncryptedDoc(
- self.db, 'doc-id', 'replica:1', tests.simple_doc, False)
- self.assertTransactionLog(['doc-id'], self.db)
- last_trans_id = self.getLastTransId(self.db)
- self.assertEqual(([], 1, last_trans_id),
- (self.other_changes, new_gen, last_trans_id))
- self.assertEqual(10, self.st.get_sync_info('replica')[3])
- sol.close()
-
- def test_sync_exchange_push_many(self):
- """
- Test sync exchange.
-
- This test was adapted to decrypt remote content before assert.
- """
- docs_by_gen = [
- (self.make_document(
- 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'),
- (self.make_document(
- 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')]
- new_gen, trans_id = self.st.sync_exchange(
- docs_by_gen, 'replica', last_known_generation=0,
- last_known_trans_id=None, return_doc_cb=self.receive_doc)
- self.assertGetEncryptedDoc(
- self.db, 'doc-id', 'replica:1', tests.simple_doc, False)
- self.assertGetEncryptedDoc(
- self.db, 'doc-id2', 'replica:1', tests.nested_doc, False)
- self.assertTransactionLog(['doc-id', 'doc-id2'], self.db)
- last_trans_id = self.getLastTransId(self.db)
- self.assertEqual(([], 2, last_trans_id),
- (self.other_changes, new_gen, trans_id))
- self.assertEqual(11, self.st.get_sync_info('replica')[3])
-
- def test_sync_exchange_returns_many_new_docs(self):
- """
- Test sync exchange.
-
- This test was adapted to avoid JSON serialization comparison as local
- and remote representations might differ. It looks directly at the
- doc's contents instead.
- """
- doc = self.db.create_doc_from_json(tests.simple_doc)
- doc2 = self.db.create_doc_from_json(tests.nested_doc)
- self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)
- new_gen, _ = self.st.sync_exchange(
- [], 'other-replica', last_known_generation=0,
- last_known_trans_id=None, return_doc_cb=self.receive_doc)
- self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db)
- self.assertEqual(2, new_gen)
- self.assertEqual(
- [(doc.doc_id, doc.rev, 1),
- (doc2.doc_id, doc2.rev, 2)],
- [c[:-3] + c[-2:-1] for c in self.other_changes])
- self.assertEqual(
- json.loads(tests.simple_doc),
- json.loads(self.other_changes[0][2]))
- self.assertEqual(
- json.loads(tests.nested_doc),
- json.loads(self.other_changes[1][2]))
- if self.whitebox:
- self.assertEqual(
- self.db._last_exchange_log['return'],
- {'last_gen': 2, 'docs':
- [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]})
-
-
-class TestSoledadDbSync(
- SoledadWithCouchServerMixin, test_sync.TestDbSync):
- """Test db.sync remote sync shortcut"""
-
- scenarios = [
- ('py-token-http', {
- 'create_db_and_target': make_local_db_and_token_soledad_target,
- 'make_app_with_state': make_token_soledad_app,
- 'make_database_for_test': make_sqlcipher_database_for_test,
- 'token': True
- }),
- ]
-
- oauth = False
- token = False
-
- def setUp(self):
- self.main_test_class = test_sync.TestDbSync
- SoledadWithCouchServerMixin.setUp(self)
-
- def do_sync(self, target_name):
- """
- Perform sync using SoledadSyncTarget and Token auth.
- """
- if self.token:
- extra = dict(creds={'token': {
- 'uuid': 'user-uuid',
- 'token': 'auth-token',
- }})
- target_url = self.getURL(target_name)
- return Synchronizer(
- self.db,
- target.SoledadSyncTarget(
- target_url,
- crypto=self._soledad._crypto,
- **extra)).sync(autocreate=True)
- else:
- return test_sync.TestDbSync.do_sync(self, target_name)
-
- def test_db_sync(self):
- """
- Test sync.
-
- Adapted to check for encrypted content.
- """
- doc1 = self.db.create_doc_from_json(tests.simple_doc)
- doc2 = self.db2.create_doc_from_json(tests.nested_doc)
- local_gen_before_sync = self.do_sync('test2')
- gen, _, changes = self.db.whats_changed(local_gen_before_sync)
- self.assertEqual(1, len(changes))
- self.assertEqual(doc2.doc_id, changes[0][0])
- self.assertEqual(1, gen - local_gen_before_sync)
- self.assertGetEncryptedDoc(
- self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False)
- self.assertGetEncryptedDoc(
- self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False)
-
- def test_db_sync_autocreate(self):
- """
- We bypass this test because we never need to autocreate databases.
- """
- pass
-
-
-load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/test_target_soledad.py b/common/src/leap/soledad/common/tests/test_target_soledad.py
deleted file mode 100644
index 899203b8..00000000
--- a/common/src/leap/soledad/common/tests/test_target_soledad.py
+++ /dev/null
@@ -1,102 +0,0 @@
-from u1db.remote import (
- http_database,
-)
-
-from leap.soledad.client import (
- auth,
- VerifiedHTTPSConnection,
-)
-from leap.soledad.common.document import SoledadDocument
-from leap.soledad.server import SoledadApp
-from leap.soledad.server.auth import SoledadTokenAuthMiddleware
-
-
-from leap.soledad.common.tests import u1db_tests as tests
-from leap.soledad.common.tests import BaseSoledadTest
-from leap.soledad.common.tests.u1db_tests import test_backends
-
-
-#-----------------------------------------------------------------------------
-# The following tests come from `u1db.tests.test_backends`.
-#-----------------------------------------------------------------------------
-
-def make_leap_document_for_test(test, doc_id, rev, content,
- has_conflicts=False):
- return SoledadDocument(
- doc_id, rev, content, has_conflicts=has_conflicts)
-
-
-def make_soledad_app(state):
- return SoledadApp(state)
-
-
-def make_token_soledad_app(state):
- app = SoledadApp(state)
-
- def _verify_authentication_data(uuid, auth_data):
- if uuid == 'user-uuid' and auth_data == 'auth-token':
- return True
- return False
-
- # we test for action authorization in leap.soledad.common.tests.test_server
- def _verify_authorization(uuid, environ):
- return True
-
- application = SoledadTokenAuthMiddleware(app)
- application._verify_authentication_data = _verify_authentication_data
- application._verify_authorization = _verify_authorization
- return application
-
-
-LEAP_SCENARIOS = [
- ('http', {
- 'make_database_for_test': test_backends.make_http_database_for_test,
- 'copy_database_for_test': test_backends.copy_http_database_for_test,
- 'make_document_for_test': make_leap_document_for_test,
- 'make_app_with_state': make_soledad_app}),
-]
-
-
-def make_token_http_database_for_test(test, replica_uid):
- test.startServer()
- test.request_state._create_database(replica_uid)
-
- class _HTTPDatabaseWithToken(
- http_database.HTTPDatabase, auth.TokenBasedAuth):
-
- def set_token_credentials(self, uuid, token):
- auth.TokenBasedAuth.set_token_credentials(self, uuid, token)
-
- def _sign_request(self, method, url_query, params):
- return auth.TokenBasedAuth._sign_request(
- self, method, url_query, params)
-
- http_db = _HTTPDatabaseWithToken(test.getURL('test'))
- http_db.set_token_credentials('user-uuid', 'auth-token')
- return http_db
-
-
-def copy_token_http_database_for_test(test, db):
- # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS
- # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE
- # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN
- # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR
- # HOUSE.
- http_db = test.request_state._copy_database(db)
- http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token')
- return http_db
-
-
-class SoledadTests(test_backends.AllDatabaseTests, BaseSoledadTest):
-
- scenarios = LEAP_SCENARIOS + [
- ('token_http', {'make_database_for_test':
- make_token_http_database_for_test,
- 'copy_database_for_test':
- copy_token_http_database_for_test,
- 'make_document_for_test': make_leap_document_for_test,
- 'make_app_with_state': make_token_soledad_app,
- })
- ]
-
-load_tests = tests.load_with_scenarios
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py
index ad66fb06..6efeb87f 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py
@@ -35,7 +35,7 @@ from pysqlcipher import dbapi2
from StringIO import StringIO
import testscenarios
-import testtools
+from twisted.trial import unittest
from u1db import (
errors,
@@ -50,7 +50,7 @@ from u1db.remote import (
)
-class TestCase(testtools.TestCase):
+class TestCase(unittest.TestCase):
def createTempDir(self, prefix='u1db-tmp-'):
"""Create a temporary directory to do some work in.
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py
index 86e76fad..27fc50dc 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py
@@ -40,6 +40,8 @@ from u1db.remote import (
http_database,
)
+from unittest import skip
+
def make_http_database_for_test(test, replica_uid, path='test', *args):
test.startServer()
@@ -79,6 +81,7 @@ class TestAlternativeDocument(DocumentBase):
"""A (not very) alternative implementation of Document."""
+@skip("Skiping tests imported from U1DB.")
class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer):
scenarios = tests.LOCAL_DATABASES_SCENARIOS + [
@@ -327,6 +330,7 @@ class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer):
self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False)
+@skip("Skiping tests imported from U1DB.")
class DocumentSizeTests(tests.DatabaseBaseTests):
scenarios = tests.LOCAL_DATABASES_SCENARIOS
@@ -351,6 +355,7 @@ class DocumentSizeTests(tests.DatabaseBaseTests):
self.assertEqual(1000000, self.db.document_size_limit)
+@skip("Skiping tests imported from U1DB.")
class LocalDatabaseTests(tests.DatabaseBaseTests):
scenarios = tests.LOCAL_DATABASES_SCENARIOS
@@ -363,6 +368,7 @@ class LocalDatabaseTests(tests.DatabaseBaseTests):
db2 = self.create_database('other-uid')
doc2 = db2.create_doc_from_json(simple_doc)
self.assertNotEqual(doc1.doc_id, doc2.doc_id)
+ db2.close()
def test_put_doc_refuses_slashes_picky(self):
doc = self.make_document('/a', None, simple_doc)
@@ -608,6 +614,7 @@ class LocalDatabaseTests(tests.DatabaseBaseTests):
self.db.whats_changed(2))
+@skip("Skiping tests imported from U1DB.")
class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests):
scenarios = tests.LOCAL_DATABASES_SCENARIOS
@@ -632,6 +639,7 @@ class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests):
self.db.validate_gen_and_trans_id, gen + 1, trans_id)
+@skip("Skiping tests imported from U1DB.")
class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests):
scenarios = tests.LOCAL_DATABASES_SCENARIOS
@@ -651,6 +659,7 @@ class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests):
self.db._validate_source, 'other', 1, 'T-sad')
+@skip("Skiping tests imported from U1DB.")
class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests):
# test supporting/functionality around storing conflicts
@@ -1027,6 +1036,7 @@ class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests):
self.assertRaises(errors.ConflictedDoc, self.db.delete_doc, doc2)
+@skip("Skiping tests imported from U1DB.")
class DatabaseIndexTests(tests.DatabaseBaseTests):
scenarios = tests.LOCAL_DATABASES_SCENARIOS
@@ -1833,6 +1843,7 @@ class DatabaseIndexTests(tests.DatabaseBaseTests):
self.assertParseError('combine(lower(x)x,foo)')
+@skip("Skiping tests imported from U1DB.")
class PythonBackendTests(tests.DatabaseBaseTests):
def setUp(self):
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py
index 8b30ed51..d8a27f51 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py
@@ -15,11 +15,13 @@
# along with u1db. If not, see <http://www.gnu.org/licenses/>.
+from unittest import skip
from u1db import errors
from leap.soledad.common.tests import u1db_tests as tests
+@skip("Skiping tests imported from U1DB.")
class TestDocument(tests.TestCase):
scenarios = ([(
@@ -83,6 +85,7 @@ class TestDocument(tests.TestCase):
self.assertEqual(len('a' + 'b'), doc_a.get_size())
+@skip("Skiping tests imported from U1DB.")
class TestPyDocument(tests.TestCase):
scenarios = ([(
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py
index 789006ba..522eb476 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py
@@ -24,6 +24,8 @@ except ImportError:
import json # noqa
import StringIO
+from unittest import skip
+
from u1db import (
__version__ as _u1db_version,
errors,
@@ -38,6 +40,7 @@ from u1db.remote import (
)
+@skip("Skiping tests imported from U1DB.")
class TestFencedReader(tests.TestCase):
def test_init(self):
@@ -145,6 +148,7 @@ class TestFencedReader(tests.TestCase):
self.assertRaises(http_app.BadRequest, reader.getline)
+@skip("Skiping tests imported from U1DB.")
class TestHTTPMethodDecorator(tests.TestCase):
def test_args(self):
@@ -253,6 +257,7 @@ class parameters:
max_entry_size = 100000
+@skip("Skiping tests imported from U1DB.")
class TestHTTPInvocationByMethodWithBody(tests.TestCase):
def test_get(self):
@@ -433,6 +438,7 @@ class TestHTTPInvocationByMethodWithBody(tests.TestCase):
self.assertRaises(http_app.BadRequest, invoke)
+@skip("Skiping tests imported from U1DB.")
class TestHTTPResponder(tests.TestCase):
def start_response(self, status, headers):
@@ -521,6 +527,7 @@ class TestHTTPResponder(tests.TestCase):
responder.content)
+@skip("Skiping tests imported from U1DB.")
class TestHTTPApp(tests.TestCase):
def setUp(self):
@@ -949,6 +956,7 @@ class TestHTTPApp(tests.TestCase):
self.assertEqual({'error': 'unavailable'}, json.loads(parts[2]))
+@skip("Skiping tests imported from U1DB.")
class TestRequestHooks(tests.TestCase):
def setUp(self):
@@ -1000,12 +1008,14 @@ class TestRequestHooks(tests.TestCase):
self.assertEqual(['begin', 'bad-request'], calls)
+@skip("Skiping tests imported from U1DB.")
class TestHTTPErrors(tests.TestCase):
def test_wire_description_to_status(self):
self.assertNotIn("error", http_errors.wire_description_to_status)
+@skip("Skiping tests imported from U1DB.")
class TestHTTPAppErrorHandling(tests.TestCase):
def setUp(self):
@@ -1113,6 +1123,7 @@ class TestHTTPAppErrorHandling(tests.TestCase):
self.assertEqual(self.exc, exc)
+@skip("Skiping tests imported from U1DB.")
class TestPluggableSyncExchange(tests.TestCase):
def setUp(self):
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py
index 08e9714e..f9e09cbd 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py
@@ -26,6 +26,8 @@ from u1db import (
errors,
)
+from unittest import skip
+
from leap.soledad.common.tests import u1db_tests as tests
from u1db.remote import (
@@ -33,6 +35,7 @@ from u1db.remote import (
)
+@skip("Skiping tests imported from U1DB.")
class TestEncoder(tests.TestCase):
def test_encode_string(self):
@@ -45,6 +48,7 @@ class TestEncoder(tests.TestCase):
self.assertEqual("false", http_client._encode_query_parameter(False))
+@skip("Skiping tests imported from U1DB.")
class TestHTTPClientBase(tests.TestCaseWithServer):
def setUp(self):
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py
index 9251000e..bf7ed5d3 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py
@@ -27,6 +27,8 @@ from u1db import (
Document,
)
+from unittest import skip
+
from leap.soledad.common.tests import u1db_tests as tests
from u1db.remote import (
@@ -38,6 +40,7 @@ from leap.soledad.common.tests.u1db_tests.test_remote_sync_target import (
)
+@skip("Skiping tests imported from U1DB.")
class TestHTTPDatabaseSimpleOperations(tests.TestCase):
def setUp(self):
@@ -190,6 +193,7 @@ class TestHTTPDatabaseSimpleOperations(tests.TestCase):
self.assertEqual(self.db._creds, st._creds)
+@skip("Skiping tests imported from U1DB.")
class TestHTTPDatabaseCtrWithCreds(tests.TestCase):
def test_ctr_with_creds(self):
@@ -202,6 +206,7 @@ class TestHTTPDatabaseCtrWithCreds(tests.TestCase):
self.assertIn('oauth', db1._creds)
+@skip("Skiping tests imported from U1DB.")
class TestHTTPDatabaseIntegration(tests.TestCaseWithServer):
make_app_with_state = staticmethod(make_http_app)
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py
index c086fbc0..cea175d6 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py
@@ -5,6 +5,7 @@ import ssl
import sys
from paste import httpserver
+from unittest import skip
from u1db.remote import (
http_client,
@@ -51,6 +52,7 @@ def oauth_https_sync_target(test, host, path):
return st
+@skip("Skiping tests imported from U1DB.")
class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer):
scenarios = [
@@ -75,7 +77,7 @@ class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer):
# order to maintain the compatibility with u1db default tests, we undo
# that replacement here.
http_client._VerifiedHTTPSConnection = \
- soledad.client.old__VerifiedHTTPSConnection
+ soledad.client.api.old__VerifiedHTTPSConnection
super(TestHttpSyncTargetHttpsSupport, self).setUp()
def getSyncTarget(self, host, path=None):
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py
index 63406245..ee249e6e 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py
@@ -22,12 +22,14 @@ from u1db import (
errors,
open as u1db_open,
)
+from unittest import skip
from leap.soledad.common.tests import u1db_tests as tests
from u1db.backends import sqlite_backend
from leap.soledad.common.tests.u1db_tests.test_backends \
import TestAlternativeDocument
+@skip("Skiping tests imported from U1DB.")
class TestU1DBOpen(tests.TestCase):
def setUp(self):
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py b/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py
index 3793e0df..bd7e4103 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py
@@ -22,6 +22,8 @@ from u1db import (
errors,
)
+from unittest import skip
+
from leap.soledad.common.tests import u1db_tests as tests
from u1db.remote import (
@@ -31,6 +33,7 @@ from u1db.remote import (
)
+@skip("Skiping tests imported from U1DB.")
class TestHTTPSyncTargetBasics(tests.TestCase):
def test_parse_url(self):
@@ -41,6 +44,7 @@ class TestHTTPSyncTargetBasics(tests.TestCase):
self.assertEqual('/', remote_target._url.path)
+@skip("Skiping tests imported from U1DB.")
class TestParsingSyncStream(tests.TestCase):
def test_wrong_start(self):
@@ -130,6 +134,7 @@ def oauth_http_sync_target(test, path):
return st
+@skip("Skiping tests imported from U1DB.")
class TestRemoteSyncTargets(tests.TestCaseWithServer):
scenarios = [
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py
index 8292dd07..aed8a6e5 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py
@@ -27,6 +27,8 @@ from u1db import (
query_parser,
)
+from unittest import skip
+
from leap.soledad.common.tests import u1db_tests as tests
from u1db.backends import sqlite_backend
@@ -38,6 +40,7 @@ simple_doc = '{"key": "value"}'
nested_doc = '{"key": "value", "sub": {"doc": "underneath"}}'
+@skip("Skiping tests imported from U1DB.")
class TestSQLiteDatabase(tests.TestCase):
def test_atomic_initialize(self):
@@ -83,6 +86,7 @@ class TestSQLiteDatabase(tests.TestCase):
self.assertTrue(db2._is_initialized(db1._get_sqlite_handle().cursor()))
+@skip("Skiping tests imported from U1DB.")
class TestSQLitePartialExpandDatabase(tests.TestCase):
def setUp(self):
diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py
index 5e2bec86..bac1f177 100644
--- a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py
+++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py
@@ -26,6 +26,8 @@ from u1db import (
SyncTarget,
)
+from unittest import skip
+
from leap.soledad.common.tests import u1db_tests as tests
from u1db.backends import (
@@ -74,6 +76,7 @@ target_scenarios = [
]
+@skip("Skiping tests imported from U1DB.")
class DatabaseSyncTargetTests(tests.DatabaseBaseTests,
tests.TestCaseWithServer):
@@ -462,6 +465,7 @@ sync_scenarios.append(('pyhttp', {
}))
+@skip("Skiping tests imported from U1DB.")
class DatabaseSyncTests(tests.DatabaseBaseTests,
tests.TestCaseWithServer):
@@ -1118,6 +1122,7 @@ class DatabaseSyncTests(tests.DatabaseBaseTests,
errors.InvalidTransactionId, self.sync, self.db1, self.db2_copy)
+@skip("Skiping tests imported from U1DB.")
class TestDbSync(tests.TestCaseWithServer):
"""Test db.sync remote sync shortcut"""
@@ -1190,6 +1195,7 @@ class TestDbSync(tests.TestCaseWithServer):
self.assertEqual(1, s_gen)
+@skip("Skiping tests imported from U1DB.")
class TestRemoteSyncIntegration(tests.TestCaseWithServer):
"""Integration tests for the most common sync scenario local -> remote"""
diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py
index 249cbdaa..17ed3855 100644
--- a/common/src/leap/soledad/common/tests/util.py
+++ b/common/src/leap/soledad/common/tests/util.py
@@ -21,33 +21,55 @@ Utilities used by multiple test suites.
"""
+import os
import tempfile
import shutil
+import random
+import string
+import u1db
+import subprocess
+import time
+import re
+import traceback
+
+from mock import Mock
from urlparse import urljoin
-
from StringIO import StringIO
from pysqlcipher import dbapi2
+
from u1db.errors import DatabaseDoesNotExist
+from u1db.remote import http_database
+
+from twisted.trial import unittest
+from leap.common.files import mkdir_p
from leap.soledad.common import soledad_assert
+from leap.soledad.common.document import SoledadDocument
from leap.soledad.common.couch import CouchDatabase, CouchServerState
-from leap.soledad.server import SoledadApp
-from leap.soledad.server.auth import SoledadTokenAuthMiddleware
+from leap.soledad.common.crypto import ENC_SCHEME_KEY
+from leap.soledad.client import Soledad
+from leap.soledad.client import target
+from leap.soledad.client import auth
+from leap.soledad.client.crypto import decrypt_doc_dict
-from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest
-from leap.soledad.common.tests.test_couch import CouchDBWrapper, CouchDBTestCase
-
+from leap.soledad.server import SoledadApp
+from leap.soledad.server.auth import SoledadTokenAuthMiddleware
-from leap.soledad.client.sqlcipher import SQLCipherDatabase
+from leap.soledad.client.sqlcipher import (
+ SQLCipherDatabase,
+ SQLCipherOptions,
+)
PASSWORD = '123456'
+ADDRESS = 'leap@leap.se'
def make_sqlcipher_database_for_test(test, replica_uid):
- db = SQLCipherDatabase(':memory:', PASSWORD)
+ db = SQLCipherDatabase(
+ SQLCipherOptions(':memory:', PASSWORD))
db._set_replica_uid(replica_uid)
return db
@@ -58,7 +80,7 @@ def copy_sqlcipher_database_for_test(test, db):
# CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN
# CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR
# HOUSE.
- new_db = SQLCipherDatabase(':memory:', PASSWORD)
+ new_db = make_sqlcipher_database_for_test(test, None)
tmpfile = StringIO()
for line in db._db_handle.iterdump():
if not 'sqlite_sequence' in line: # work around bug in iterdump
@@ -94,6 +116,324 @@ def make_token_soledad_app(state):
return application
+def make_soledad_document_for_test(test, doc_id, rev, content,
+ has_conflicts=False):
+ return SoledadDocument(
+ doc_id, rev, content, has_conflicts=has_conflicts)
+
+
+def make_token_http_database_for_test(test, replica_uid):
+ test.startServer()
+ test.request_state._create_database(replica_uid)
+
+ class _HTTPDatabaseWithToken(
+ http_database.HTTPDatabase, auth.TokenBasedAuth):
+
+ def set_token_credentials(self, uuid, token):
+ auth.TokenBasedAuth.set_token_credentials(self, uuid, token)
+
+ def _sign_request(self, method, url_query, params):
+ return auth.TokenBasedAuth._sign_request(
+ self, method, url_query, params)
+
+ http_db = _HTTPDatabaseWithToken(test.getURL('test'))
+ http_db.set_token_credentials('user-uuid', 'auth-token')
+ return http_db
+
+
+def copy_token_http_database_for_test(test, db):
+ # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS
+ # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE
+ # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN
+ # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR
+ # HOUSE.
+ http_db = test.request_state._copy_database(db)
+ http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token')
+ return http_db
+
+
+class MockedSharedDBTest(object):
+
+ def get_default_shared_mock(self, put_doc_side_effect=None,
+ get_doc_return_value=None):
+ """
+ Get a default class for mocking the shared DB
+ """
+ class defaultMockSharedDB(object):
+ get_doc = Mock(return_value=get_doc_return_value)
+ put_doc = Mock(side_effect=put_doc_side_effect)
+ lock = Mock(return_value=('atoken', 300))
+ unlock = Mock(return_value=True)
+ open = Mock(return_value=None)
+ syncable = True
+
+ def __call__(self):
+ return self
+ return defaultMockSharedDB
+
+
+def soledad_sync_target(test, path):
+ return target.SoledadSyncTarget(
+ test.getURL(path), crypto=test._soledad._crypto)
+
+
+def token_soledad_sync_target(test, path):
+ st = soledad_sync_target(test, path)
+ st.set_token_credentials('user-uuid', 'auth-token')
+ return st
+
+
+class BaseSoledadTest(unittest.TestCase, MockedSharedDBTest):
+ """
+ Instantiates Soledad for usage in tests.
+ """
+ defer_sync_encryption = False
+
+ def setUp(self):
+ # The following snippet comes from BaseLeapTest.setUpClass, but we
+ # repeat it here because twisted.trial does not work with
+ # setUpClass/tearDownClass.
+ self.old_path = os.environ['PATH']
+ self.old_home = os.environ['HOME']
+ self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
+ self.home = self.tempdir
+ bin_tdir = os.path.join(
+ self.tempdir,
+ 'bin')
+ os.environ["PATH"] = bin_tdir
+ os.environ["HOME"] = self.tempdir
+
+ # config info
+ self.db1_file = os.path.join(self.tempdir, "db1.u1db")
+ self.db2_file = os.path.join(self.tempdir, "db2.u1db")
+ self.email = ADDRESS
+ # open test dbs
+ self._db1 = u1db.open(self.db1_file, create=True,
+ document_factory=SoledadDocument)
+ self._db2 = u1db.open(self.db2_file, create=True,
+ document_factory=SoledadDocument)
+ # get a random prefix for each test, so we do not mess with
+ # concurrency during initialization and shutting down of
+ # each local db.
+ self.rand_prefix = ''.join(
+ map(lambda x: random.choice(string.ascii_letters), range(6)))
+ # initialize soledad by hand so we can control keys
+ # XXX check if this soledad is actually used
+ self._soledad = self._soledad_instance(
+ prefix=self.rand_prefix, user=self.email)
+
+ def tearDown(self):
+ self._db1.close()
+ self._db2.close()
+ self._soledad.close()
+
+ # restore paths
+ os.environ["PATH"] = self.old_path
+ os.environ["HOME"] = self.old_home
+
+ def _delete_temporary_dirs():
+ # XXX should not access "private" attrs
+ for f in [self._soledad.local_db_path,
+ self._soledad.secrets.secrets_path]:
+ if os.path.isfile(f):
+ os.unlink(f)
+ # The following snippet comes from BaseLeapTest.setUpClass, but we
+ # repeat it here because twisted.trial does not work with
+ # setUpClass/tearDownClass.
+ soledad_assert(
+ self.tempdir.startswith('/tmp/leap_tests-'),
+ "beware! tried to remove a dir which does not "
+ "live in temporal folder!")
+ shutil.rmtree(self.tempdir)
+
+ from twisted.internet import reactor
+ reactor.addSystemEventTrigger(
+ "after", "shutdown", _delete_temporary_dirs)
+
+
+ def _soledad_instance(self, user=ADDRESS, passphrase=u'123',
+ prefix='',
+ secrets_path='secrets.json',
+ local_db_path='soledad.u1db',
+ server_url='https://127.0.0.1/',
+ cert_file=None,
+ shared_db_class=None,
+ auth_token='auth-token'):
+
+ def _put_doc_side_effect(doc):
+ self._doc_put = doc
+
+ if shared_db_class is not None:
+ MockSharedDB = shared_db_class
+ else:
+ MockSharedDB = self.get_default_shared_mock(
+ _put_doc_side_effect)
+
+ return Soledad(
+ user,
+ passphrase,
+ secrets_path=os.path.join(
+ self.tempdir, prefix, secrets_path),
+ local_db_path=os.path.join(
+ self.tempdir, prefix, local_db_path),
+ server_url=server_url, # Soledad will fail if not given an url.
+ cert_file=cert_file,
+ defer_encryption=self.defer_sync_encryption,
+ shared_db=MockSharedDB(),
+ auth_token=auth_token)
+
+ def assertGetEncryptedDoc(
+ self, db, doc_id, doc_rev, content, has_conflicts):
+ """
+ Assert that the document in the database looks correct.
+ """
+ exp_doc = self.make_document(doc_id, doc_rev, content,
+ has_conflicts=has_conflicts)
+ doc = db.get_doc(doc_id)
+
+ if ENC_SCHEME_KEY in doc.content:
+ # XXX check for SYM_KEY too
+ key = self._soledad._crypto.doc_passphrase(doc.doc_id)
+ secret = self._soledad._crypto.secret
+ decrypted = decrypt_doc_dict(
+ doc.content, doc.doc_id, doc.rev,
+ key, secret)
+ doc.set_json(decrypted)
+ self.assertEqual(exp_doc.doc_id, doc.doc_id)
+ self.assertEqual(exp_doc.rev, doc.rev)
+ self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts)
+ self.assertEqual(exp_doc.content, doc.content)
+
+
+#-----------------------------------------------------------------------------
+# A wrapper for running couchdb locally.
+#-----------------------------------------------------------------------------
+
+# from: https://github.com/smcq/paisley/blob/master/paisley/test/util.py
+# TODO: include license of above project.
+class CouchDBWrapper(object):
+ """
+ Wrapper for external CouchDB instance which is started and stopped for
+ testing.
+ """
+ BOOT_TIMEOUT_SECONDS = 5
+ RETRY_LIMIT = 3
+
+ def start(self):
+ tries = 0
+ while tries < self.RETRY_LIMIT and not hasattr(self, 'port'):
+ try:
+ self._try_start()
+ return
+ except Exception, e:
+ print traceback.format_exc()
+ self.stop()
+ tries += 1
+ raise Exception("Check your couchdb: Tried to start 3 times and failed badly")
+
+ def _try_start(self):
+ """
+ Start a CouchDB instance for a test.
+ """
+ self.tempdir = tempfile.mkdtemp(suffix='.couch.test')
+
+ path = os.path.join(os.path.dirname(__file__),
+ 'couchdb.ini.template')
+ handle = open(path)
+ conf = handle.read() % {
+ 'tempdir': self.tempdir,
+ }
+ handle.close()
+
+ shutil.copy('/etc/couchdb/default.ini', self.tempdir)
+ defaultConfPath = os.path.join(self.tempdir, 'default.ini')
+
+ confPath = os.path.join(self.tempdir, 'test.ini')
+ handle = open(confPath, 'w')
+ handle.write(conf)
+ handle.close()
+
+ # create the dirs from the template
+ mkdir_p(os.path.join(self.tempdir, 'lib'))
+ mkdir_p(os.path.join(self.tempdir, 'log'))
+ args = ['/usr/bin/couchdb', '-n', '-a', defaultConfPath, '-a', confPath]
+ null = open('/dev/null', 'w')
+
+ self.process = subprocess.Popen(
+ args, env=None, stdout=null.fileno(), stderr=null.fileno(),
+ close_fds=True)
+ boot_time = time.time()
+ # find port
+ logPath = os.path.join(self.tempdir, 'log', 'couch.log')
+ while not os.path.exists(logPath):
+ if self.process.poll() is not None:
+ got_stdout, got_stderr = "", ""
+ if self.process.stdout is not None:
+ got_stdout = self.process.stdout.read()
+
+ if self.process.stderr is not None:
+ got_stderr = self.process.stderr.read()
+ raise Exception("""
+couchdb exited with code %d.
+stdout:
+%s
+stderr:
+%s""" % (
+ self.process.returncode, got_stdout, got_stderr))
+ time.sleep(0.01)
+ if (time.time() - boot_time) > self.BOOT_TIMEOUT_SECONDS:
+ self.stop()
+ raise Exception("Timeout starting couch")
+ while os.stat(logPath).st_size == 0:
+ time.sleep(0.01)
+ if (time.time() - boot_time) > self.BOOT_TIMEOUT_SECONDS:
+ self.stop()
+ raise Exception("Timeout starting couch")
+ PORT_RE = re.compile(
+ 'Apache CouchDB has started on http://127.0.0.1:(?P<port>\d+)')
+
+ handle = open(logPath)
+ line = handle.read()
+ handle.close()
+ m = PORT_RE.search(line)
+ if not m:
+ self.stop()
+ raise Exception("Cannot find port in line %s" % line)
+ self.port = int(m.group('port'))
+
+ def stop(self):
+ """
+ Terminate the CouchDB instance.
+ """
+ try:
+ self.process.terminate()
+ self.process.communicate()
+ except:
+ # just to clean up
+ # if it can't, the process wasn't created anyway
+ pass
+ shutil.rmtree(self.tempdir)
+
+
+class CouchDBTestCase(unittest.TestCase, MockedSharedDBTest):
+ """
+ TestCase base class for tests against a real CouchDB server.
+ """
+
+ def setUp(self):
+ """
+ Make sure we have a CouchDB instance for a test.
+ """
+ self.wrapper = CouchDBWrapper()
+ self.wrapper.start()
+ #self.db = self.wrapper.db
+
+ def tearDown(self):
+ """
+ Stop CouchDB instance for test.
+ """
+ self.wrapper.stop()
+
class CouchServerStateForTests(CouchServerState):
"""
This is a slightly modified CouchDB server state that allows for creating
@@ -122,43 +462,15 @@ class SoledadWithCouchServerMixin(
BaseSoledadTest,
CouchDBTestCase):
- @classmethod
- def setUpClass(cls):
- """
- Make sure we have a CouchDB instance for a test.
- """
- # from BaseLeapTest
- cls.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
- # from CouchDBTestCase
- cls.wrapper = CouchDBWrapper()
- cls.wrapper.start()
- #self.db = self.wrapper.db
-
- @classmethod
- def tearDownClass(cls):
- """
- Stop CouchDB instance for test.
- """
- # from BaseLeapTest
- soledad_assert(
- cls.tempdir.startswith('/tmp/leap_tests-'),
- "beware! tried to remove a dir which does not "
- "live in temporal folder!")
- shutil.rmtree(cls.tempdir)
- # from CouchDBTestCase
- cls.wrapper.stop()
-
def setUp(self):
- BaseSoledadTest.setUp(self)
CouchDBTestCase.setUp(self)
+ BaseSoledadTest.setUp(self)
main_test_class = getattr(self, 'main_test_class', None)
if main_test_class is not None:
main_test_class.setUp(self)
self._couch_url = 'http://localhost:%d' % self.wrapper.port
def tearDown(self):
- BaseSoledadTest.tearDown(self)
- CouchDBTestCase.tearDown(self)
main_test_class = getattr(self, 'main_test_class', None)
if main_test_class is not None:
main_test_class.tearDown(self)
@@ -168,10 +480,11 @@ class SoledadWithCouchServerMixin(
db.delete_database()
except DatabaseDoesNotExist:
pass
+ BaseSoledadTest.tearDown(self)
+ CouchDBTestCase.tearDown(self)
def make_app(self):
couch_url = urljoin(
'http://localhost:' + str(self.wrapper.port), 'tests')
- self.request_state = CouchServerStateForTests(
- couch_url, 'shared', 'tokens')
+ self.request_state = CouchServerStateForTests(couch_url)
return self.make_app_with_state(self.request_state)
diff --git a/docs/debian-repackaging.rst b/docs/debian-repackaging.rst
index a7488a84..888d6c03 100644
--- a/docs/debian-repackaging.rst
+++ b/docs/debian-repackaging.rst
@@ -6,7 +6,7 @@ How to repackage latest code
Enter debian branch::
- git checkout debian
+ git checkout debian/experimental
Merge your latest and greatest::
diff --git a/docs/leap-commit-template b/docs/leap-commit-template
new file mode 100644
index 00000000..8a5c7cd0
--- /dev/null
+++ b/docs/leap-commit-template
@@ -0,0 +1,7 @@
+[bug|feat|docs|style|refactor|test|pkg|i18n] ...
+...
+
+- Resolves: #XYZ
+- Related: #XYZ
+- Documentation: #XYZ
+- Releases: XYZ
diff --git a/docs/leap-commit-template.README b/docs/leap-commit-template.README
new file mode 100644
index 00000000..ce8809e7
--- /dev/null
+++ b/docs/leap-commit-template.README
@@ -0,0 +1,47 @@
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+HOW TO USE THIS TEMPLATE:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Run `git config commit.template docs/leap-commit-template` or
+edit the .git/config for this project and add
+`template = docs/leap-commit-template`
+under the [commit] block
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+COMMIT TEMPLATE FORMAT EXPLAINED
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+[type] <subject>
+
+<body>
+<footer>
+
+Type should be one of the following:
+- bug (bug fix)
+- feat (new feature)
+- docs (changes to documentation)
+- style (formatting, pep8 violations, etc; no code change)
+- refactor (refactoring production code)
+- test (adding missing tests, refactoring tests; no production code change)
+- pkg (packaging related changes; no production code change)
+- i18n translation related changes
+
+Subject should use imperative tone and say what you did.
+For example, use 'change', NOT 'changed' or 'changes'.
+
+The body should go into detail about changes made.
+
+The footer should contain any issue references or actions.
+You can use one or several of the following:
+
+- Resolves: #XYZ
+- Related: #XYZ
+- Documentation: #XYZ
+- Releases: XYZ
+
+The Documentation field should be included in every new feature commit, and it
+should link to an issue in the bug tracker where the new feature is analyzed
+and documented.
+
+For a full example of how to write a good commit message, check out
+https://github.com/sparkbox/how_to/tree/master/style/git
diff --git a/scripts/build_debian_package.sh b/scripts/build_debian_package.sh
index 1ec9b00a..b9fb93a9 100755
--- a/scripts/build_debian_package.sh
+++ b/scripts/build_debian_package.sh
@@ -26,7 +26,7 @@ export GIT_DIR=${workdir}/soledad/.git
export GIT_WORK_TREE=${workdir}/soledad
git remote add leapcode ${SOLEDAD_MAIN_REPO}
git fetch leapcode
-git checkout -b debian leapcode/debian
+git checkout -b debian/experimental leapcode/debian/experimental
git merge --no-edit ${branch}
(cd ${workdir}/soledad && debuild -uc -us)
echo "Packages generated in ${workdir}"
diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py
index 6c456c41..1d8d32e2 100644
--- a/scripts/db_access/client_side_db.py
+++ b/scripts/db_access/client_side_db.py
@@ -2,23 +2,22 @@
# This script gives client-side access to one Soledad user database.
-
-import sys
import os
import argparse
-import re
import tempfile
import getpass
import requests
-import json
import srp._pysrp as srp
import binascii
import logging
+import json
+from twisted.internet import reactor
+from twisted.internet.defer import inlineCallbacks
-from leap.common.config import get_path_prefix
from leap.soledad.client import Soledad
-
+from leap.keymanager import KeyManager
+from leap.keymanager.openpgp import OpenPGPKey
from util import ValidateUserHandle
@@ -26,37 +25,37 @@ from util import ValidateUserHandle
# create a logger
logger = logging.getLogger(__name__)
LOG_FORMAT = '%(asctime)s %(message)s'
-logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG)
safe_unhexlify = lambda x: binascii.unhexlify(x) if (
len(x) % 2 == 0) else binascii.unhexlify('0' + x)
-def fail(reason):
+def _fail(reason):
logger.error('Fail: ' + reason)
exit(2)
-def get_api_info(provider):
+def _get_api_info(provider):
info = requests.get(
'https://'+provider+'/provider.json', verify=False).json()
return info['api_uri'], info['api_version']
-def login(username, passphrase, provider, api_uri, api_version):
+def _login(username, passphrase, provider, api_uri, api_version):
usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024)
auth = None
try:
- auth = authenticate(api_uri, api_version, usr).json()
+ auth = _authenticate(api_uri, api_version, usr).json()
except requests.exceptions.ConnectionError:
- fail('Could not connect to server.')
+ _fail('Could not connect to server.')
if 'errors' in auth:
- fail(str(auth['errors']))
+ _fail(str(auth['errors']))
return api_uri, api_version, auth
-def authenticate(api_uri, api_version, usr):
+def _authenticate(api_uri, api_version, usr):
api_url = "%s/%s" % (api_uri, api_version)
session = requests.session()
uname, A = usr.start_authentication()
@@ -64,16 +63,16 @@ def authenticate(api_uri, api_version, usr):
init = session.post(
api_url + '/sessions', data=params, verify=False).json()
if 'errors' in init:
- fail('test user not found')
+ _fail('test user not found')
M = usr.process_challenge(
safe_unhexlify(init['salt']), safe_unhexlify(init['B']))
return session.put(api_url + '/sessions/' + uname, verify=False,
data={'client_auth': binascii.hexlify(M)})
-def get_soledad_info(username, provider, passphrase, basedir):
- api_uri, api_version = get_api_info(provider)
- auth = login(username, passphrase, provider, api_uri, api_version)
+def _get_soledad_info(username, provider, passphrase, basedir):
+ api_uri, api_version = _get_api_info(provider)
+ auth = _login(username, passphrase, provider, api_uri, api_version)
# get soledad server url
service_url = '%s/%s/config/soledad-service.json' % \
(api_uri, api_version)
@@ -101,10 +100,9 @@ def get_soledad_info(username, provider, passphrase, basedir):
return auth[2]['id'], server_url, cert_file, auth[2]['token']
-def get_soledad_instance(username, provider, passphrase, basedir):
+def _get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file,
+ token):
# setup soledad info
- uuid, server_url, cert_file, token = \
- get_soledad_info(username, provider, passphrase, basedir)
logger.info('UUID is %s' % uuid)
logger.info('Server URL is %s' % server_url)
secrets_path = os.path.join(
@@ -119,37 +117,135 @@ def get_soledad_instance(username, provider, passphrase, basedir):
local_db_path=local_db_path,
server_url=server_url,
cert_file=cert_file,
- auth_token=token)
-
-
-# main program
-
-if __name__ == '__main__':
-
+ auth_token=token,
+ defer_encryption=False)
+
+
+def _get_keymanager_instance(username, provider, soledad, token,
+ ca_cert_path=None, api_uri=None, api_version=None, uid=None,
+ gpgbinary=None):
+ return KeyManager(
+ "{username}@{provider}".format(username=username, provider=provider),
+ "http://uri",
+ soledad,
+ token=token,
+ ca_cert_path=ca_cert_path,
+ api_uri=api_uri,
+ api_version=api_version,
+ uid=uid,
+ gpgbinary=gpgbinary)
+
+
+def _parse_args():
# parse command line
parser = argparse.ArgumentParser()
parser.add_argument(
'user@provider', action=ValidateUserHandle, help='the user handle')
parser.add_argument(
- '-b', dest='basedir', required=False, default=None,
+ '--basedir', '-b', default=None,
help='soledad base directory')
parser.add_argument(
- '-p', dest='passphrase', required=False, default=None,
+ '--passphrase', '-p', default=None,
help='the user passphrase')
- args = parser.parse_args()
+ parser.add_argument(
+ '--get-all-docs', '-a', action='store_true',
+ help='get all documents from the local database')
+ parser.add_argument(
+ '--create-doc', '-c', default=None,
+ help='create a document with give content')
+ parser.add_argument(
+ '--sync', '-s', action='store_true',
+ help='synchronize with the server replica')
+ parser.add_argument(
+ '--export-public-key', help="export the public key to a file")
+ parser.add_argument(
+ '--export-private-key', help="export the private key to a file")
+ parser.add_argument(
+ '--export-incoming-messages',
+ help="export incoming messages to a directory")
+ return parser.parse_args()
- # get the password
+
+def _get_passphrase(args):
passphrase = args.passphrase
if passphrase is None:
passphrase = getpass.getpass(
'Password for %s@%s: ' % (args.username, args.provider))
+ return passphrase
+
- # get the basedir
+def _get_basedir(args):
basedir = args.basedir
if basedir is None:
basedir = tempfile.mkdtemp()
+ elif not os.path.isdir(basedir):
+ os.mkdir(basedir)
logger.info('Using %s as base directory.' % basedir)
+ return basedir
+
+
+@inlineCallbacks
+def _export_key(args, km, fname, private=False):
+ address = args.username + "@" + args.provider
+ pkey = yield km.get_key(address, OpenPGPKey, private=private, fetch_remote=False)
+ with open(args.export_private_key, "w") as f:
+ f.write(pkey.key_data)
+
+
+@inlineCallbacks
+def _export_incoming_messages(soledad, directory):
+ yield soledad.create_index("by-incoming", "bool(incoming)")
+ docs = yield soledad.get_from_index("by-incoming", '1')
+ i = 1
+ for doc in docs:
+ with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f:
+ f.write(doc.content["_enc_json"])
+ i += 1
+
+
+@inlineCallbacks
+def _get_all_docs(soledad):
+ _, docs = yield soledad.get_all_docs()
+ for doc in docs:
+ print json.dumps(doc.content, indent=4)
- # get the soledad instance
- s = get_soledad_instance(
- args.username, args.provider, passphrase, basedir)
+
+# main program
+
+@inlineCallbacks
+def _main(soledad, km, args):
+ try:
+ if args.create_doc:
+ yield soledad.create_doc({'content': args.create_doc})
+ if args.sync:
+ yield soledad.sync()
+ if args.get_all_docs:
+ yield _get_all_docs(soledad)
+ if args.export_private_key:
+ yield _export_key(args, km, args.export_private_key, private=True)
+ if args.export_public_key:
+ yield _export_key(args, km, args.expoert_public_key, private=False)
+ if args.export_incoming_messages:
+ yield _export_incoming_messages(soledad, args.export_incoming_messages)
+ except:
+ pass
+ finally:
+ reactor.stop()
+
+
+if __name__ == '__main__':
+ args = _parse_args()
+ passphrase = _get_passphrase(args)
+ basedir = _get_basedir(args)
+ uuid, server_url, cert_file, token = \
+ _get_soledad_info(args.username, args.provider, passphrase, basedir)
+ soledad = _get_soledad_instance(
+ uuid, passphrase, basedir, server_url, cert_file, token)
+ km = _get_keymanager_instance(
+ args.username,
+ args.provider,
+ soledad,
+ token,
+ uid=uuid)
+ _main(soledad, km, args)
+ reactor.run()
diff --git a/scripts/db_access/reset_db.py b/scripts/db_access/reset_db.py
index 80871856..7c6d281b 100644
--- a/scripts/db_access/reset_db.py
+++ b/scripts/db_access/reset_db.py
@@ -5,20 +5,21 @@
# WARNING: running this script over a database will delete all documents but
# the one with id u1db_config (which contains db metadata) and design docs
# needed for couch backend.
+#
+# Run it like this to get some help:
+#
+# ./reset_db.py --help
-import sys
-from ConfigParser import ConfigParser
import threading
import logging
-from couchdb import Database as CouchDatabase
-
+import argparse
+import re
-if len(sys.argv) != 2:
- print 'Usage: %s <uuid>' % sys.argv[0]
- exit(1)
-uuid = sys.argv[1]
+from ConfigParser import ConfigParser
+from couchdb import Database as CouchDatabase
+from couchdb import Server as CouchServer
# create a logger
@@ -27,23 +28,6 @@ LOG_FORMAT = '%(asctime)s %(message)s'
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
-# get couch url
-cp = ConfigParser()
-cp.read('/etc/leap/soledad-server.conf')
-url = cp.get('soledad-server', 'couch_url')
-
-
-# confirm
-yes = raw_input("Are you sure you want to reset the database for user %s "
- "(type YES)? " % uuid)
-if yes != 'YES':
- print 'Bailing out...'
- exit(2)
-
-
-db = CouchDatabase('%s/user-%s' % (url, uuid))
-
-
class _DeleterThread(threading.Thread):
def __init__(self, db, doc_id, release_fun):
@@ -59,21 +43,95 @@ class _DeleterThread(threading.Thread):
self._release_fun()
-semaphore_pool = threading.BoundedSemaphore(value=20)
-
-
-threads = []
-for doc_id in db:
- if doc_id != 'u1db_config' and not doc_id.startswith('_design'):
+def get_confirmation(noconfirm, uuid, shared):
+ msg = "Are you sure you want to reset %s (type YES)? "
+ if shared:
+ msg = msg % "the shared database"
+ elif uuid:
+ msg = msg % ("the database for user %s" % uuid)
+ else:
+ msg = msg % "all databases"
+ if noconfirm is False:
+ yes = raw_input(msg)
+ if yes != 'YES':
+ print 'Bailing out...'
+ exit(2)
+
+
+def get_url(empty):
+ url = None
+ if empty is False:
+ # get couch url
+ cp = ConfigParser()
+ cp.read('/etc/leap/soledad-server.conf')
+ url = cp.get('soledad-server', 'couch_url')
+ else:
+ with open('/etc/couchdb/couchdb.netrc') as f:
+ netrc = f.read()
+ admin_password = re.match('^.* password (.*)$', netrc).groups()[0]
+ url = 'http://admin:%s@127.0.0.1:5984' % admin_password
+ return url
+
+
+def reset_all_dbs(url, empty):
+ server = CouchServer('%s' % (url))
+ for dbname in server:
+ if dbname.startswith('user-') or dbname == 'shared':
+ reset_db(url, dbname, empty)
+
+
+def reset_db(url, dbname, empty):
+ db = CouchDatabase('%s/%s' % (url, dbname))
+ semaphore_pool = threading.BoundedSemaphore(value=20)
+
+ # launch threads for deleting docs
+ threads = []
+ for doc_id in db:
+ if empty is False:
+ if doc_id == 'u1db_config' or doc_id.startswith('_design'):
+ continue
semaphore_pool.acquire()
logger.info('[main] launching thread for doc: %s' % doc_id)
t = _DeleterThread(db, doc_id, semaphore_pool.release)
t.start()
threads.append(t)
-
-logger.info('[main] waiting for threads.')
-map(lambda thread: thread.join(), threads)
-
-
-logger.info('[main] done.')
+ # wait for threads to finish
+ logger.info('[main] waiting for threads.')
+ map(lambda thread: thread.join(), threads)
+ logger.info('[main] done.')
+
+
+def _parse_args():
+ parser = argparse.ArgumentParser()
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('-u', dest='uuid', default=False,
+ help='Reset database of given user.')
+ group.add_argument('-s', dest='shared', action='store_true', default=False,
+ help='Reset the shared database.')
+ group.add_argument('-a', dest='all', action='store_true', default=False,
+ help='Reset all user databases.')
+ parser.add_argument(
+ '-e', dest='empty', action='store_true', required=False, default=False,
+ help='Empty database (do not preserve minimal set of u1db documents).')
+ parser.add_argument(
+ '-y', dest='noconfirm', action='store_true', required=False,
+ default=False,
+ help='Do not ask for confirmation.')
+ return parser.parse_args(), parser
+
+
+if __name__ == '__main__':
+ args, parser = _parse_args()
+ if not (args.uuid or args.shared or args.all):
+ parser.print_help()
+ exit(1)
+
+ url = get_url(args.empty)
+ get_confirmation(args.noconfirm, args.uuid, args.shared)
+ if args.uuid:
+ reset_db(url, "user-%s" % args.uuid, args.empty)
+ elif args.shared:
+ reset_db(url, "shared", args.empty)
+ elif args.all:
+ reset_all_dbs(url, args.empty)
diff --git a/scripts/db_access/util.py b/scripts/db_access/util.py
new file mode 120000
index 00000000..368734f7
--- /dev/null
+++ b/scripts/db_access/util.py
@@ -0,0 +1 @@
+../profiling/util.py \ No newline at end of file
diff --git a/scripts/ddocs/update_design_docs.py b/scripts/ddocs/update_design_docs.py
index e7b5a29c..2e2fa8f0 100644
--- a/scripts/ddocs/update_design_docs.py
+++ b/scripts/ddocs/update_design_docs.py
@@ -11,84 +11,83 @@ import re
import threading
import binascii
-
+from urlparse import urlparse
from getpass import getpass
from ConfigParser import ConfigParser
-from couchdb.client import Server
-from couchdb.http import Resource, Session
-from datetime import datetime
-from urlparse import urlparse
+from couchdb.client import Server
+from couchdb.http import Resource
+from couchdb.http import Session
+from couchdb.http import ResourceNotFound
from leap.soledad.common import ddocs
-# parse command line for the log file name
-logger_fname = "/tmp/update-design-docs_%s.log" % \
- str(datetime.now()).replace(' ', '_')
-parser = argparse.ArgumentParser()
-parser.add_argument('--log', action='store', default=logger_fname, type=str,
- required=False, help='the name of the log file', nargs=1)
-args = parser.parse_args()
+MAX_THREADS = 20
+DESIGN_DOCS = {
+ '_design/docs': json.loads(binascii.a2b_base64(ddocs.docs)),
+ '_design/syncs': json.loads(binascii.a2b_base64(ddocs.syncs)),
+ '_design/transactions': json.loads(
+ binascii.a2b_base64(ddocs.transactions)),
+}
-# configure the logger
+# create a logger
logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-print "Logging to %s." % args.log
-logging.basicConfig(
- filename=args.log,
- format="%(asctime)-15s %(message)s")
+LOG_FORMAT = '%(asctime)s %(message)s'
+logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
-# configure threads
-max_threads = 20
-semaphore_pool = threading.BoundedSemaphore(value=max_threads)
-threads = []
+def _parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-u', dest='uuid', default=None, type=str,
+ help='the UUID of the user')
+ parser.add_argument('-t', dest='threads', default=MAX_THREADS, type=int,
+ help='the number of parallel threads')
+ return parser.parse_args()
-# get couch url
-cp = ConfigParser()
-cp.read('/etc/leap/soledad-server.conf')
-url = urlparse(cp.get('soledad-server', 'couch_url'))
-# get admin password
-netloc = re.sub('^.*@', '', url.netloc)
-url = url._replace(netloc=netloc)
-password = getpass("Admin password for %s: " % url.geturl())
-url = url._replace(netloc='admin:%s@%s' % (password, netloc))
+def _get_url():
+ # get couch url
+ cp = ConfigParser()
+ cp.read('/etc/leap/soledad-server.conf')
+ url = urlparse(cp.get('soledad-server', 'couch_url'))
+ # get admin password
+ netloc = re.sub('^.*@', '', url.netloc)
+ url = url._replace(netloc=netloc)
+ password = getpass("Admin password for %s: " % url.geturl())
+ return url._replace(netloc='admin:%s@%s' % (password, netloc))
-resource = Resource(url.geturl(), Session(retry_delays=[1,2,4,8], timeout=10))
-server = Server(url=resource)
-hidden_url = re.sub(
- 'http://(.*):.*@',
- 'http://\\1:xxxxx@',
- url.geturl())
+def _get_server(url):
+ resource = Resource(
+ url.geturl(), Session(retry_delays=[1, 2, 4, 8], timeout=10))
+ return Server(url=resource)
-print """
-==========
-ATTENTION!
-==========
-This script will modify Soledad's shared and user databases in:
+def _confirm(url):
+ hidden_url = re.sub(
+ 'http://(.*):.*@',
+ 'http://\\1:xxxxx@',
+ url.geturl())
- %s
+ print """
+ ==========
+ ATTENTION!
+ ==========
-This script does not make a backup of the couch db data, so make sure you
-have a copy or you may loose data.
-""" % hidden_url
-confirm = raw_input("Proceed (type uppercase YES)? ")
+ This script will modify Soledad's shared and user databases in:
-if confirm != "YES":
- exit(1)
+ %s
-# convert design doc content
+ This script does not make a backup of the couch db data, so make sure you
+ have a copy or you may loose data.
+ """ % hidden_url
+ confirm = raw_input("Proceed (type uppercase YES)? ")
+
+ if confirm != "YES":
+ exit(1)
-design_docs = {
- '_design/docs': json.loads(binascii.a2b_base64(ddocs.docs)),
- '_design/syncs': json.loads(binascii.a2b_base64(ddocs.syncs)),
- '_design/transactions': json.loads(binascii.a2b_base64(ddocs.transactions)),
-}
#
# Thread
@@ -106,42 +105,66 @@ class DBWorkerThread(threading.Thread):
def run(self):
- logger.info("(%d/%d) Updating db %s." % (self._db_idx, self._db_len,
- self._dbname))
+ logger.info(
+ "(%d/%d) Updating db %s."
+ % (self._db_idx, self._db_len, self._dbname))
- for doc_id in design_docs:
- doc = self._cdb[doc_id]
+ for doc_id in DESIGN_DOCS:
+ try:
+ doc = self._cdb[doc_id]
+ except ResourceNotFound:
+ doc = {'_id': doc_id}
for key in ['lists', 'views', 'updates']:
- if key in design_docs[doc_id]:
- doc[key] = design_docs[doc_id][key]
+ if key in DESIGN_DOCS[doc_id]:
+ doc[key] = DESIGN_DOCS[doc_id][key]
self._cdb.save(doc)
# release the semaphore
self._release_fun()
-db_idx = 0
-db_len = len(server)
-for dbname in server:
-
- db_idx += 1
-
- if not (dbname.startswith('user-') or dbname == 'shared') \
- or dbname == 'user-test-db':
- logger.info("(%d/%d) Skipping db %s." % (db_idx, db_len, dbname))
- continue
-
-
- # get access to couch db
- cdb = Server(url.geturl())[dbname]
-
- #---------------------------------------------------------------------
- # Start DB worker thread
- #---------------------------------------------------------------------
- semaphore_pool.acquire()
- thread = DBWorkerThread(server, dbname, db_idx, db_len, semaphore_pool.release)
+def _launch_update_design_docs_thread(
+ server, dbname, db_idx, db_len, semaphore_pool):
+ semaphore_pool.acquire() # wait for an available working slot
+ thread = DBWorkerThread(
+ server, dbname, db_idx, db_len, semaphore_pool.release)
thread.daemon = True
thread.start()
- threads.append(thread)
-
-map(lambda thread: thread.join(), threads)
+ return thread
+
+
+def _update_design_docs(args, server):
+
+ # find the actual databases to be updated
+ dbs = []
+ if args.uuid:
+ dbs.append('user-%s' % args.uuid)
+ else:
+ for dbname in server:
+ if dbname.startswith('user-') or dbname == 'shared':
+ dbs.append(dbname)
+ else:
+ logger.info("Skipping db %s." % dbname)
+
+ db_idx = 0
+ db_len = len(dbs)
+ semaphore_pool = threading.BoundedSemaphore(value=args.threads)
+ threads = []
+
+ # launch the update
+ for db in dbs:
+ db_idx += 1
+ threads.append(
+ _launch_update_design_docs_thread(
+ server, db, db_idx, db_len, semaphore_pool))
+
+ # wait for all threads to finish
+ map(lambda thread: thread.join(), threads)
+
+
+if __name__ == "__main__":
+ args = _parse_args()
+ url = _get_url()
+ _confirm(url)
+ server = _get_server(url)
+ _update_design_docs(args, server)
diff --git a/scripts/profiling/mail/__init__.py b/scripts/profiling/mail/__init__.py
new file mode 100644
index 00000000..352faae6
--- /dev/null
+++ b/scripts/profiling/mail/__init__.py
@@ -0,0 +1,184 @@
+import threading
+import time
+import logging
+import argparse
+
+from twisted.internet import reactor
+
+from util import log
+from couchdb_server import get_couchdb_wrapper_and_u1db
+from mx import put_lots_of_messages
+from soledad_server import get_soledad_server
+from soledad_client import SoledadClient
+from mail import get_imap_server
+
+
+UUID = 'blah'
+AUTH_TOKEN = 'bleh'
+
+
+logging.basicConfig(level=logging.DEBUG)
+
+modules = [
+ 'gnupg',
+ 'leap.common',
+ 'leap.keymanager',
+ 'taskthread',
+]
+
+for module in modules:
+ logger = logging.getLogger(name=module)
+ logger.setLevel(logging.WARNING)
+
+
+class TestWatcher(threading.Thread):
+
+ def __init__(self, couchdb_wrapper, couchdb_u1db, soledad_server,
+ soledad_client, imap_service, number_of_msgs, lock):
+ threading.Thread.__init__(self)
+ self._couchdb_wrapper = couchdb_wrapper
+ self._couchdb_u1db = couchdb_u1db
+ self._soledad_server = soledad_server
+ self._soledad_client = soledad_client
+ self._imap_service = imap_service
+ self._number_of_msgs = number_of_msgs
+ self._lock = lock
+ self._mails_available_time = None
+ self._mails_available_time_lock = threading.Lock()
+ self._conditions = None
+
+ def run(self):
+ self._set_conditions()
+ while not self._test_finished():
+ time.sleep(5)
+ log("TestWatcher: Tests finished, cleaning up...",
+ line_break=False)
+ self._stop_reactor()
+ self._cleanup()
+ log("done.")
+ self._lock.release()
+
+ def _set_conditions(self):
+ self._conditions = []
+
+ # condition 1: number of received messages is equal to number of
+ # expected messages
+ def _condition1(*args):
+ msgcount = self._imap_service._inbox.getMessageCount()
+ cond = msgcount == self._number_of_msgs
+ log("[condition 1] received messages: %d (expected: %d) :: %s"
+ % (msgcount, self._number_of_msgs, cond))
+ if self.mails_available_time == None \
+ and cond:
+ with self._mails_available_time_lock:
+ self._mails_available_time = time.time()
+ return cond
+
+
+ # condition 2: number of documents in server is equal to in client
+ def _condition2(client_docs, server_docs):
+ cond = client_docs == server_docs
+ log("[condition 2] number of documents: client %d; server %d :: %s"
+ % (client_docs, server_docs, cond))
+ return cond
+
+ # condition 3: number of documents bigger than 3 x number of msgs
+ def _condition3(client_docs, *args):
+ cond = client_docs > (2 * self._number_of_msgs)
+ log("[condition 3] documents (%d) > 2 * msgs (%d) :: %s"
+ % (client_docs, self._number_of_msgs, cond))
+ return cond
+
+ # condition 4: not syncing
+ def _condition4(*args):
+ cond = not self._soledad_client.instance.syncing
+ log("[condition 4] not syncing :: %s" % cond)
+ return cond
+
+ self._conditions.append(_condition1)
+ self._conditions.append(_condition2)
+ self._conditions.append(_condition3)
+ self._conditions.append(_condition4)
+
+ def _test_finished(self):
+ client_docs = self._get_soledad_client_number_of_docs()
+ server_docs = self._get_couchdb_number_of_docs()
+ return not bool(filter(lambda x: not x(client_docs, server_docs),
+ self._conditions))
+
+ def _stop_reactor(self):
+ reactor.stop()
+
+ def _cleanup(self):
+ self._imap_service.stop()
+ self._soledad_client.close()
+ self._soledad_server.stop()
+ self._couchdb_wrapper.stop()
+
+ def _get_soledad_client_number_of_docs(self):
+ c = self._soledad_client.instance._db._db_handle.cursor()
+ c.execute('SELECT COUNT(*) FROM document WHERE content IS NOT NULL')
+ row = c.fetchone()
+ return int(row[0])
+
+ def _get_couchdb_number_of_docs(self):
+ couchdb = self._couchdb_u1db._database
+ view = couchdb.view('_all_docs', include_docs=True)
+ return len(filter(
+ lambda r: '_attachments' in r.values()[1]
+ and 'u1db_content' in r.values()[1]['_attachments'],
+ view.rows))
+
+ @property
+ def mails_available_time(self):
+ with self._mails_available_time_lock:
+ return self._mails_available_time
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('number_of_msgs', help="The number of documents",
+ type=int)
+ parser.add_argument('report_file', help="The name of the report file",
+ type=str)
+ args = parser.parse_args()
+
+ # start a couchdb server
+ couchdb_wrapper, couchdb_u1db = get_couchdb_wrapper_and_u1db(
+ UUID, AUTH_TOKEN)
+
+ put_time = put_lots_of_messages(couchdb_u1db, args.number_of_msgs)
+
+ soledad_server = get_soledad_server(couchdb_wrapper.port)
+
+ soledad_client = SoledadClient(
+ uuid='blah',
+ server_url='http://127.0.0.1:%d' % soledad_server.port,
+ auth_token=AUTH_TOKEN)
+
+ imap_service = get_imap_server(
+ soledad_client.instance, UUID, 'snowden@bitmask.net', AUTH_TOKEN)
+
+ lock = threading.Lock()
+ lock.acquire()
+ test_watcher = TestWatcher(
+ couchdb_wrapper, couchdb_u1db, soledad_server, soledad_client,
+ imap_service, args.number_of_msgs, lock)
+ test_watcher.start()
+
+ # reactor.run() will block until TestWatcher stops the reactor.
+ start_time = time.time()
+ reactor.run()
+ log("Reactor stopped.")
+ end_time = time.time()
+ lock.acquire()
+ mails_available_time = test_watcher.mails_available_time - start_time
+ sync_time = end_time - start_time
+ log("Total syncing time: %f" % sync_time)
+ log("# number_of_msgs put_time mails_available_time sync_time")
+ result = "%d %f %f %f" \
+ % (args.number_of_msgs, put_time, mails_available_time,
+ sync_time)
+ log(result)
+ with open(args.report_file, 'a') as f:
+ f.write(result + "\n")
diff --git a/scripts/profiling/mail/couchdb.ini.template b/scripts/profiling/mail/couchdb.ini.template
new file mode 100644
index 00000000..1fc2205b
--- /dev/null
+++ b/scripts/profiling/mail/couchdb.ini.template
@@ -0,0 +1,224 @@
+; etc/couchdb/default.ini.tpl. Generated from default.ini.tpl.in by configure.
+
+; Upgrading CouchDB will overwrite this file.
+
+[couchdb]
+database_dir = %(tempdir)s/lib
+view_index_dir = %(tempdir)s/lib
+max_document_size = 4294967296 ; 4 GB
+os_process_timeout = 120000 ; 120 seconds. for view and external servers.
+max_dbs_open = 100
+delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned
+uri_file = %(tempdir)s/lib/couch.uri
+file_compression = snappy
+
+[database_compaction]
+; larger buffer sizes can originate smaller files
+doc_buffer_size = 524288 ; value in bytes
+checkpoint_after = 5242880 ; checkpoint after every N bytes were written
+
+[view_compaction]
+; larger buffer sizes can originate smaller files
+keyvalue_buffer_size = 2097152 ; value in bytes
+
+[httpd]
+port = 0
+bind_address = 127.0.0.1
+authentication_handlers = {couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler}
+default_handler = {couch_httpd_db, handle_request}
+secure_rewrites = true
+vhost_global_handlers = _utils, _uuids, _session, _oauth, _users
+allow_jsonp = false
+; Options for the MochiWeb HTTP server.
+;server_options = [{backlog, 128}, {acceptor_pool_size, 16}]
+; For more socket options, consult Erlang's module 'inet' man page.
+;socket_options = [{recbuf, 262144}, {sndbuf, 262144}, {nodelay, true}]
+log_max_chunk_size = 1000000
+
+[log]
+file = %(tempdir)s/log/couch.log
+level = info
+include_sasl = true
+
+[couch_httpd_auth]
+authentication_db = _users
+authentication_redirect = /_utils/session.html
+require_valid_user = false
+timeout = 600 ; number of seconds before automatic logout
+auth_cache_size = 50 ; size is number of cache entries
+allow_persistent_cookies = false ; set to true to allow persistent cookies
+
+[couch_httpd_oauth]
+; If set to 'true', oauth token and consumer secrets will be looked up
+; in the authentication database (_users). These secrets are stored in
+; a top level property named "oauth" in user documents. Example:
+; {
+; "_id": "org.couchdb.user:joe",
+; "type": "user",
+; "name": "joe",
+; "password_sha": "fe95df1ca59a9b567bdca5cbaf8412abd6e06121",
+; "salt": "4e170ffeb6f34daecfd814dfb4001a73"
+; "roles": ["foo", "bar"],
+; "oauth": {
+; "consumer_keys": {
+; "consumerKey1": "key1Secret",
+; "consumerKey2": "key2Secret"
+; },
+; "tokens": {
+; "token1": "token1Secret",
+; "token2": "token2Secret"
+; }
+; }
+; }
+use_users_db = false
+
+[query_servers]
+; javascript = %(tempdir)s/server/main.js
+javascript = /usr/bin/couchjs /usr/share/couchdb/server/main.js
+coffeescript = /usr/bin/couchjs /usr/share/couchdb/server/main-coffee.js
+
+
+; Changing reduce_limit to false will disable reduce_limit.
+; If you think you're hitting reduce_limit with a "good" reduce function,
+; please let us know on the mailing list so we can fine tune the heuristic.
+[query_server_config]
+reduce_limit = true
+os_process_limit = 25
+
+[daemons]
+view_manager={couch_view, start_link, []}
+external_manager={couch_external_manager, start_link, []}
+query_servers={couch_query_servers, start_link, []}
+vhosts={couch_httpd_vhost, start_link, []}
+httpd={couch_httpd, start_link, []}
+stats_aggregator={couch_stats_aggregator, start, []}
+stats_collector={couch_stats_collector, start, []}
+uuids={couch_uuids, start, []}
+auth_cache={couch_auth_cache, start_link, []}
+replication_manager={couch_replication_manager, start_link, []}
+os_daemons={couch_os_daemons, start_link, []}
+compaction_daemon={couch_compaction_daemon, start_link, []}
+
+[httpd_global_handlers]
+/ = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>}
+
+_all_dbs = {couch_httpd_misc_handlers, handle_all_dbs_req}
+_active_tasks = {couch_httpd_misc_handlers, handle_task_status_req}
+_config = {couch_httpd_misc_handlers, handle_config_req}
+_replicate = {couch_httpd_replicator, handle_req}
+_uuids = {couch_httpd_misc_handlers, handle_uuids_req}
+_restart = {couch_httpd_misc_handlers, handle_restart_req}
+_stats = {couch_httpd_stats_handlers, handle_stats_req}
+_log = {couch_httpd_misc_handlers, handle_log_req}
+_session = {couch_httpd_auth, handle_session_req}
+_oauth = {couch_httpd_oauth, handle_oauth_req}
+
+[httpd_db_handlers]
+_view_cleanup = {couch_httpd_db, handle_view_cleanup_req}
+_compact = {couch_httpd_db, handle_compact_req}
+_design = {couch_httpd_db, handle_design_req}
+_temp_view = {couch_httpd_view, handle_temp_view_req}
+_changes = {couch_httpd_db, handle_changes_req}
+
+; The external module takes an optional argument allowing you to narrow it to a
+; single script. Otherwise the script name is inferred from the first path section
+; after _external's own path.
+; _mypath = {couch_httpd_external, handle_external_req, <<"mykey">>}
+; _external = {couch_httpd_external, handle_external_req}
+
+[httpd_design_handlers]
+_view = {couch_httpd_view, handle_view_req}
+_show = {couch_httpd_show, handle_doc_show_req}
+_list = {couch_httpd_show, handle_view_list_req}
+_info = {couch_httpd_db, handle_design_info_req}
+_rewrite = {couch_httpd_rewrite, handle_rewrite_req}
+_update = {couch_httpd_show, handle_doc_update_req}
+
+; enable external as an httpd handler, then link it with commands here.
+; note, this api is still under consideration.
+; [external]
+; mykey = /path/to/mycommand
+
+; Here you can setup commands for CouchDB to manage
+; while it is alive. It will attempt to keep each command
+; alive if it exits.
+; [os_daemons]
+; some_daemon_name = /path/to/script -with args
+
+
+[uuids]
+; Known algorithms:
+; random - 128 bits of random awesome
+; All awesome, all the time.
+; sequential - monotonically increasing ids with random increments
+; First 26 hex characters are random. Last 6 increment in
+; random amounts until an overflow occurs. On overflow, the
+; random prefix is regenerated and the process starts over.
+; utc_random - Time since Jan 1, 1970 UTC with microseconds
+; First 14 characters are the time in hex. Last 18 are random.
+algorithm = sequential
+
+[stats]
+; rate is in milliseconds
+rate = 1000
+; sample intervals are in seconds
+samples = [0, 60, 300, 900]
+
+[attachments]
+compression_level = 8 ; from 1 (lowest, fastest) to 9 (highest, slowest), 0 to disable compression
+compressible_types = text/*, application/javascript, application/json, application/xml
+
+[replicator]
+db = _replicator
+; Maximum replicaton retry count can be a non-negative integer or "infinity".
+max_replication_retry_count = 10
+; More worker processes can give higher network throughput but can also
+; imply more disk and network IO.
+worker_processes = 4
+; With lower batch sizes checkpoints are done more frequently. Lower batch sizes
+; also reduce the total amount of used RAM memory.
+worker_batch_size = 500
+; Maximum number of HTTP connections per replication.
+http_connections = 20
+; HTTP connection timeout per replication.
+; Even for very fast/reliable networks it might need to be increased if a remote
+; database is too busy.
+connection_timeout = 30000
+; If a request fails, the replicator will retry it up to N times.
+retries_per_request = 10
+; Some socket options that might boost performance in some scenarios:
+; {nodelay, boolean()}
+; {sndbuf, integer()}
+; {recbuf, integer()}
+; {priority, integer()}
+; See the `inet` Erlang module's man page for the full list of options.
+socket_options = [{keepalive, true}, {nodelay, false}]
+; Path to a file containing the user's certificate.
+;cert_file = /full/path/to/server_cert.pem
+; Path to file containing user's private PEM encoded key.
+;key_file = /full/path/to/server_key.pem
+; String containing the user's password. Only used if the private keyfile is password protected.
+;password = somepassword
+; Set to true to validate peer certificates.
+verify_ssl_certificates = false
+; File containing a list of peer trusted certificates (in the PEM format).
+;ssl_trusted_certificates_file = /etc/ssl/certs/ca-certificates.crt
+; Maximum peer certificate depth (must be set even if certificate validation is off).
+ssl_certificate_max_depth = 3
+
+[compaction_daemon]
+; The delay, in seconds, between each check for which database and view indexes
+; need to be compacted.
+check_interval = 300
+; If a database or view index file is smaller then this value (in bytes),
+; compaction will not happen. Very small files always have a very high
+; fragmentation therefore it's not worth to compact them.
+min_file_size = 131072
+
+[compactions]
+; List of compaction rules for the compaction daemon.
+
+
+;[admins]
+;testuser = -hashed-f50a252c12615697c5ed24ec5cd56b05d66fe91e,b05471ba260132953930cf9f97f327f5
+; pass for above user is 'testpass'
diff --git a/scripts/profiling/mail/couchdb_server.py b/scripts/profiling/mail/couchdb_server.py
new file mode 100644
index 00000000..2cf0a3fd
--- /dev/null
+++ b/scripts/profiling/mail/couchdb_server.py
@@ -0,0 +1,42 @@
+import hashlib
+import couchdb
+
+from leap.soledad.common.couch import CouchDatabase
+
+from util import log
+from couchdb_wrapper import CouchDBWrapper
+
+
+def start_couchdb_wrapper():
+ log("Starting couchdb... ", line_break=False)
+ couchdb_wrapper = CouchDBWrapper()
+ couchdb_wrapper.start()
+ log("couchdb started on port %d." % couchdb_wrapper.port)
+ return couchdb_wrapper
+
+
+def get_u1db_database(dbname, port):
+ return CouchDatabase.open_database(
+ 'http://127.0.0.1:%d/%s' % (port, dbname),
+ True,
+ ensure_ddocs=True)
+
+
+def create_tokens_database(port, uuid, token_value):
+ tokens_database = couchdb.Server(
+ 'http://127.0.0.1:%d' % port).create('tokens')
+ token = couchdb.Document()
+ token['_id'] = hashlib.sha512(token_value).hexdigest()
+ token['user_id'] = uuid
+ token['type'] = 'Token'
+ tokens_database.save(token)
+
+
+def get_couchdb_wrapper_and_u1db(uuid, token_value):
+ couchdb_wrapper = start_couchdb_wrapper()
+
+ couchdb_u1db = get_u1db_database('user-%s' % uuid, couchdb_wrapper.port)
+ get_u1db_database('shared', couchdb_wrapper.port)
+ create_tokens_database(couchdb_wrapper.port, uuid, token_value)
+
+ return couchdb_wrapper, couchdb_u1db
diff --git a/scripts/profiling/mail/couchdb_wrapper.py b/scripts/profiling/mail/couchdb_wrapper.py
new file mode 100644
index 00000000..cad1205b
--- /dev/null
+++ b/scripts/profiling/mail/couchdb_wrapper.py
@@ -0,0 +1,84 @@
+import re
+import os
+import tempfile
+import subprocess
+import time
+import shutil
+
+
+from leap.common.files import mkdir_p
+
+
+class CouchDBWrapper(object):
+ """
+ Wrapper for external CouchDB instance.
+ """
+
+ def start(self):
+ """
+ Start a CouchDB instance for a test.
+ """
+ self.tempdir = tempfile.mkdtemp(suffix='.couch.test')
+
+ path = os.path.join(os.path.dirname(__file__),
+ 'couchdb.ini.template')
+ handle = open(path)
+ conf = handle.read() % {
+ 'tempdir': self.tempdir,
+ }
+ handle.close()
+
+ confPath = os.path.join(self.tempdir, 'test.ini')
+ handle = open(confPath, 'w')
+ handle.write(conf)
+ handle.close()
+
+ # create the dirs from the template
+ mkdir_p(os.path.join(self.tempdir, 'lib'))
+ mkdir_p(os.path.join(self.tempdir, 'log'))
+ args = ['couchdb', '-n', '-a', confPath]
+ null = open('/dev/null', 'w')
+
+ self.process = subprocess.Popen(
+ args, env=None, stdout=null.fileno(), stderr=null.fileno(),
+ close_fds=True)
+ # find port
+ logPath = os.path.join(self.tempdir, 'log', 'couch.log')
+ while not os.path.exists(logPath):
+ if self.process.poll() is not None:
+ got_stdout, got_stderr = "", ""
+ if self.process.stdout is not None:
+ got_stdout = self.process.stdout.read()
+
+ if self.process.stderr is not None:
+ got_stderr = self.process.stderr.read()
+ raise Exception("""
+couchdb exited with code %d.
+stdout:
+%s
+stderr:
+%s""" % (
+ self.process.returncode, got_stdout, got_stderr))
+ time.sleep(0.01)
+ while os.stat(logPath).st_size == 0:
+ time.sleep(0.01)
+ PORT_RE = re.compile(
+ 'Apache CouchDB has started on http://127.0.0.1:(?P<port>\d+)')
+
+ handle = open(logPath)
+ m = None
+ line = handle.readline()
+ while m is None:
+ m = PORT_RE.search(line)
+ line = handle.readline()
+ handle.close()
+ self.port = int(m.group('port'))
+
+ def stop(self):
+ """
+ Terminate the CouchDB instance.
+ """
+ self.process.terminate()
+ self.process.communicate()
+ shutil.rmtree(self.tempdir)
+
diff --git a/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub
new file mode 100644
index 00000000..fee53b6d
--- /dev/null
+++ b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub
@@ -0,0 +1,30 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.12 (GNU/Linux)
+
+mQENBFQEwmABCADC4wYD3mFt8xJtl3gjxRPEGN+FcgvzxxECIhyjYCHszrJu3f65
+/nyruriYdQLGR4YmUdERIwsZ7AMkAM1NAXe7sMq/gRPCb4PwrE7pRKzPAmaLeJMQ
+DC9CSCP+2gUmzeKHS71GkddcUI1HFr1AX9lLVW2ScvmSzOllenyUoFKRvz2uGkLG
+r5pvKsxJUHl9enpHRZV/0X5Y6PCinb4+eN2/ZTdpAywOycU+L+zflA0SOTCtf+dg
+8k839T30piuBulDLNeOX84YcyXTW7XeCeRTg/ryoFaYhbOGt68BwnP9xlpU62LW0
+8vzSZ0mLm4Ttz2uaALEoLmsa91nyLi9pLtrRABEBAAG0IEVkIFNub3dkZW4gPHNu
+b3dkZW5AYml0bWFzay5uZXQ+iQE4BBMBAgAiBQJUBMJgAhsDBgsJCAcDAgYVCAIJ
+CgsEFgIDAQIeAQIXgAAKCRAbRQ5mX+Y1cx4RCACzEiHpmknl+HnB3bHGcr8VZvU9
+hIoclVR/OBjWQFUynr66XmaMHMOLAVoZkIPnezWQe3gDY7QlFCNCfz8SC2++4WtB
+aBzal9IREkVnQBdnWalxLRviNH+zoFQ0URunBAyH4QAJRUC5tWfNj4yI6BCFPwXL
+o0CCISIN+VMRAnwjABQD840/TbcMHDqmJyk/vpPYPFQqQudN3eB2hphKUkZMistP
+O9++ui6glso+MgsbIUdqgnblM3FSrbjfLKekC+MeunFr8qRjettdaVyFD4GLg2SH
+/JpsjZKYoZStatpdJcrNjUMsGtXLxaCPl+VldNuOKIsA85TZJomMiaBDqG9YuQEN
+BFQEwmABCACrYiPXyGWHvs/aFKM63y9l6Th/+SKfzeq+ksLUI6fJIQytGORiiYZC
+1LrhOTmir+dY3IygkFlldxehGt/OMUKLB774WhBDRI43rAhImwhNutTIuUTO7DsD
+y7u83oVQH6xGZW5afs5BEU56Oa8DdUUA5gLfnpqAJG2mLB12JhClxzOYXK/VB0wJ
+QsIWl+zyN7uLQr5xZOthzvP6p7MmsAjhzU1imwyEm8s91DLhwonuqadkMGKi2qHW
+xuwxnr9aHQmobzy68/vOiBFeumr0YarirUdEDiUIti4rqy+0oteTNeMtXWo5rTtx
+xeayw+TjjaOT2fZ6CAbq0I+lOW0aJrPFABEBAAGJAR8EGAECAAkFAlQEwmACGwwA
+CgkQG0UOZl/mNXM0SggAuXzaLafCZiWx28K6mPKdgDOwTMm2rD7ukf3JiswlIyIU
+/K19BENu82iHRSu4nb9amhHOLEhaf1Ep2JTf2Trmd+/SNh0kv3dSBNjCrvrMvtcA
+qVxGc3DtRufGeRoy8ow/sEg+BCcfxJgR1efHOSQfMELDz2v8vbLbkR3Ubm7YRtKr
+Ri2HWYrAXRrwFC07yqO2zptCND/LBtnMrp08AOSSLpRWVD/Ww6IE1v1UEN53aGsm
+D+L/1XkuP4L9cqG3E2NYfsOPiblqRiKSe1adVid/rLn94u+fpE4kuvxoGKn1FJ/m
+FqU8aPtxvPbsMkSoNOalxqJGpuWRTXTLb5I+Ed2Szw==
+=yRE/
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec
new file mode 100644
index 00000000..64cb6c2a
--- /dev/null
+++ b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec
@@ -0,0 +1,57 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1.4.12 (GNU/Linux)
+
+lQOYBFQEwmABCADC4wYD3mFt8xJtl3gjxRPEGN+FcgvzxxECIhyjYCHszrJu3f65
+/nyruriYdQLGR4YmUdERIwsZ7AMkAM1NAXe7sMq/gRPCb4PwrE7pRKzPAmaLeJMQ
+DC9CSCP+2gUmzeKHS71GkddcUI1HFr1AX9lLVW2ScvmSzOllenyUoFKRvz2uGkLG
+r5pvKsxJUHl9enpHRZV/0X5Y6PCinb4+eN2/ZTdpAywOycU+L+zflA0SOTCtf+dg
+8k839T30piuBulDLNeOX84YcyXTW7XeCeRTg/ryoFaYhbOGt68BwnP9xlpU62LW0
+8vzSZ0mLm4Ttz2uaALEoLmsa91nyLi9pLtrRABEBAAEAB/0cLb885/amczMC7ZfN
+dD17aS1ImkjoIqxu5ofFh6zgFLLwHOEr+4QDQKhYQvL3wHfBKqtUEwET6nA50HPe
+4otxdAqczgkRYBZvwjpWuDtUY0B4giKhe2GJ7+xkeRmtlq9eaLEhdwzwqCUFVmBe
+4n0Ey4FgX4d+lmpY5fEFfHjz4bZpoCrNZKtiGtOqdlKXm8PnU+ek+G7DFuavJ+g5
+B4fiqkLAYFX/IDFfaTSBYzNDPbSQR5n4Q4r9PdKazPXg7bnLuxAIY4i6KEXq2YpS
+T1vLanCnBd4BEDUODCPZdc/AtbE0U+XoKTBjTvk3UEGIRJSsju8A1vWOG7UCl+0d
+UMmRBADaiQYnp9QiwPDbpqxzlWN8Ms/+tAyRnBbhghcRqtrDSke6fSJAqXzVGVmF
+FSJPMFf4mBYbr1U3YlYOJrlrb3tVhVN+7PTZDIaaENbtcsUAu7hTr7Ko6r1+WONC
+yhtrtOR9sWHVbTZ09ZvyvjHnBqZVA2PuZLUn2wrimnIJbVNdlwQA5EwgoS8UuDob
+hs6tLg29bAEDZRBHXQcDuEwdAX0KCHW0oQ0UE7exbDXXfQJSD9X3fDeqI+BdI+qQ
+Yuauz+fJxKl+qHAcy5l5NT7qomEjHCzjGUnn4NJzkn6a3T4SrBdSMFY2hL/tJN0i
+v1hXVNatjCEotqqsor+C6bf+Sl4I59cEAK+tYLTo/d+KOWtW4XbVhcYHjTBKtJGH
+p2/wNb49ibYpkgOUqW2ebiCB0Lg6QEupomcaMOJGol3v8vwBKsuwQJhWJrAXC2sT
+Bck5mI+DbabyAbYFtZgNHbcdDy62ADg1xD2Je7IjUDcpYaGB3VFhpD2rSvWDeSjR
+3jTG3PPINfoBODK0IEVkIFNub3dkZW4gPHNub3dkZW5AYml0bWFzay5uZXQ+iQE4
+BBMBAgAiBQJUBMJgAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAbRQ5m
+X+Y1cx4RCACzEiHpmknl+HnB3bHGcr8VZvU9hIoclVR/OBjWQFUynr66XmaMHMOL
+AVoZkIPnezWQe3gDY7QlFCNCfz8SC2++4WtBaBzal9IREkVnQBdnWalxLRviNH+z
+oFQ0URunBAyH4QAJRUC5tWfNj4yI6BCFPwXLo0CCISIN+VMRAnwjABQD840/TbcM
+HDqmJyk/vpPYPFQqQudN3eB2hphKUkZMistPO9++ui6glso+MgsbIUdqgnblM3FS
+rbjfLKekC+MeunFr8qRjettdaVyFD4GLg2SH/JpsjZKYoZStatpdJcrNjUMsGtXL
+xaCPl+VldNuOKIsA85TZJomMiaBDqG9YnQOYBFQEwmABCACrYiPXyGWHvs/aFKM6
+3y9l6Th/+SKfzeq+ksLUI6fJIQytGORiiYZC1LrhOTmir+dY3IygkFlldxehGt/O
+MUKLB774WhBDRI43rAhImwhNutTIuUTO7DsDy7u83oVQH6xGZW5afs5BEU56Oa8D
+dUUA5gLfnpqAJG2mLB12JhClxzOYXK/VB0wJQsIWl+zyN7uLQr5xZOthzvP6p7Mm
+sAjhzU1imwyEm8s91DLhwonuqadkMGKi2qHWxuwxnr9aHQmobzy68/vOiBFeumr0
+YarirUdEDiUIti4rqy+0oteTNeMtXWo5rTtxxeayw+TjjaOT2fZ6CAbq0I+lOW0a
+JrPFABEBAAEAB/4kyb13Z4MRyy37OkRakgdu2QvhfoVF59Hso/yxxFCTHibGLkpx
+82LQTDEsQNgkGZ2vp7IBElM6MkDuemIRtOW7icdesJh+lAPyI9moWi0DYGgmCQzh
+3PgDBdPQBDT6IL5eYw3323HjKjeeCW1NsPnFqlnyDe3MtWUbDyuozZ1ztA+Rekhb
+UhEDK8ZccEKwpzrE2H5zBZLeY0OKKROGnwd1RBVXnHMgVRF7vbellYaR4h2odxOp
+X8Ho4Xbs1h2VRNIuZwtfXxTIuTIfujlIPXMtVY40dgnEGt9PosJNr9IfGpfE3JCu
+k9PTvq37aZkQbYj52nccwKdos+sLQgqAdHhZBADHg7B5jyRRObsCUXQ+jMHXxuqT
+5l1twwOovvLC7YZoC8NAl4Bi0rh1Zj0ZEJJLFGzeiH+15C4qFTY+ospWpGu6X6g5
+I8ZWya8m2NSEWyJZNI1zKIU0iXucLevVTx+ctnovUNnb89v52/+BKr4k2iRISAzT
+7RL63aFTgnLw9GKweQQA2+eU5jcQ6LobPY/fZZImnhwLDq/OaUV+7u1RfB04GA15
+HOGQV77np/QTM6b+ezKTFhG/HMCTqxf+HPHfzohBPF9zvboLvCkqaHBDiV9qYE96
+id/el3ZeWloLcEe62sMGbv0YYmsYWgJxL8BFGw5v1QpYbfQCnXLjyG+/9f6Ygq0D
+/0W9X/NxWUyAXOv5KRy+rpkpNVxvie4tduvyVUa/9XHF7D/DMaXqkIvVX8yZUIDR
+bjuIvGZkZ9QP8zf8NKkB98zbqZi6CbNrerjrDpb7Pj7uQd3GIcjW4UmENGA6t7U9
+IWen966PAXSzh3996tRHxwXexVIEdX5n4pO39ZiodEIOPzmJAR8EGAECAAkFAlQE
+wmACGwwACgkQG0UOZl/mNXM0SggAuXzaLafCZiWx28K6mPKdgDOwTMm2rD7ukf3J
+iswlIyIU/K19BENu82iHRSu4nb9amhHOLEhaf1Ep2JTf2Trmd+/SNh0kv3dSBNjC
+rvrMvtcAqVxGc3DtRufGeRoy8ow/sEg+BCcfxJgR1efHOSQfMELDz2v8vbLbkR3U
+bm7YRtKrRi2HWYrAXRrwFC07yqO2zptCND/LBtnMrp08AOSSLpRWVD/Ww6IE1v1U
+EN53aGsmD+L/1XkuP4L9cqG3E2NYfsOPiblqRiKSe1adVid/rLn94u+fpE4kuvxo
+GKn1FJ/mFqU8aPtxvPbsMkSoNOalxqJGpuWRTXTLb5I+Ed2Szw==
+=9xZX
+-----END PGP PRIVATE KEY BLOCK-----
diff --git a/scripts/profiling/mail/mail.py b/scripts/profiling/mail/mail.py
new file mode 100644
index 00000000..8504c762
--- /dev/null
+++ b/scripts/profiling/mail/mail.py
@@ -0,0 +1,50 @@
+import os
+import threading
+
+from twisted.internet import reactor
+
+from leap.mail.imap.service import imap
+from leap.keymanager import KeyManager
+
+from util import log
+
+
+class IMAPServerThread(threading.Thread):
+ def __init__(self, imap_service):
+ threading.Thread.__init__(self)
+ self._imap_service = imap_service
+
+ def run(self):
+ self._imap_service.start_loop()
+ reactor.run()
+
+ def stop(self):
+ self._imap_service.stop()
+ reactor.stop()
+
+
+def get_imap_server(soledad, uuid, address, token):
+ log("Starting imap... ", line_break=False)
+
+ keymanager = KeyManager(address, '', soledad, token=token, uid=uuid)
+ with open(
+ os.path.join(
+ os.path.dirname(__file__),
+ 'keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec'), 'r') as f:
+ pubkey, privkey = keymanager.parse_openpgp_ascii_key(f.read())
+ keymanager.put_key(privkey)
+
+ imap_service, imap_port, imap_factory = imap.run_service(
+ soledad, keymanager, userid=address, offline=False)
+
+ imap_service.start_loop()
+ log("started.")
+ return imap_service
+
+ #imap_server = IMAPServerThread(imap_service)
+ #try:
+ # imap_server.start()
+ #except Exception as e:
+ # print str(e)
+
+ #return imap_server
diff --git a/scripts/profiling/mail/mx.py b/scripts/profiling/mail/mx.py
new file mode 100644
index 00000000..b6a1e5cf
--- /dev/null
+++ b/scripts/profiling/mail/mx.py
@@ -0,0 +1,80 @@
+import datetime
+import uuid
+import json
+import timeit
+
+
+from leap.keymanager import openpgp
+from leap.soledad.common.couch import CouchDocument
+from leap.soledad.common.crypto import (
+ EncryptionSchemes,
+ ENC_JSON_KEY,
+ ENC_SCHEME_KEY,
+)
+
+
+from util import log
+
+
+message = """To: Ed Snowden <snowden@bitmask.net>
+Date: %s
+From: Glenn Greenwald <greenwald@bitmask.net>
+
+hi!
+
+"""
+
+
+def get_message():
+ return message % datetime.datetime.now().strftime("%a %b %d %H:%M:%S:%f %Y")
+
+
+def get_enc_json(pubkey, message):
+ with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg:
+ gpg.import_keys(pubkey)
+ key = gpg.list_keys().pop()
+ # We don't care about the actual address, so we use a
+ # dummy one, we just care about the import of the pubkey
+ openpgp_key = openpgp._build_key_from_gpg("dummy@mail.com",
+ key, pubkey)
+ enc_json = str(gpg.encrypt(
+ json.dumps(
+ {'incoming': True, 'content': message},
+ ensure_ascii=False),
+ openpgp_key.fingerprint,
+ symmetric=False))
+ return enc_json
+
+
+def get_new_doc(enc_json):
+ doc = CouchDocument(doc_id=str(uuid.uuid4()))
+ doc.content = {
+ 'incoming': True,
+ ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
+ ENC_JSON_KEY: enc_json
+ }
+ return doc
+
+
+def get_pubkey():
+ with open('./keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub') as f:
+ return f.read()
+
+
+def put_one_message(pubkey, db):
+ enc_json = get_enc_json(pubkey, get_message())
+ doc = get_new_doc(enc_json)
+ db.put_doc(doc)
+
+
+def put_lots_of_messages(db, number):
+ log("Populating database with %d encrypted messages... "
+ % number, line_break=False)
+ pubkey = get_pubkey()
+ def _put_one_message():
+ put_one_message(pubkey, db)
+ time = timeit.timeit(_put_one_message, number=number)
+ log("done.")
+ average_time = time / number
+ log("put_one_message average time: %f" % average_time)
+ return average_time
diff --git a/scripts/profiling/mail/soledad_client.py b/scripts/profiling/mail/soledad_client.py
new file mode 100644
index 00000000..5ac8ce39
--- /dev/null
+++ b/scripts/profiling/mail/soledad_client.py
@@ -0,0 +1,40 @@
+import tempfile
+import os
+import shutil
+
+from leap.soledad.client import Soledad
+
+
+class SoledadClient(object):
+
+ def __init__(self, uuid, server_url, auth_token):
+ self._uuid = uuid
+ self._server_url = server_url
+ self._auth_token = auth_token
+ self._tempdir = None
+ self._soledad = None
+
+ @property
+ def instance(self):
+ if self._soledad is None:
+ self._soledad = self._get_soledad_client()
+ return self._soledad
+
+ def _get_soledad_client(self):
+ self._tempdir = tempfile.mkdtemp()
+ return Soledad(
+ uuid=self._uuid,
+ passphrase=u'123',
+ secrets_path=os.path.join(self._tempdir, 'secrets.json'),
+ local_db_path=os.path.join(self._tempdir, 'soledad.db'),
+ server_url=self._server_url,
+ cert_file=None,
+ auth_token=self._auth_token,
+ secret_id=None,
+ defer_encryption=True)
+
+ def close(self):
+ if self._soledad is not None:
+ self._soledad.close()
+ if self._tempdir is not None:
+ shutil.rmtree(self._tempdir)
diff --git a/scripts/profiling/mail/soledad_server.py b/scripts/profiling/mail/soledad_server.py
new file mode 100644
index 00000000..ad014456
--- /dev/null
+++ b/scripts/profiling/mail/soledad_server.py
@@ -0,0 +1,48 @@
+import threading
+
+from wsgiref.simple_server import make_server
+
+from leap.soledad.common.couch import CouchServerState
+
+from leap.soledad.server import SoledadApp
+from leap.soledad.server.gzip_middleware import GzipMiddleware
+from leap.soledad.server.auth import SoledadTokenAuthMiddleware
+
+from util import log
+
+
+class SoledadServerThread(threading.Thread):
+ def __init__(self, server):
+ threading.Thread.__init__(self)
+ self._server = server
+
+ def run(self):
+ self._server.serve_forever()
+
+ def stop(self):
+ self._server.shutdown()
+
+ @property
+ def port(self):
+ return self._server.server_port
+
+
+def make_soledad_server_thread(couch_port):
+ state = CouchServerState(
+ 'http://127.0.0.1:%d' % couch_port,
+ 'shared',
+ 'tokens')
+ application = GzipMiddleware(
+ SoledadTokenAuthMiddleware(SoledadApp(state)))
+ server = make_server('', 0, application)
+ t = SoledadServerThread(server)
+ return t
+
+
+def get_soledad_server(couchdb_port):
+ log("Starting soledad server... ", line_break=False)
+ soledad_server = make_soledad_server_thread(couchdb_port)
+ soledad_server.start()
+ log("soledad server started on port %d." % soledad_server.port)
+ return soledad_server
+
diff --git a/scripts/profiling/mail/util.py b/scripts/profiling/mail/util.py
new file mode 100644
index 00000000..86118e88
--- /dev/null
+++ b/scripts/profiling/mail/util.py
@@ -0,0 +1,8 @@
+import sys
+
+
+def log(msg, line_break=True):
+ sys.stdout.write(msg)
+ if line_break:
+ sys.stdout.write("\n")
+ sys.stdout.flush()
diff --git a/scripts/profiling/spam.py b/scripts/profiling/spam.py
new file mode 100755
index 00000000..091a8c48
--- /dev/null
+++ b/scripts/profiling/spam.py
@@ -0,0 +1,123 @@
+#!/usr/bin/python
+
+# Send a lot of messages in parallel.
+
+
+import string
+import smtplib
+import threading
+import logging
+
+from argparse import ArgumentParser
+
+
+SMTP_HOST = 'chipmonk.cdev.bitmask.net'
+NUMBER_OF_THREADS = 20
+
+
+logger = logging.getLogger(__name__)
+LOG_FORMAT = '%(asctime)s %(message)s'
+logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+
+
+def _send_email(host, subject, to_addr, from_addr, body_text):
+ """
+ Send an email
+ """
+ body = string.join((
+ "From: %s" % from_addr,
+ "To: %s" % to_addr,
+ "Subject: %s" % subject,
+ "",
+ body_text
+ ), "\r\n")
+ server = smtplib.SMTP(host)
+ server.sendmail(from_addr, [to_addr], body)
+ server.quit()
+
+
+def _parse_args():
+ parser = ArgumentParser()
+ parser.add_argument(
+ 'target_address',
+ help='The target email address to spam')
+ parser.add_argument(
+ 'number_of_messages', type=int,
+ help='The amount of messages email address to spam')
+ parser.add_argument(
+ '-s', dest='server', default=SMTP_HOST,
+ help='The SMTP server to use')
+ parser.add_argument(
+ '-t', dest='threads', default=NUMBER_OF_THREADS,
+ help='The maximum number of parallel threads to launch')
+ return parser.parse_args()
+
+
+class EmailSenderThread(threading.Thread):
+
+ def __init__(self, host, subject, to_addr, from_addr, body_text,
+ finished_fun):
+ threading.Thread.__init__(self)
+ self._host = host
+ self._subject = subject
+ self._to_addr = to_addr
+ self._from_addr = from_addr
+ self._body_text = body_text
+ self._finished_fun = finished_fun
+
+ def run(self):
+ _send_email(
+ self._host, self._subject, self._to_addr, self._from_addr,
+ self._body_text)
+ self._finished_fun()
+
+
+def _launch_email_thread(host, subject, to_addr, from_addr, body_text,
+ finished_fun):
+ thread = EmailSenderThread(
+ host, subject, to_addr, from_addr, body_text, finished_fun)
+ thread.start()
+ return thread
+
+
+class FinishedThreads(object):
+
+ def __init__(self):
+ self._finished = 0
+ self._lock = threading.Lock()
+
+ def signal(self):
+ with self._lock:
+ self._finished = self._finished + 1
+ logger.info('Number of messages sent: %d.' % self._finished)
+
+
+def _send_messages(args):
+ host = args.server
+ subject = "Message from Soledad script"
+ to_addr = args.target_address
+ from_addr = args.target_address
+ body_text = "Test message"
+
+ semaphore = threading.Semaphore(args.threads)
+ threads = []
+ finished_threads = FinishedThreads()
+
+ def _finished_fun():
+ semaphore.release()
+ finished_threads.signal()
+
+ for i in xrange(args.number_of_messages):
+ semaphore.acquire()
+ threads.append(
+ _launch_email_thread(
+ host, subject, to_addr, from_addr, body_text,
+ _finished_fun))
+
+ for t in threads:
+ t.join()
+
+
+if __name__ == "__main__":
+ args = _parse_args()
+ _send_messages(args)
diff --git a/scripts/profiling/storage/benchmark-storage.py b/scripts/profiling/storage/benchmark-storage.py
new file mode 100644
index 00000000..79ee3270
--- /dev/null
+++ b/scripts/profiling/storage/benchmark-storage.py
@@ -0,0 +1,104 @@
+#!/usr/bin/python
+
+# scenarios:
+# 1. soledad instantiation time.
+# a. for unexisting db.
+# b. for existing db.
+# 2. soledad doc storage/retrieval.
+# a. 1 KB document.
+# b 10 KB.
+# c. 100 KB.
+# d. 1 MB.
+
+
+import logging
+import getpass
+import tempfile
+import argparse
+import shutil
+import timeit
+
+
+from util import ValidateUserHandle
+
+# benchmarking args
+REPEAT_NUMBER = 1000
+DOC_SIZE = 1024
+
+
+# create a logger
+logger = logging.getLogger(__name__)
+LOG_FORMAT = '%(asctime)s %(message)s'
+logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+
+
+def parse_args():
+ # parse command line
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ 'user@provider', action=ValidateUserHandle, help='the user handle')
+ parser.add_argument(
+ '-b', dest='basedir', required=False, default=None,
+ help='soledad base directory')
+ parser.add_argument(
+ '-p', dest='passphrase', required=False, default=None,
+ help='the user passphrase')
+ parser.add_argument(
+ '-l', dest='logfile', required=False, default='/tmp/benchhmark-storage.log',
+ help='the file to which write the benchmark logs')
+ args = parser.parse_args()
+ # get the password
+ passphrase = args.passphrase
+ if passphrase is None:
+ passphrase = getpass.getpass(
+ 'Password for %s@%s: ' % (args.username, args.provider))
+ # get the basedir
+ basedir = args.basedir
+ if basedir is None:
+ basedir = tempfile.mkdtemp()
+ logger.info('Using %s as base directory.' % basedir)
+
+ return args.username, args.provider, passphrase, basedir, args.logfile
+
+
+if __name__ == '__main__':
+ username, provider, passphrase, basedir, logfile = parse_args()
+ create_results = []
+ getall_results = []
+ for i in [1, 200, 400, 600, 800, 1000]:
+ tempdir = tempfile.mkdtemp(dir=basedir)
+ setup_common = """
+import os
+#from benchmark_storage_utils import benchmark_fun
+#from benchmark_storage_utils import get_soledad_instance
+from client_side_db import get_soledad_instance
+sol = get_soledad_instance('%s', '%s', '%s', '%s')
+ """ % (username, provider, passphrase, tempdir)
+
+ setup_create = setup_common + """
+content = {'data': os.urandom(%d/2).encode('hex')}
+""" % (DOC_SIZE * i)
+ time = timeit.timeit(
+ 'sol.create_doc(content);',
+ setup=setup_create, number=REPEAT_NUMBER)
+ create_results.append((DOC_SIZE*i, time))
+ print "CREATE: %d %f" % (DOC_SIZE*i, time)
+
+ setup_get = setup_common + """
+doc_ids = [doc.doc_id for doc in sol.get_all_docs()[1]]
+"""
+
+ time = timeit.timeit(
+ "[sol.get_doc(doc_id) for doc_id in doc_ids]",
+ setup=setup_get, number=1)
+ getall_results.append((DOC_SIZE*i, time))
+ print "GET_ALL: %d %f" % (DOC_SIZE*i, time)
+ shutil.rmtree(tempdir)
+ print "# size, time for creation of %d docs" % REPEAT_NUMBER
+ for size, time in create_results:
+ print size, time
+ print "# size, time for retrieval of %d docs" % REPEAT_NUMBER
+ for size, time in getall_results:
+ print size, time
+ shutil.rmtree(basedir)
+
diff --git a/scripts/profiling/storage/benchmark_storage_utils.py b/scripts/profiling/storage/benchmark_storage_utils.py
new file mode 100644
index 00000000..fa8bb658
--- /dev/null
+++ b/scripts/profiling/storage/benchmark_storage_utils.py
@@ -0,0 +1,4 @@
+from client_side_db import get_soledad_instance
+
+def benchmark_fun(sol, content):
+ sol.create_doc(content)
diff --git a/scripts/profiling/storage/client_side_db.py b/scripts/profiling/storage/client_side_db.py
new file mode 120000
index 00000000..9e49a7f0
--- /dev/null
+++ b/scripts/profiling/storage/client_side_db.py
@@ -0,0 +1 @@
+../../db_access/client_side_db.py \ No newline at end of file
diff --git a/scripts/profiling/storage/plot.py b/scripts/profiling/storage/plot.py
new file mode 100755
index 00000000..280b9375
--- /dev/null
+++ b/scripts/profiling/storage/plot.py
@@ -0,0 +1,94 @@
+#!/usr/bin/python
+
+
+# Create a plot of the results of running the ./benchmark-storage.py script.
+
+
+import argparse
+from matplotlib import pyplot as plt
+
+from sets import Set
+
+
+def plot(filename, subtitle=''):
+
+ # config the plot
+ plt.xlabel('doc size (KB)')
+ plt.ylabel('operation time (s)')
+ title = 'soledad 1000 docs creation/retrieval times'
+ if subtitle != '':
+ title += '- %s' % subtitle
+ plt.title(title)
+
+ x = Set()
+ ycreate = []
+ yget = []
+
+ ys = []
+ #ys.append((ycreate, 'creation time', 'r', '-'))
+ #ys.append((yget, 'retrieval time', 'b', '-'))
+
+ # read data from file
+ with open(filename, 'r') as f:
+ f.readline()
+ for i in xrange(6):
+ size, y = f.readline().strip().split(' ')
+ x.add(int(size))
+ ycreate.append(float(y))
+
+ f.readline()
+ for i in xrange(6):
+ size, y = f.readline().strip().split(' ')
+ x.add(int(size))
+ yget.append(float(y))
+
+ # get doc size in KB
+ x = list(x)
+ x.sort()
+ x = map(lambda val: val / 1024, x)
+
+ # get normalized results per KB
+ nycreate = []
+ nyget = []
+ for i in xrange(len(x)):
+ nycreate.append(ycreate[i]/x[i])
+ nyget.append(yget[i]/x[i])
+
+ ys.append((nycreate, 'creation time per KB', 'r', '-.'))
+ ys.append((nyget, 'retrieval time per KB', 'b', '-.'))
+
+ for y in ys:
+ kwargs = {
+ 'linewidth': 1.0,
+ 'marker': '.',
+ 'color': y[2],
+ 'linestyle': y[3],
+ }
+ # normalize by doc size
+ plt.plot(
+ x,
+ y[0],
+ label=y[1], **kwargs)
+
+ #plt.axes().get_xaxis().set_ticks(x)
+ #plt.axes().get_xaxis().set_ticklabels(x)
+
+ # annotate max and min values
+ plt.xlim(0, 1100)
+ #plt.ylim(0, 350)
+ plt.grid()
+ plt.legend()
+ plt.show()
+
+
+if __name__ == '__main__':
+ # parse command line
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ 'datafile',
+ help='the data file to plot')
+ parser.add_argument(
+ '-s', dest='subtitle', required=False, default='',
+ help='a subtitle for the plot')
+ args = parser.parse_args()
+ plot(args.datafile, args.subtitle)
diff --git a/scripts/profiling/storage/profile-format.py b/scripts/profiling/storage/profile-format.py
new file mode 100644
index 00000000..262a52ab
--- /dev/null
+++ b/scripts/profiling/storage/profile-format.py
@@ -0,0 +1,29 @@
+#!/usr/bin/python
+
+import argparse
+import pstats
+
+
+def parse_args():
+ # parse command line
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '-f', dest='statsfiles', action='append', required=True,
+ help='a stats file')
+ args = parser.parse_args()
+ return args.statsfiles
+
+
+def format_stats(statsfiles):
+ for f in statsfiles:
+ ps = pstats.Stats(f)
+ ps.strip_dirs()
+ ps.sort_stats('time')
+ ps.print_stats()
+ ps.sort_stats('cumulative')
+ ps.print_stats()
+
+
+if __name__ == '__main__':
+ statsfiles = parse_args()
+ format_stats(statsfiles)
diff --git a/scripts/profiling/storage/profile-storage.py b/scripts/profiling/storage/profile-storage.py
new file mode 100755
index 00000000..305e6d5a
--- /dev/null
+++ b/scripts/profiling/storage/profile-storage.py
@@ -0,0 +1,107 @@
+#!/usr/bin/python
+
+import os
+import logging
+import getpass
+import tempfile
+import argparse
+import cProfile
+import shutil
+import pstats
+import StringIO
+import datetime
+
+
+from client_side_db import get_soledad_instance
+from util import ValidateUserHandle
+
+# profiling args
+NUM_DOCS = 1
+DOC_SIZE = 1024**2
+
+
+# create a logger
+logger = logging.getLogger(__name__)
+LOG_FORMAT = '%(asctime)s %(message)s'
+logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
+
+
+def parse_args():
+ # parse command line
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ 'user@provider', action=ValidateUserHandle, help='the user handle')
+ parser.add_argument(
+ '-b', dest='basedir', required=False, default=None,
+ help='soledad base directory')
+ parser.add_argument(
+ '-p', dest='passphrase', required=False, default=None,
+ help='the user passphrase')
+ parser.add_argument(
+ '-d', dest='logdir', required=False, default='/tmp/',
+ help='the direcroty to which write the profile stats')
+ args = parser.parse_args()
+ # get the password
+ passphrase = args.passphrase
+ if passphrase is None:
+ passphrase = getpass.getpass(
+ 'Password for %s@%s: ' % (args.username, args.provider))
+ # get the basedir
+ basedir = args.basedir
+ if basedir is None:
+ basedir = tempfile.mkdtemp()
+ logger.info('Using %s as base directory.' % basedir)
+
+ return args.username, args.provider, passphrase, basedir, args.logdir
+
+created_docs = []
+
+def create_docs(sol, content):
+ for i in xrange(NUM_DOCS):
+ doc = sol.create_doc(content)
+ created_docs.append(doc.doc_id)
+
+def get_all_docs(sol):
+ for doc_id in created_docs:
+ sol.get_doc(doc_id)
+
+def do_profile(logdir, sol):
+ fname_prefix = os.path.join(
+ logdir,
+ "profile_%s" \
+ % datetime.datetime.now().strftime('%Y-%m-%d_%H-%m-%S'))
+
+ # profile create docs
+ content = {'data': os.urandom(DOC_SIZE/2).encode('hex')}
+ pr = cProfile.Profile()
+ pr.runcall(
+ create_docs,
+ sol, content)
+ s = StringIO.StringIO()
+ ps = pstats.Stats(pr, stream=s).sort_stats('cumulative')
+ ps.print_stats()
+ ps.dump_stats("%s_creation.stats" % fname_prefix)
+ print s.getvalue()
+
+ # profile get all docs
+ pr = cProfile.Profile()
+ pr.runcall(
+ get_all_docs,
+ sol)
+ s = StringIO.StringIO()
+ ps = pstats.Stats(pr, stream=s).sort_stats('cumulative')
+ ps.dump_stats("%s_retrieval.stats" % fname_prefix)
+ ps.print_stats()
+ print s.getvalue()
+
+
+if __name__ == '__main__':
+ username, provider, passphrase, basedir, logdir = parse_args()
+ sol = get_soledad_instance(
+ username,
+ provider,
+ passphrase,
+ basedir)
+ do_profile(logdir, sol)
+ shutil.rmtree(basedir)
+
diff --git a/scripts/profiling/storage/util.py b/scripts/profiling/storage/util.py
new file mode 120000
index 00000000..7f16d684
--- /dev/null
+++ b/scripts/profiling/storage/util.py
@@ -0,0 +1 @@
+../util.py \ No newline at end of file
diff --git a/scripts/profiling/sync/movingaverage.py b/scripts/profiling/sync/movingaverage.py
new file mode 120000
index 00000000..098b0a01
--- /dev/null
+++ b/scripts/profiling/sync/movingaverage.py
@@ -0,0 +1 @@
+../movingaverage.py \ No newline at end of file
diff --git a/scripts/profiling/sync/profile-decoupled.py b/scripts/profiling/sync/profile-decoupled.py
new file mode 100644
index 00000000..a844c3c6
--- /dev/null
+++ b/scripts/profiling/sync/profile-decoupled.py
@@ -0,0 +1,24 @@
+# test_name: soledad-sync
+# start_time: 2014-06-12 20:09:11.232317+00:00
+# elapsed_time total_cpu total_memory proc_cpu proc_memory
+0.000225 68.400000 46.100000 105.300000 0.527224 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.250167 0.000000 0.255160
+0.707006 76.200000 46.200000 90.000000 0.562369 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+1.413140 63.200000 46.100000 0.000000 0.360199 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+2.123962 0.000000 46.100000 0.000000 0.360199 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+2.833941 31.600000 46.100000 0.000000 0.360248 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+3.541532 5.300000 46.100000 0.000000 0.360298 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+4.253390 14.300000 46.100000 11.100000 0.360347 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+4.967365 5.000000 46.100000 0.000000 0.360347 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+5.680172 5.600000 46.100000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+6.390501 10.500000 46.100000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+7.101711 23.800000 46.000000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+7.810529 30.000000 46.000000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+8.517835 25.000000 46.100000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+9.227455 5.300000 46.000000 9.500000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+9.936479 9.500000 46.000000 10.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+10.645015 52.400000 46.200000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+11.355179 21.100000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+12.066252 36.800000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+12.777689 28.600000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+13.489886 0.000000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160
+# end_time: 2014-06-12 20:09:25.434677+00:00 \ No newline at end of file
diff --git a/run_tests.sh b/scripts/run_tests.sh
index e36466f8..e36466f8 100755
--- a/run_tests.sh
+++ b/scripts/run_tests.sh
diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip
index be5d156b..df6ad95d 100644
--- a/server/pkg/requirements.pip
+++ b/server/pkg/requirements.pip
@@ -4,19 +4,12 @@ simplejson
u1db
routes
PyOpenSSL<0.14
-
-# TODO: maybe we just want twisted-web?
-twisted>=12.0.0
+twisted
# leap deps -- bump me!
-leap.soledad.common>=0.3.0
-
-#
-# Things yet to fix:
-#
-
-# oauth is not strictly needed by us, but we need it
-# until u1db adds it to its release as a dep.
+leap.soledad.common>=0.6.5
+# XXX -- fix me!
+# oauth is not strictly needed by us, but we need it until u1db adds it to its
+# release as a dep.
oauth
-
diff --git a/server/pkg/soledad b/server/pkg/soledad-server
index ccb3e9b0..811ad55b 100644
--- a/server/pkg/soledad
+++ b/server/pkg/soledad-server
@@ -34,8 +34,8 @@ case "${1}" in
start)
echo -n "Starting soledad: twistd"
start-stop-daemon --start --quiet \
- --user=${USER} --group=${GROUP} \
--exec ${TWISTD_PATH} -- \
+ --uid=${USER} --gid=${GROUP} \
--pidfile=${PIDFILE} \
--logfile=${LOGFILE} \
web \
diff --git a/server/setup.py b/server/setup.py
index 573622ce..124ddd32 100644
--- a/server/setup.py
+++ b/server/setup.py
@@ -35,7 +35,7 @@ if isset('VIRTUAL_ENV') or isset('LEAP_SKIP_INIT'):
data_files = None
else:
# XXX this should go only for linux/mac
- data_files = [("/etc/init.d/", ["pkg/soledad"])]
+ data_files = [("/etc/init.d/", ["pkg/soledad-server"])]
trove_classifiers = (
diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py
index 57f600a1..7af4e54b 100644
--- a/server/src/leap/soledad/server/auth.py
+++ b/server/src/leap/soledad/server/auth.py
@@ -21,10 +21,10 @@ Authentication facilities for Soledad Server.
"""
+import time
import httplib
import simplejson as json
-
from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors
from abc import ABCMeta, abstractmethod
from routes.mapper import Mapper
@@ -32,12 +32,8 @@ from couchdb.client import Server
from twisted.python import log
from hashlib import sha512
-
-from leap.soledad.common import (
- SHARED_DB_NAME,
- SHARED_DB_LOCK_DOC_ID_PREFIX,
- USER_DB_PREFIX,
-)
+from leap.soledad.common import SHARED_DB_NAME
+from leap.soledad.common import USER_DB_PREFIX
from leap.soledad.common.errors import InvalidAuthTokenError
@@ -354,7 +350,8 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware):
Token based authentication.
"""
- TOKENS_DB = "tokens"
+ TOKENS_DB_PREFIX = "tokens_"
+ TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds
TOKENS_TYPE_KEY = "type"
TOKENS_TYPE_DEF = "Token"
TOKENS_USER_ID_KEY = "user_id"
@@ -414,7 +411,14 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware):
invalid.
"""
server = Server(url=self._app.state.couch_url)
- dbname = self.TOKENS_DB
+ # the tokens db rotates every 30 days, and the current db name is
+ # "tokens_NNN", where NNN is the number of seconds since epoch divided
+ # by the rotate period in seconds. When rotating, old and new tokens
+ # db coexist during a certain window of time and valid tokens are
+ # replicated from the old db to the new one. See:
+ # https://leap.se/code/issues/6785
+ dbname = self.TOKENS_DB_PREFIX + \
+ str(int(time.time() / self.TOKENS_DB_EXPIRE))
db = server[dbname]
# lookup key is a hash of the token to prevent timing attacks.
token = db.get(sha512(token).hexdigest())