From cfff46ff9becdbe5cf48816870e625ed253ecc57 Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 17 Sep 2017 12:08:25 -0300 Subject: [refactor] move tests to root of repository Tests entrypoint was in a testing/ subfolder in the root of the repository. This was made mainly because we had some common files for tests and we didn't want to ship them (files in testing/test_soledad, which is itself a python package. This sometimes causes errors when loading tests (it seems setuptools is confused with having one python package in a subdirectory of another). This commit moves the tests entrypoint to the root of the repository. Closes: #8952 --- tests/benchmarks/README.md | 51 + tests/benchmarks/assets/cert_default.conf | 15 + tests/benchmarks/conftest.py | 154 + tests/benchmarks/pytest.ini | 2 + tests/benchmarks/test_crypto.py | 109 + tests/benchmarks/test_legacy_vs_blobs.py | 305 + tests/benchmarks/test_misc.py | 9 + tests/benchmarks/test_resources.py | 50 + tests/benchmarks/test_sqlcipher.py | 47 + tests/benchmarks/test_sqlite_blobs_backend.py | 82 + tests/benchmarks/test_sync.py | 92 + tests/blobs/test_blob_manager.py | 177 + tests/blobs/test_decrypter_buffer.py | 72 + tests/blobs/test_fs_backend.py | 173 + tests/blobs/test_sqlcipher_client_backend.py | 75 + tests/client/__init__.py | 0 tests/client/test_api.py | 36 + tests/client/test_app.py | 52 + tests/client/test_attachments.py | 81 + tests/client/test_aux_methods.py | 132 + tests/client/test_crypto.py | 384 + tests/client/test_deprecated_crypto.py | 94 + tests/client/test_doc.py | 50 + tests/client/test_http.py | 60 + tests/client/test_https.py | 137 + tests/client/test_incoming_processing_flow.py | 197 + tests/client/test_recovery_code.py | 38 + tests/client/test_secrets.py | 166 + tests/client/test_shared_db.py | 40 + tests/client/test_signals.py | 149 + tests/client/test_soledad | 1 + tests/client/test_soledad_doc.py | 46 + tests/conftest.py | 398 + tests/couch/__init__.py | 0 tests/couch/common.py | 75 + tests/couch/test_atomicity.py | 370 + tests/couch/test_backend.py | 111 + tests/couch/test_command.py | 31 + tests/couch/test_ddocs.py | 60 + tests/couch/test_soledad | 1 + tests/couch/test_state.py | 32 + tests/couch/test_sync.py | 700 + tests/couch/test_sync_target.py | 343 + tests/pipes/test_pipes.py | 51 + tests/responsiveness/conftest.py | 20 + tests/responsiveness/elastic.py | 47 + tests/responsiveness/test_responsiveness.py | 50 + tests/responsiveness/watchdog.py | 53 + tests/server/__init__.py | 0 tests/server/test__resource.py | 85 + tests/server/test__server_info.py | 43 + tests/server/test_auth.py | 138 + tests/server/test_blobs_resource_validation.py | 63 + tests/server/test_blobs_server.py | 286 + tests/server/test_config.py | 70 + tests/server/test_incoming_flow_integration.py | 97 + tests/server/test_incoming_resource.py | 57 + tests/server/test_incoming_server.py | 92 + tests/server/test_server.py | 230 + tests/server/test_session.py | 195 + tests/server/test_shared_db.py | 69 + tests/server/test_soledad | 1 + tests/server/test_tac.py | 87 + tests/server/test_url_mapper.py | 133 + tests/sqlcipher/hacker_crackdown.txt | 13005 +++++++++++++++++++ tests/sqlcipher/test_async.py | 146 + tests/sqlcipher/test_backend.py | 677 + tests/sqlcipher/test_soledad | 1 + tests/sync/__init__.py | 0 tests/sync/test_soledad | 1 + tests/sync/test_sqlcipher_sync.py | 719 + tests/sync/test_sync.py | 233 + tests/sync/test_sync_mutex.py | 133 + tests/sync/test_sync_target.py | 968 ++ tests/test_soledad/__init__.py | 5 + tests/test_soledad/fixture_soledad.conf | 12 + tests/test_soledad/u1db_tests/README | 23 + tests/test_soledad/u1db_tests/__init__.py | 420 + tests/test_soledad/u1db_tests/test_backends.py | 1888 +++ tests/test_soledad/u1db_tests/test_document.py | 153 + tests/test_soledad/u1db_tests/test_http_client.py | 304 + .../test_soledad/u1db_tests/test_http_database.py | 233 + tests/test_soledad/u1db_tests/test_https.py | 105 + tests/test_soledad/u1db_tests/test_open.py | 74 + .../test_soledad/u1db_tests/testing-certs/Makefile | 35 + .../u1db_tests/testing-certs/cacert.pem | 58 + .../u1db_tests/testing-certs/testing.cert | 61 + .../u1db_tests/testing-certs/testing.key | 16 + tests/test_soledad/util.py | 399 + 89 files changed, 26733 insertions(+) create mode 100644 tests/benchmarks/README.md create mode 100644 tests/benchmarks/assets/cert_default.conf create mode 100644 tests/benchmarks/conftest.py create mode 100644 tests/benchmarks/pytest.ini create mode 100644 tests/benchmarks/test_crypto.py create mode 100644 tests/benchmarks/test_legacy_vs_blobs.py create mode 100644 tests/benchmarks/test_misc.py create mode 100644 tests/benchmarks/test_resources.py create mode 100644 tests/benchmarks/test_sqlcipher.py create mode 100644 tests/benchmarks/test_sqlite_blobs_backend.py create mode 100644 tests/benchmarks/test_sync.py create mode 100644 tests/blobs/test_blob_manager.py create mode 100644 tests/blobs/test_decrypter_buffer.py create mode 100644 tests/blobs/test_fs_backend.py create mode 100644 tests/blobs/test_sqlcipher_client_backend.py create mode 100644 tests/client/__init__.py create mode 100644 tests/client/test_api.py create mode 100644 tests/client/test_app.py create mode 100644 tests/client/test_attachments.py create mode 100644 tests/client/test_aux_methods.py create mode 100644 tests/client/test_crypto.py create mode 100644 tests/client/test_deprecated_crypto.py create mode 100644 tests/client/test_doc.py create mode 100644 tests/client/test_http.py create mode 100644 tests/client/test_https.py create mode 100644 tests/client/test_incoming_processing_flow.py create mode 100644 tests/client/test_recovery_code.py create mode 100644 tests/client/test_secrets.py create mode 100644 tests/client/test_shared_db.py create mode 100644 tests/client/test_signals.py create mode 120000 tests/client/test_soledad create mode 100644 tests/client/test_soledad_doc.py create mode 100644 tests/conftest.py create mode 100644 tests/couch/__init__.py create mode 100644 tests/couch/common.py create mode 100644 tests/couch/test_atomicity.py create mode 100644 tests/couch/test_backend.py create mode 100644 tests/couch/test_command.py create mode 100644 tests/couch/test_ddocs.py create mode 120000 tests/couch/test_soledad create mode 100644 tests/couch/test_state.py create mode 100644 tests/couch/test_sync.py create mode 100644 tests/couch/test_sync_target.py create mode 100644 tests/pipes/test_pipes.py create mode 100644 tests/responsiveness/conftest.py create mode 100644 tests/responsiveness/elastic.py create mode 100644 tests/responsiveness/test_responsiveness.py create mode 100644 tests/responsiveness/watchdog.py create mode 100644 tests/server/__init__.py create mode 100644 tests/server/test__resource.py create mode 100644 tests/server/test__server_info.py create mode 100644 tests/server/test_auth.py create mode 100644 tests/server/test_blobs_resource_validation.py create mode 100644 tests/server/test_blobs_server.py create mode 100644 tests/server/test_config.py create mode 100644 tests/server/test_incoming_flow_integration.py create mode 100644 tests/server/test_incoming_resource.py create mode 100644 tests/server/test_incoming_server.py create mode 100644 tests/server/test_server.py create mode 100644 tests/server/test_session.py create mode 100644 tests/server/test_shared_db.py create mode 120000 tests/server/test_soledad create mode 100644 tests/server/test_tac.py create mode 100644 tests/server/test_url_mapper.py create mode 100644 tests/sqlcipher/hacker_crackdown.txt create mode 100644 tests/sqlcipher/test_async.py create mode 100644 tests/sqlcipher/test_backend.py create mode 120000 tests/sqlcipher/test_soledad create mode 100644 tests/sync/__init__.py create mode 120000 tests/sync/test_soledad create mode 100644 tests/sync/test_sqlcipher_sync.py create mode 100644 tests/sync/test_sync.py create mode 100644 tests/sync/test_sync_mutex.py create mode 100644 tests/sync/test_sync_target.py create mode 100644 tests/test_soledad/__init__.py create mode 100644 tests/test_soledad/fixture_soledad.conf create mode 100644 tests/test_soledad/u1db_tests/README create mode 100644 tests/test_soledad/u1db_tests/__init__.py create mode 100644 tests/test_soledad/u1db_tests/test_backends.py create mode 100644 tests/test_soledad/u1db_tests/test_document.py create mode 100644 tests/test_soledad/u1db_tests/test_http_client.py create mode 100644 tests/test_soledad/u1db_tests/test_http_database.py create mode 100644 tests/test_soledad/u1db_tests/test_https.py create mode 100644 tests/test_soledad/u1db_tests/test_open.py create mode 100644 tests/test_soledad/u1db_tests/testing-certs/Makefile create mode 100644 tests/test_soledad/u1db_tests/testing-certs/cacert.pem create mode 100644 tests/test_soledad/u1db_tests/testing-certs/testing.cert create mode 100644 tests/test_soledad/u1db_tests/testing-certs/testing.key create mode 100644 tests/test_soledad/util.py (limited to 'tests') diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md new file mode 100644 index 00000000..b2465a78 --- /dev/null +++ b/tests/benchmarks/README.md @@ -0,0 +1,51 @@ +Benchmark tests +=============== + +This folder contains benchmark tests for Soledad. It aims to provide a fair +account on the time and resources taken to perform some actions. + +These benchmarks are built on top of `pytest-benchmark`, a `pytest` fixture that +provides means for running test functions multiple times and generating +reports. The results are printed to screen and also posted to elasticsearch. + +`pytest-benchmark` runs tests multiple times so it can provide meaningful +statistics for the time taken for a tipical run of a test function. The number +of times that the test is run can be manually or automatically configured. When +automatically configured, the number of runs is decided by taking into account +multiple `pytest-benchmark` configuration parameters. See the following page +for more details on how `pytest-benchmark` works: + + https://pytest-benchmark.readthedocs.io/en/stable/calibration.html + +Some graphs and analysis resulting from these tests can be seen on: + + https://benchmarks.leap.se/ + + +Resource consumption +-------------------- + +For each test, CPU and memory usage statistics are also collected, by querying +`cpu_percent()` and `memory_percent()` from `psutil.Process` for the current +test process. Some notes about the current resource consumption estimation process: + +* Currently, resources are measured for the whole set of rounds that a test + function is run. That means that the CPU and memory percentage include the + `pytest` and `pytest-benchmark` machinery overhead. Anyway, for now this might + provide a fair approximation of per-run test function resource usage. + +* CPU is measured before and after the run of the benchmark function and + returns the percentage that the currnet process occupied of the CPU time + between the two calls. + +* Memory is sampled during the benchmark run by a separate thread. Sampling + interval might have to be configured on a per-test basis, as different tests + take different times to execute (from milliseconds to tens of seconds). For + now, an interval of 0.1s seems to cover all tests. + + +Benchmarks website +------------------ + +To update the benchmarks website, see the documentation in +``../../../docs/misc/benchmarks-website.rst``. diff --git a/tests/benchmarks/assets/cert_default.conf b/tests/benchmarks/assets/cert_default.conf new file mode 100644 index 00000000..8043cea3 --- /dev/null +++ b/tests/benchmarks/assets/cert_default.conf @@ -0,0 +1,15 @@ +[ req ] +default_bits = 1024 +default_keyfile = keyfile.pem +distinguished_name = req_distinguished_name +prompt = no +output_password = mypass + +[ req_distinguished_name ] +C = GB +ST = Test State or Province +L = Test Locality +O = Organization Name +OU = Organizational Unit Name +CN = localhost +emailAddress = test@email.address diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py new file mode 100644 index 00000000..80eccb08 --- /dev/null +++ b/tests/benchmarks/conftest.py @@ -0,0 +1,154 @@ +import functools +import numpy +import os +import psutil +import pytest +import threading +import time + +from twisted.internet import threads, reactor + + +# +# pytest customizations +# + +# mark benchmark tests using their group names (thanks ionelmc! :) +def pytest_collection_modifyitems(items, config): + for item in items: + bench = item.get_marker("benchmark") + if bench and bench.kwargs.get('group'): + group = bench.kwargs['group'] + marker = getattr(pytest.mark, 'benchmark_' + group) + item.add_marker(marker) + + subdir = config.getoption('subdir') + if subdir == 'benchmarks': + # we have to manually setup the events server in order to be able to + # signal events. This is usually done by the enclosing application + # using soledad client (i.e. bitmask client). + from leap.common.events import server + server.ensure_server() + + +# +# benchmark fixtures +# + +@pytest.fixture() +def txbenchmark(monitored_benchmark): + def blockOnThread(*args, **kwargs): + return threads.deferToThread( + monitored_benchmark, threads.blockingCallFromThread, + reactor, *args, **kwargs) + return blockOnThread + + +@pytest.fixture() +def txbenchmark_with_setup(monitored_benchmark_with_setup): + def blockOnThreadWithSetup(setup, f, *args, **kwargs): + def blocking_runner(*args, **kwargs): + return threads.blockingCallFromThread(reactor, f, *args, **kwargs) + + def blocking_setup(): + args = threads.blockingCallFromThread(reactor, setup) + try: + return tuple(arg for arg in args), {} + except TypeError: + return ((args,), {}) if args else None + + def bench(): + return monitored_benchmark_with_setup( + blocking_runner, setup=blocking_setup, + rounds=4, warmup_rounds=1, iterations=1, + args=args, kwargs=kwargs) + return threads.deferToThread(bench) + return blockOnThreadWithSetup + + +# +# resource monitoring +# + +class ResourceWatcher(threading.Thread): + + sampling_interval = 0.1 + + def __init__(self, watch_memory): + threading.Thread.__init__(self) + self.process = psutil.Process(os.getpid()) + self.running = False + # monitored resources + self.cpu_percent = None + self.watch_memory = watch_memory + self.memory_samples = [] + self.memory_percent = None + + def run(self): + self.running = True + self.process.cpu_percent() + # decide how long to sleep based on need to sample memory + sleep = self.sampling_interval if not self.watch_memory else 1 + while self.running: + if self.watch_memory: + sample = self.process.memory_percent(memtype='rss') + self.memory_samples.append(sample) + time.sleep(sleep) + + def stop(self): + self.running = False + self.join() + # save cpu usage info + self.cpu_percent = self.process.cpu_percent() + # save memory usage info + if self.watch_memory: + memory_percent = { + 'sampling_interval': self.sampling_interval, + 'samples': self.memory_samples, + 'stats': {}, + } + for stat in 'max', 'min', 'mean', 'std': + fun = getattr(numpy, stat) + memory_percent['stats'][stat] = fun(self.memory_samples) + self.memory_percent = memory_percent + + +def _monitored_benchmark(benchmark_fixture, benchmark_function, request, + *args, **kwargs): + # setup resource monitoring + watch_memory = _watch_memory(request) + watcher = ResourceWatcher(watch_memory) + watcher.start() + # run benchmarking function + benchmark_function(*args, **kwargs) + # store results + watcher.stop() + benchmark_fixture.extra_info.update({ + 'cpu_percent': watcher.cpu_percent + }) + if watch_memory: + benchmark_fixture.extra_info.update({ + 'memory_percent': watcher.memory_percent, + }) + # add docstring info + if request.scope == 'function': + fun = request.function + doc = fun.__doc__ or '' + benchmark_fixture.extra_info.update({'doc': doc.strip()}) + + +def _watch_memory(request): + return request.config.getoption('--watch-memory') + + +@pytest.fixture +def monitored_benchmark(benchmark, request): + return functools.partial( + _monitored_benchmark, benchmark, benchmark, request) + + +@pytest.fixture +def monitored_benchmark_with_setup(benchmark, request, *args, **kwargs): + return functools.partial( + _monitored_benchmark, benchmark, benchmark.pedantic, request, + *args, **kwargs) diff --git a/tests/benchmarks/pytest.ini b/tests/benchmarks/pytest.ini new file mode 100644 index 00000000..7a0508ce --- /dev/null +++ b/tests/benchmarks/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +twisted = yes diff --git a/tests/benchmarks/test_crypto.py b/tests/benchmarks/test_crypto.py new file mode 100644 index 00000000..3be447a5 --- /dev/null +++ b/tests/benchmarks/test_crypto.py @@ -0,0 +1,109 @@ +""" +Benchmarks for crypto operations. +If you don't want to stress your local machine too much, you can pass the +SIZE_LIMT environment variable. + +For instance, to keep the maximum payload at 1MB: + +SIZE_LIMIT=1E6 py.test -s tests/perf/test_crypto.py +""" +import pytest +import os +import json +from uuid import uuid4 + +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client import _crypto + +LIMIT = int(float(os.environ.get('SIZE_LIMIT', 50 * 1000 * 1000))) + + +def create_doc_encryption(size): + @pytest.mark.benchmark(group="test_crypto_encrypt_doc") + @pytest.inlineCallbacks + def test_doc_encryption(soledad_client, txbenchmark, payload): + """ + Encrypt a document of a given size. + """ + crypto = soledad_client()._crypto + + DOC_CONTENT = {'payload': payload(size)} + doc = SoledadDocument( + doc_id=uuid4().hex, rev='rev', + json=json.dumps(DOC_CONTENT)) + + yield txbenchmark(crypto.encrypt_doc, doc) + return test_doc_encryption + + +# TODO this test is really bullshit, because it's still including +# the json serialization. + +def create_doc_decryption(size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_crypto_decrypt_doc") + def test_doc_decryption(soledad_client, txbenchmark, payload): + """ + Decrypt a document of a given size. + """ + crypto = soledad_client()._crypto + + DOC_CONTENT = {'payload': payload(size)} + doc = SoledadDocument( + doc_id=uuid4().hex, rev='rev', + json=json.dumps(DOC_CONTENT)) + + encrypted_doc = yield crypto.encrypt_doc(doc) + doc.set_json(encrypted_doc) + + yield txbenchmark(crypto.decrypt_doc, doc) + return test_doc_decryption + + +def create_raw_encryption(size): + @pytest.mark.benchmark(group="test_crypto_raw_encrypt") + def test_raw_encrypt(monitored_benchmark, payload): + """ + Encrypt raw payload using default mode from crypto module. + """ + key = payload(32) + monitored_benchmark(_crypto.encrypt_sym, payload(size), key) + return test_raw_encrypt + + +def create_raw_decryption(size): + @pytest.mark.benchmark(group="test_crypto_raw_decrypt") + def test_raw_decrypt(monitored_benchmark, payload): + """ + Decrypt raw payload using default mode from crypto module. + """ + key = payload(32) + iv, ciphertext = _crypto.encrypt_sym(payload(size), key) + monitored_benchmark(_crypto.decrypt_sym, ciphertext, key, iv) + return test_raw_decrypt + + +# Create the TESTS in the global namespace, they'll be picked by the benchmark +# plugin. + +encryption_tests = [ + ('10k', 1E4), + ('100k', 1E5), + ('500k', 5E5), + ('1M', 1E6), + ('10M', 1E7), + ('50M', 5E7), +] + +for name, size in encryption_tests: + if size < LIMIT: + sz = int(size) + globals()['test_encrypt_doc_' + name] = create_doc_encryption(sz) + globals()['test_decrypt_doc_' + name] = create_doc_decryption(sz) + + +for name, size in encryption_tests: + if size < LIMIT: + sz = int(size) + globals()['test_encrypt_raw_' + name] = create_raw_encryption(sz) + globals()['test_decrypt_raw_' + name] = create_raw_decryption(sz) diff --git a/tests/benchmarks/test_legacy_vs_blobs.py b/tests/benchmarks/test_legacy_vs_blobs.py new file mode 100644 index 00000000..47d6482c --- /dev/null +++ b/tests/benchmarks/test_legacy_vs_blobs.py @@ -0,0 +1,305 @@ +# "Legacy" versus "Incoming blobs" pipeline comparison +# ==================================================== +# +# This benchmarking aims to compare the legacy and new mail incoming pipeline, +# to asses performance improvements brought by the introduction of blobs. +# +# We use the following sizes in these tests: +# +# - headers: 4 KB +# - metadata: 0.1 KB +# - flags: 0.5 KB +# - content: variable +# +# "Legacy" incoming mail pipeline: +# +# - email arrives at MX. +# - MX encrypts to public key and puts into couch. +# - pubkey encrypted doc is synced to soledad client as "incoming". +# - bitmask mail processes "incoming" and generates 3 metadocs + 1 payload +# doc per message. +# - soledad client syncs 4 documents back to server. +# +# "Incoming blobs" mail pipeline: +# +# - email arrives at MX. +# - MX encyrpts to public key and puts into soledad server. +# - soledad server writes a blob to filesystem. +# - soledad client gets the incoming blob from server and generates 3 +# metadocs + 1 blob. +# - soledad client syncs 3 meta documents and 1 blob back to server. +# +# Some notes about the tests in this file: +# +# - This is a simulation of the legacy and new incoming mail pipelines. +# There is no actual mail processing operation done (i.e. no pubkey crypto, +# no mail parsing), only usual soledad document manipulation and sync (with +# local 1network and crypto). +# +# - Each test simulates a whole incoming mail pipeline, including get new +# incoming messages from server, create new documents that represent the +# parsed message, and synchronize those back to the server. +# +# - These tests are disabled by default because it doesn't make much sense to +# have them run automatically for all commits in the repository. Instead, +# we will run them manually for specific releases and store results and +# analisys in a subfolder. + +import base64 +import pytest +import random +import sys +import treq +import uuid + +from io import BytesIO + +from twisted.internet.defer import gatherResults +from twisted.internet.defer import returnValue +from twisted.internet.defer import DeferredSemaphore + +from leap.soledad.common.blobs import Flags +from leap.soledad.client._db.blobs import BlobDoc + + +def payload(size): + random.seed(1337) # same seed to avoid different bench results + payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size)) + # encode as base64 to avoid ascii encode/decode errors + return base64.b64encode(payload_bytes)[:size] # remove b64 overhead + + +PARTS = { + 'headers': payload(4000), + 'metadata': payload(100), + 'flags': payload(500), +} + + +# +# "Legacy" incoming mail pipeline. +# + +@pytest.inlineCallbacks +def load_up_legacy(client, amount, content): + # make sure there are no document from previous runs + yield client.sync() + _, docs = yield client.get_all_docs() + deferreds = [] + for doc in docs: + d = client.delete_doc(doc) + deferreds.append(d) + yield gatherResults(deferreds) + yield client.sync() + + # create a bunch of local documents representing email messages + deferreds = [] + for i in xrange(amount): + deferreds.append(client.create_doc(content)) + yield gatherResults(deferreds) + yield client.sync() + + +@pytest.inlineCallbacks +def process_incoming_docs(client, docs): + deferreds = [] + for doc in docs: + + # create fake documents that represent message + for name in PARTS.keys(): + d = client.create_doc({name: doc.content[name]}) + deferreds.append(d) + + # create one document with content + key = 'content' + d = client.create_doc({key: doc.content[key]}) + deferreds.append(d) + + # delete the old incoming document + d = client.delete_doc(doc) + deferreds.append(d) + + # wait for all operatios to succeed + yield gatherResults(deferreds) + + +def create_legacy_test(amount, size): + group = 'test_legacy_vs_blobs_%d_%dk' % (amount, (size / 1000)) + + @pytest.inlineCallbacks + @pytest.mark.skip(reason="avoid running for all commits") + @pytest.mark.benchmark(group=group) + def test(soledad_client, txbenchmark_with_setup): + client = soledad_client() + + # setup the content of initial documents representing incoming emails + content = {'content': payload(size), 'incoming': True} + for name, data in PARTS.items(): + content[name] = data + + @pytest.inlineCallbacks + def setup(): + yield load_up_legacy(client, amount, content) + clean_client = soledad_client(force_fresh_db=True) + yield clean_client.create_index('incoming', 'bool(incoming)') + returnValue(clean_client) + + @pytest.inlineCallbacks + def legacy_pipeline(client): + yield client.sync() + docs = yield client.get_from_index('incoming', '1') + yield process_incoming_docs(client, docs) + yield client.sync() + + yield txbenchmark_with_setup(setup, legacy_pipeline) + return test + + +# ATTENTION: update the documentation in ../docs/benchmarks.rst if you change +# the number of docs or the doc sizes for the tests below. +test_legacy_10_1000k = create_legacy_test(10, 1000 * 1000) +test_legacy_100_100k = create_legacy_test(100, 100 * 1000) +test_legacy_1000_10k = create_legacy_test(1000, 10 * 1000) + + +# +# "Incoming blobs" mail pipeline: +# + +# used to limit the amount of concurrent accesses to the blob manager +semaphore = DeferredSemaphore(2) + + +# deliver data to a user by using the incoming api at given url. +def deliver_using_incoming_api(url, user_uuid, token, data): + auth = 'Token %s' % base64.b64encode('%s:%s' % (user_uuid, token)) + uri = "%s/incoming/%s/%s?namespace=MX" % (url, user_uuid, uuid.uuid4().hex) + return treq.put(uri, headers={'Authorization': auth}, data=BytesIO(data)) + + +# deliver data to a user by faking incoming using blobs +@pytest.inlineCallbacks +def deliver_using_blobs(client, fd): + # put + blob_id = uuid.uuid4().hex + doc = BlobDoc(fd, blob_id=blob_id) + size = sys.getsizeof(fd) + yield client.blobmanager.put(doc, size, namespace='MX') + # and flag + flags = [Flags.PENDING] + yield client.blobmanager.set_flags(blob_id, flags, namespace='MX') + + +def reclaim_free_space(client): + return client.blobmanager.local.dbpool.runQuery("VACUUM") + + +@pytest.inlineCallbacks +def load_up_blobs(client, amount, data): + # make sure there are no document from previous runs + yield client.sync() + _, docs = yield client.get_all_docs() + deferreds = [] + for doc in docs: + d = client.delete_doc(doc) + deferreds.append(d) + yield gatherResults(deferreds) + yield client.sync() + + # delete all payload from blobs db and server + for namespace in ['MX', 'payload']: + ids = yield client.blobmanager.remote_list(namespace=namespace) + deferreds = [] + for blob_id in ids: + d = semaphore.run( + client.blobmanager.delete, blob_id, namespace=namespace) + deferreds.append(d) + yield gatherResults(deferreds) + + # create a bunch of incoming blobs + deferreds = [] + for i in xrange(amount): + # choose method of delivery based in test being local or remote + if '127.0.0.1' in client.server_url: + fun = deliver_using_incoming_api + args = (client.server_url, client.uuid, client.token, data) + else: + fun = deliver_using_blobs + args = (client, BytesIO(data)) + d = semaphore.run(fun, *args) + deferreds.append(d) + yield gatherResults(deferreds) + + # empty local blobs db + yield client.blobmanager.local.dbpool.runQuery( + "DELETE FROM blobs WHERE 1;") + yield reclaim_free_space(client) + + +@pytest.inlineCallbacks +def process_incoming_blobs(client, pending): + # process items + deferreds = [] + for item in pending: + d = process_one_incoming_blob(client, item) + deferreds.append(d) + yield gatherResults(deferreds) + + +@pytest.inlineCallbacks +def process_one_incoming_blob(client, item): + fd = yield semaphore.run( + client.blobmanager.get, item, namespace='MX') + + # create metadata docs + deferreds = [] + for name, data in PARTS.items(): + d = client.create_doc({name: data}) + deferreds.append(d) + + # put the incoming blob as it would be done after mail processing + doc = BlobDoc(fd, blob_id=uuid.uuid4().hex) + size = sys.getsizeof(fd) + d = semaphore.run( + client.blobmanager.put, doc, size, namespace='payload') + deferreds.append(d) + yield gatherResults(deferreds) + + # delete incoming blob + yield semaphore.run( + client.blobmanager.delete, item, namespace='MX') + + +def create_blobs_test(amount, size): + group = 'test_legacy_vs_blobs_%d_%dk' % (amount, (size / 1000)) + + @pytest.inlineCallbacks + @pytest.mark.skip(reason="avoid running for all commits") + @pytest.mark.benchmark(group=group) + def test(soledad_client, txbenchmark_with_setup): + client = soledad_client() + blob_payload = payload(size) + + @pytest.inlineCallbacks + def setup(): + yield load_up_blobs(client, amount, blob_payload) + returnValue(soledad_client(force_fresh_db=True)) + + @pytest.inlineCallbacks + def blobs_pipeline(client): + pending = yield client.blobmanager.remote_list( + namespace='MX', filter_flags=Flags.PENDING) + yield process_incoming_blobs(client, pending) + # reclaim_free_space(client) + yield client.sync() + yield client.blobmanager.send_missing(namespace='payload') + + yield txbenchmark_with_setup(setup, blobs_pipeline) + return test + + +# ATTENTION: update the documentation in ../docs/benchmarks.rst if you change +# the number of docs or the doc sizes for the tests below. +test_blobs_10_1000k = create_blobs_test(10, 1000 * 1000) +test_blobs_100_100k = create_blobs_test(100, 100 * 1000) +test_blobs_1000_10k = create_blobs_test(1000, 10 * 1000) diff --git a/tests/benchmarks/test_misc.py b/tests/benchmarks/test_misc.py new file mode 100644 index 00000000..8b2178b9 --- /dev/null +++ b/tests/benchmarks/test_misc.py @@ -0,0 +1,9 @@ +import pytest + + +@pytest.mark.benchmark(group="test_instance") +def test_initialization(soledad_client, monitored_benchmark): + """ + Soledad client object initialization. + """ + monitored_benchmark(soledad_client) diff --git a/tests/benchmarks/test_resources.py b/tests/benchmarks/test_resources.py new file mode 100644 index 00000000..173edbd1 --- /dev/null +++ b/tests/benchmarks/test_resources.py @@ -0,0 +1,50 @@ +import pytest +import random +import time + +from decimal import Decimal + + +def bellardBig(n): + # http://en.wikipedia.org/wiki/Bellard%27s_formula + pi = Decimal(0) + k = 0 + while k < n: + pi += (Decimal(-1) ** k / (1024 ** k)) * ( + Decimal(256) / (10 * k + 1) + + Decimal(1) / (10 * k + 9) - + Decimal(64) / (10 * k + 3) - + Decimal(32) / (4 * k + 1) - + Decimal(4) / (10 * k + 5) - + Decimal(4) / (10 * k + 7) - + Decimal(1) / (4 * k + 3)) + k += 1 + pi = pi * 1 / (2 ** 6) + return pi + + +@pytest.mark.skip(reason='not a real use case, used only for instrumentation') +def test_cpu_intensive(monitored_benchmark): + + def _cpu_intensive(): + sleep = [random.uniform(0.5, 1.5) for _ in xrange(3)] + while sleep: + t = sleep.pop() + time.sleep(t) + bellardBig(int((10 ** 3) * t)) + + monitored_benchmark(_cpu_intensive) + + +@pytest.mark.skip(reason='not a real use case, used only for instrumentation') +def test_memory_intensive(monitored_benchmark): + + def _memory_intensive(): + sleep = [random.uniform(0.5, 1.5) for _ in xrange(3)] + bigdata = "" + while sleep: + t = sleep.pop() + bigdata += "b" * 10 * int(10E6) + time.sleep(t) + + monitored_benchmark(_memory_intensive) diff --git a/tests/benchmarks/test_sqlcipher.py b/tests/benchmarks/test_sqlcipher.py new file mode 100644 index 00000000..9108084c --- /dev/null +++ b/tests/benchmarks/test_sqlcipher.py @@ -0,0 +1,47 @@ +''' +Tests SoledadClient/SQLCipher interaction +''' +import pytest + +from twisted.internet.defer import gatherResults + + +def load_up(client, amount, payload, defer=True): + results = [client.create_doc({'content': payload}) for _ in xrange(amount)] + if defer: + return gatherResults(results) + + +def build_test_sqlcipher_async_create(amount, size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_sqlcipher_async_create") + def test(soledad_client, txbenchmark_with_setup, payload): + """ + Create many documents of a given size concurrently. + """ + client = soledad_client() + yield txbenchmark_with_setup( + lambda: None, load_up, client, amount, payload(size)) + return test + + +def build_test_sqlcipher_create(amount, size): + @pytest.mark.skip(reason="this test is lengthy and not a real use case") + @pytest.mark.benchmark(group="test_sqlcipher_create") + def test(soledad_client, monitored_benchmark, payload): + """ + Create many documents of a given size serially. + """ + client = soledad_client()._dbsyncer + monitored_benchmark( + load_up, client, amount, payload(size), defer=False) + return test + + +test_async_create_10_1000k = build_test_sqlcipher_async_create(10, 1000 * 1000) +test_async_create_100_100k = build_test_sqlcipher_async_create(100, 100 * 1000) +test_async_create_1000_10k = build_test_sqlcipher_async_create(1000, 10 * 1000) +# synchronous +test_create_10_1000k = build_test_sqlcipher_create(10, 1000 * 1000) +test_create_100_100k = build_test_sqlcipher_create(100, 100 * 1000) +test_create_1000_10k = build_test_sqlcipher_create(1000, 10 * 1000) diff --git a/tests/benchmarks/test_sqlite_blobs_backend.py b/tests/benchmarks/test_sqlite_blobs_backend.py new file mode 100644 index 00000000..e02cacad --- /dev/null +++ b/tests/benchmarks/test_sqlite_blobs_backend.py @@ -0,0 +1,82 @@ +import pytest +import os + +from uuid import uuid4 +from io import BytesIO + +from twisted.internet.defer import gatherResults +from twisted.internet.defer import DeferredSemaphore + +from leap.soledad.client._db.blobs import SQLiteBlobBackend + + +semaphore = DeferredSemaphore(2) + + +# +# put +# + +def put(backend, amount, data): + deferreds = [] + for _ in xrange(amount): + blob_id = uuid4().hex + fd = BytesIO(data) + size = len(data) + d = semaphore.run(backend.put, blob_id, fd, size) + deferreds.append(d) + return gatherResults(deferreds) + + +def create_put_test(amount, size): + + @pytest.inlineCallbacks + @pytest.mark.sqlite_blobs_backend_put + def test(txbenchmark, payload, tmpdir): + dbpath = os.path.join(tmpdir.strpath, 'blobs.db') + backend = SQLiteBlobBackend(dbpath, key='123') + data = payload(size) + yield txbenchmark(put, backend, amount, data) + + return test + + +test_sqlite_blobs_backend_put_1_10000k = create_put_test(1, 10000 * 1000) +test_sqlite_blobs_backend_put_10_1000k = create_put_test(10, 1000 * 1000) +test_sqlite_blobs_backend_put_100_100k = create_put_test(100, 100 * 1000) +test_sqlite_blobs_backend_put_1000_10k = create_put_test(1000, 10 * 1000) + + +# +# put +# + +@pytest.inlineCallbacks +def get(backend): + local = yield backend.list() + deferreds = [] + for blob_id in local: + d = backend.get(blob_id) + deferreds.append(d) + yield gatherResults(deferreds) + + +def create_get_test(amount, size): + + @pytest.inlineCallbacks + @pytest.mark.sqlite_blobs_backend_get + def test(txbenchmark, payload, tmpdir): + dbpath = os.path.join(tmpdir.strpath, 'blobs.db') + backend = SQLiteBlobBackend(dbpath, key='123') + data = payload(size) + + yield put(backend, amount, data) + yield txbenchmark(get, backend) + + return test + + +test_sqlite_blobs_backend_get_1_10000k = create_get_test(1, 10000 * 1000) +test_sqlite_blobs_backend_get_10_1000k = create_get_test(10, 1000 * 1000) +test_sqlite_blobs_backend_get_100_100k = create_get_test(100, 100 * 1000) +test_sqlite_blobs_backend_get_1000_10k = create_get_test(1000, 10 * 1000) diff --git a/tests/benchmarks/test_sync.py b/tests/benchmarks/test_sync.py new file mode 100644 index 00000000..45506d77 --- /dev/null +++ b/tests/benchmarks/test_sync.py @@ -0,0 +1,92 @@ +import pytest +from twisted.internet.defer import gatherResults + + +@pytest.inlineCallbacks +def load_up(client, amount, payload): + # create a bunch of local documents + deferreds = [] + for i in xrange(amount): + deferreds.append(client.create_doc({'content': payload})) + yield gatherResults(deferreds) + + +# Each test created with this function will: +# +# - get a fresh client. +# - iterate: +# - setup: create N docs of a certain size +# - benchmark: sync() -- uploads N docs. +def create_upload(uploads, size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_upload") + def test(soledad_client, txbenchmark_with_setup, payload): + """ + Upload many documents of a given size. + """ + client = soledad_client() + + def setup(): + return load_up(client, uploads, payload(size)) + + yield txbenchmark_with_setup(setup, client.sync) + return test + + +# ATTENTION: update the documentation in ../docs/benchmarks.rst if you change +# the number of docs or the doc sizes for the tests below. +test_upload_10_1000k = create_upload(10, 1000 * 1000) +test_upload_100_100k = create_upload(100, 100 * 1000) +test_upload_1000_10k = create_upload(1000, 10 * 1000) + + +# Each test created with this function will: +# +# - get a fresh client. +# - create N docs of a certain size +# - sync (uploads those docs) +# - iterate: +# - setup: get a fresh client with empty local db +# - benchmark: sync() -- downloads N docs. +def create_download(downloads, size): + @pytest.inlineCallbacks + @pytest.mark.benchmark(group="test_download") + def test(soledad_client, txbenchmark_with_setup, payload): + """ + Download many documents of the same size. + """ + client = soledad_client() + + yield load_up(client, downloads, payload(size)) + yield client.sync() + # We could create them directly on couch, but sending them + # ensures we are dealing with properly encrypted docs + + def setup(): + return soledad_client(force_fresh_db=True) + + def sync(clean_client): + return clean_client.sync() + yield txbenchmark_with_setup(setup, sync) + return test + + +# ATTENTION: update the documentation in ../docs/benchmarks.rst if you change +# the number of docs or the doc sizes for the tests below. +test_download_10_1000k = create_download(10, 1000 * 1000) +test_download_100_100k = create_download(100, 100 * 1000) +test_download_1000_10k = create_download(1000, 10 * 1000) + + +@pytest.inlineCallbacks +@pytest.mark.benchmark(group="test_nothing_to_sync") +def test_nothing_to_sync(soledad_client, txbenchmark_with_setup): + """ + Sync two replicas that are already in sync. + """ + def setup(): + return soledad_client() + + def sync(clean_client): + return clean_client.sync() + yield txbenchmark_with_setup(setup, sync) diff --git a/tests/blobs/test_blob_manager.py b/tests/blobs/test_blob_manager.py new file mode 100644 index 00000000..7d985768 --- /dev/null +++ b/tests/blobs/test_blob_manager.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +# test_local_backend.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for BlobManager. +""" +from twisted.trial import unittest +from twisted.internet import defer +from twisted.web.error import SchemeNotSupported +from leap.soledad.client._db.blobs import BlobManager, BlobDoc, FIXED_REV +from leap.soledad.client._db.blobs import BlobAlreadyExistsError +from leap.soledad.client._db.blobs import SyncStatus +from io import BytesIO +from mock import Mock +from uuid import uuid4 +import pytest +import os + + +class BlobManagerTestCase(unittest.TestCase): + + class doc_info: + doc_id = 'D-deadbeef' + rev = FIXED_REV + + def setUp(self): + self.cleartext = BytesIO('rosa de foc') + self.secret = 'A' * 96 + self.manager = BlobManager( + self.tempdir, '', + 'A' * 32, self.secret, + uuid4().hex, 'token', None) + self.addCleanup(self.manager.close) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_get_missing(self): + self.manager._download_and_decrypt = Mock(return_value=None) + missing_blob_id = uuid4().hex + result = yield self.manager.get(missing_blob_id) + self.assertIsNone(result) + args = missing_blob_id, '' + self.manager._download_and_decrypt.assert_called_once_with(*args) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_get_from_existing_value(self): + self.manager._download_and_decrypt = Mock(return_value=None) + msg, blob_id = "It's me, M4r10!", uuid4().hex + yield self.manager.local.put(blob_id, BytesIO(msg), + size=len(msg)) + result = yield self.manager.get(blob_id) + self.assertEquals(result.getvalue(), msg) + self.assertNot(self.manager._download_and_decrypt.called) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_put_stores_on_local_db(self): + self.manager._encrypt_and_upload = Mock(return_value=None) + msg, blob_id = "Hey Joe", uuid4().hex + doc = BlobDoc(BytesIO(msg), blob_id=blob_id) + yield self.manager.put(doc, size=len(msg)) + result = yield self.manager.local.get(blob_id) + self.assertEquals(result.getvalue(), msg) + self.assertTrue(self.manager._encrypt_and_upload.called) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_put_then_get_using_real_file_descriptor(self): + self.manager._encrypt_and_upload = Mock(return_value=None) + self.manager._download_and_decrypt = Mock(return_value=None) + msg, blob_id = "Fuuuuull cycleee! \o/", uuid4().hex + tmpfile = os.tmpfile() + tmpfile.write(msg) + tmpfile.seek(0) + doc = BlobDoc(tmpfile, blob_id) + yield self.manager.put(doc, size=len(msg)) + result = yield self.manager.get(doc.blob_id) + self.assertEquals(result.getvalue(), msg) + self.assertTrue(self.manager._encrypt_and_upload.called) + self.assertFalse(self.manager._download_and_decrypt.called) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_local_list_blobs(self): + self.manager._encrypt_and_upload = Mock(return_value=None) + msg, blob_id1, blob_id2 = "1337", uuid4().hex, uuid4().hex + doc = BlobDoc(BytesIO(msg), blob_id1) + yield self.manager.put(doc, size=len(msg)) + doc2 = BlobDoc(BytesIO(msg), blob_id2) + yield self.manager.put(doc2, size=len(msg)) + blobs_list = yield self.manager.local_list() + + self.assertEquals(set([blob_id1, blob_id2]), set(blobs_list)) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_send_missing(self): + fd, missing_id = BytesIO('test'), uuid4().hex + self.manager._encrypt_and_upload = Mock(return_value=None) + self.manager.remote_list = Mock(return_value=[]) + yield self.manager.local.put(missing_id, fd, 4) + yield self.manager.send_missing() + + call_list = self.manager._encrypt_and_upload.call_args_list + self.assertEquals(1, len(call_list)) + call_blob_id, call_fd = call_list[0][0] + self.assertEquals(missing_id, call_blob_id) + self.assertEquals('test', call_fd.getvalue()) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_duplicated_blob_error_on_put(self): + self.manager._encrypt_and_upload = Mock(return_value=None) + content, existing_id = "Blob content", uuid4().hex + doc1 = BlobDoc(BytesIO(content), existing_id) + yield self.manager.put(doc1, len(content)) + doc2 = BlobDoc(BytesIO(content), existing_id) + self.manager._encrypt_and_upload.reset_mock() + with pytest.raises(BlobAlreadyExistsError): + yield self.manager.put(doc2, len(content)) + self.assertFalse(self.manager._encrypt_and_upload.called) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_delete_from_local_and_remote(self): + self.manager._encrypt_and_upload = Mock(return_value=None) + self.manager._delete_from_remote = Mock(return_value=None) + content, blob_id = "Blob content", uuid4().hex + doc1 = BlobDoc(BytesIO(content), blob_id) + yield self.manager.put(doc1, len(content)) + yield self.manager.delete(blob_id) + local_list = yield self.manager.local_list() + self.assertEquals(0, len(local_list)) + params = {'namespace': ''} + self.manager._delete_from_remote.assert_called_with(blob_id, **params) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_local_sync_status_pending_upload(self): + upload_failure = defer.fail(Exception()) + self.manager._encrypt_and_upload = Mock(return_value=upload_failure) + content, blob_id = "Blob content", uuid4().hex + doc1 = BlobDoc(BytesIO(content), blob_id) + with pytest.raises(Exception): + yield self.manager.put(doc1, len(content)) + pending_upload = SyncStatus.PENDING_UPLOAD + local_list = yield self.manager.local_list(sync_status=pending_upload) + self.assertIn(blob_id, local_list) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_upload_retry_limit(self): + self.manager.remote_list = Mock(return_value=[]) + content, blob_id = "Blob content", uuid4().hex + doc1 = BlobDoc(BytesIO(content), blob_id) + with pytest.raises(Exception): + yield self.manager.put(doc1, len(content)) + for _ in range(self.manager.max_retries + 1): + with pytest.raises(SchemeNotSupported): + yield self.manager.send_missing() + failed_upload = SyncStatus.FAILED_UPLOAD + local_list = yield self.manager.local_list(sync_status=failed_upload) + self.assertIn(blob_id, local_list) diff --git a/tests/blobs/test_decrypter_buffer.py b/tests/blobs/test_decrypter_buffer.py new file mode 100644 index 00000000..83fbaad3 --- /dev/null +++ b/tests/blobs/test_decrypter_buffer.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# test_blobs.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for blobs decrypter buffer. A component which is used as a decryption +sink during blob stream download. +""" +from io import BytesIO +from mock import Mock + +from twisted.trial import unittest +from twisted.internet import defer + +from leap.soledad.client._db.blobs import DecrypterBuffer +from leap.soledad.client._db.blobs import BlobManager +from leap.soledad.client._db.blobs import FIXED_REV +from leap.soledad.client import _crypto + + +class DecrypterBufferCase(unittest.TestCase): + + class doc_info: + doc_id = 'D-BLOB-ID' + rev = FIXED_REV + + def setUp(self): + self.cleartext = BytesIO('rosa de foc') + self.secret = 'A' * 96 + self.blob = _crypto.BlobEncryptor( + self.doc_info, self.cleartext, + armor=False, + secret='A' * 96) + + @defer.inlineCallbacks + def test_decrypt_buffer(self): + encrypted = (yield self.blob.encrypt()).getvalue() + tag = encrypted[-16:] + buf = DecrypterBuffer(self.doc_info.doc_id, self.secret, tag) + buf.write(encrypted) + fd, size = buf.close() + self.assertEquals(fd.getvalue(), 'rosa de foc') + + @defer.inlineCallbacks + def test_decrypt_uploading_encrypted_blob(self): + + @defer.inlineCallbacks + def _check_result(uri, data, *args, **kwargs): + decryptor = _crypto.BlobDecryptor( + self.doc_info, data, + armor=False, + secret=self.secret) + decrypted = yield decryptor.decrypt() + self.assertEquals(decrypted.getvalue(), 'up and up') + defer.returnValue(Mock(code=200)) + + manager = BlobManager('', '', self.secret, self.secret, 'user') + fd = BytesIO('up and up') + manager._client.put = _check_result + yield manager._encrypt_and_upload(self.doc_info.doc_id, fd) diff --git a/tests/blobs/test_fs_backend.py b/tests/blobs/test_fs_backend.py new file mode 100644 index 00000000..53f3127d --- /dev/null +++ b/tests/blobs/test_fs_backend.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# test_fs_backend.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for blobs backend on server side. +""" +from twisted.trial import unittest +from twisted.internet import defer +from twisted.web.test.test_web import DummyRequest +from leap.soledad.server import _blobs +from io import BytesIO +from mock import Mock +import mock +import os +import base64 +import json +import pytest + + +class FilesystemBackendTestCase(unittest.TestCase): + + @mock.patch.object(_blobs, 'open') + def test_tag_header(self, open_mock): + open_mock.return_value = BytesIO('A' * 40 + 'B' * 16) + expected_tag = base64.urlsafe_b64encode('B' * 16) + expected_method = Mock() + backend = _blobs.FilesystemBlobsBackend() + request = Mock(responseHeaders=Mock(setRawHeaders=expected_method)) + backend.add_tag_header('user', 'blob_id', request) + + expected_method.assert_called_once_with('Tag', [expected_tag]) + + @mock.patch.object(_blobs.static, 'File') + def test_read_blob(self, file_mock): + render_mock = Mock() + file_mock.return_value = render_mock + backend = _blobs.FilesystemBlobsBackend() + request = DummyRequest(['']) + backend._get_path = Mock(return_value='path') + backend.read_blob('user', 'blob_id', request) + + backend._get_path.assert_called_once_with('user', 'blob_id', '') + ctype = 'application/octet-stream' + _blobs.static.File.assert_called_once_with('path', defaultType=ctype) + render_mock.render_GET.assert_called_once_with(request) + + @mock.patch.object(os.path, 'isfile') + @defer.inlineCallbacks + def test_cannot_overwrite(self, isfile): + isfile.return_value = True + backend = _blobs.FilesystemBlobsBackend() + backend._get_path = Mock(return_value='path') + request = DummyRequest(['']) + yield backend.write_blob('user', 'blob_id', request) + self.assertEquals(request.written[0], "Blob already exists: blob_id") + self.assertEquals(request.responseCode, 409) + + @pytest.mark.usefixtures("method_tmpdir") + @mock.patch.object(os.path, 'isfile') + @defer.inlineCallbacks + def test_write_cannot_exceed_quota(self, isfile): + isfile.return_value = False + backend = _blobs.FilesystemBlobsBackend() + backend._get_path = Mock(return_value=self.tempdir) + request = Mock() + + backend.get_total_storage = lambda x: 100 + backend.quota = 90 + yield backend.write_blob('user', 'blob_id', request) + + request.setResponseCode.assert_called_once_with(507) + request.write.assert_called_once_with('Quota Exceeded!') + + def test_get_path_partitioning_by_default(self): + backend = _blobs.FilesystemBlobsBackend() + backend.path = '/somewhere/' + path = backend._get_path('user', 'blob_id', '') + expected = '/somewhere/user/default/b/blo/blob_i/blob_id' + self.assertEquals(path, expected) + + def test_get_path_custom(self): + backend = _blobs.FilesystemBlobsBackend() + backend.path = '/somewhere/' + path = backend._get_path('user', 'blob_id', 'wonderland') + expected = '/somewhere/user/wonderland/b/blo/blob_i/blob_id' + self.assertEquals(expected, path) + + def test_get_path_namespace_traversal_raises(self): + backend = _blobs.FilesystemBlobsBackend() + backend.path = '/somewhere/' + with pytest.raises(Exception): + backend._get_path('user', 'blob_id', '..') + + @pytest.mark.usefixtures("method_tmpdir") + @mock.patch('leap.soledad.server._blobs.os.walk') + def test_list_blobs(self, walk_mock): + backend, _ = _blobs.FilesystemBlobsBackend(self.tempdir), None + walk_mock.return_value = [('', _, ['blob_0']), ('', _, ['blob_1'])] + result = json.loads(backend.list_blobs('user', DummyRequest(['']))) + self.assertEquals(result, ['blob_0', 'blob_1']) + + @pytest.mark.usefixtures("method_tmpdir") + @mock.patch('leap.soledad.server._blobs.os.walk') + def test_list_blobs_limited_by_namespace(self, walk_mock): + backend, _ = _blobs.FilesystemBlobsBackend(self.tempdir), None + walk_mock.return_value = [('', _, ['blob_0']), ('', _, ['blob_1'])] + result = json.loads(backend.list_blobs('user', DummyRequest(['']), + namespace='incoming')) + self.assertEquals(result, ['blob_0', 'blob_1']) + target_dir = os.path.join(self.tempdir, 'user', 'incoming') + walk_mock.assert_called_once_with(target_dir) + + @pytest.mark.usefixtures("method_tmpdir") + def test_path_validation_on_read_blob(self): + blobs_path, request = self.tempdir, DummyRequest(['']) + backend = _blobs.FilesystemBlobsBackend(blobs_path) + with pytest.raises(Exception): + backend.read_blob('..', '..', request) + with pytest.raises(Exception): + backend.read_blob('user', '../../../', request) + with pytest.raises(Exception): + backend.read_blob('../../../', 'blob_id', request) + with pytest.raises(Exception): + backend.read_blob('user', 'blob_id', request, namespace='..') + + @pytest.mark.usefixtures("method_tmpdir") + @defer.inlineCallbacks + def test_path_validation_on_write_blob(self): + blobs_path, request = self.tempdir, DummyRequest(['']) + backend = _blobs.FilesystemBlobsBackend(blobs_path) + with pytest.raises(Exception): + yield backend.write_blob('..', '..', request) + with pytest.raises(Exception): + yield backend.write_blob('user', '../../../', request) + with pytest.raises(Exception): + yield backend.write_blob('../../../', 'id1', request) + with pytest.raises(Exception): + yield backend.write_blob('user', 'id2', request, namespace='..') + + @pytest.mark.usefixtures("method_tmpdir") + @mock.patch('leap.soledad.server._blobs.os.unlink') + def test_delete_blob(self, unlink_mock): + backend = _blobs.FilesystemBlobsBackend(self.tempdir) + backend.delete_blob('user', 'blob_id') + unlink_mock.assert_any_call(backend._get_path('user', + 'blob_id')) + unlink_mock.assert_any_call(backend._get_path('user', + 'blob_id') + '.flags') + + @pytest.mark.usefixtures("method_tmpdir") + @mock.patch('leap.soledad.server._blobs.os.unlink') + def test_delete_blob_custom_namespace(self, unlink_mock): + backend = _blobs.FilesystemBlobsBackend(self.tempdir) + backend.delete_blob('user', 'blob_id', namespace='trash') + unlink_mock.assert_any_call(backend._get_path('user', + 'blob_id', + 'trash')) + unlink_mock.assert_any_call(backend._get_path('user', + 'blob_id', + 'trash') + '.flags') diff --git a/tests/blobs/test_sqlcipher_client_backend.py b/tests/blobs/test_sqlcipher_client_backend.py new file mode 100644 index 00000000..daf561c7 --- /dev/null +++ b/tests/blobs/test_sqlcipher_client_backend.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# test_sqlcipher_client_backend.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for sqlcipher backend on blobs client. +""" +from twisted.trial import unittest +from twisted.internet import defer +from leap.soledad.client._db.blobs import SQLiteBlobBackend +from io import BytesIO +from uuid import uuid4 +import pytest + + +class SQLBackendTestCase(unittest.TestCase): + + def setUp(self): + self.key = "A" * 96 + self.local = SQLiteBlobBackend(self.tempdir, self.key) + self.addCleanup(self.local.close) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_get_inexisting(self): + bad_blob_id = uuid4().hex + self.assertFalse((yield self.local.exists(bad_blob_id))) + result = yield self.local.get(bad_blob_id) + self.assertIsNone(result) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_get_existing(self): + blob_id = uuid4().hex + content = "x" + yield self.local.put(blob_id, BytesIO(content), len(content)) + result = yield self.local.get(blob_id) + self.assertTrue((yield self.local.exists(blob_id))) + self.assertEquals(result.getvalue(), content) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_delete(self): + blob_id1, blob_id2 = uuid4().hex, uuid4().hex + content = "x" + yield self.local.put(blob_id1, BytesIO(content), len(content)) + yield self.local.put(blob_id2, BytesIO(content), len(content)) + yield self.local.delete(blob_id1) + self.assertFalse((yield self.local.exists(blob_id1))) + self.assertTrue((yield self.local.exists(blob_id2))) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_list(self): + blob_ids = [uuid4().hex for _ in range(10)] + content = "x" + deferreds = [] + for blob_id in blob_ids: + deferreds.append(self.local.put(blob_id, BytesIO(content), + len(content))) + yield defer.gatherResults(deferreds) + result = yield self.local.list() + self.assertEquals(set(blob_ids), set(result)) diff --git a/tests/client/__init__.py b/tests/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/client/test_api.py b/tests/client/test_api.py new file mode 100644 index 00000000..3c6a8155 --- /dev/null +++ b/tests/client/test_api.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# test_api.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for soledad api. +""" + +from mock import MagicMock + +from test_soledad.util import BaseSoledadTest + + +class ApiTestCase(BaseSoledadTest): + + def test_recovery_code_creation(self): + recovery_code_mock = MagicMock() + generated_code = '4645a2f8997e5d0d' + recovery_code_mock.generate.return_value = generated_code + self._soledad._recovery_code = recovery_code_mock + + code = self._soledad.create_recovery_code() + + self.assertEqual(generated_code, code) diff --git a/tests/client/test_app.py b/tests/client/test_app.py new file mode 100644 index 00000000..6867473e --- /dev/null +++ b/tests/client/test_app.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# test_soledad_app.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test ObjectStore and Couch backend bits. +""" +import pytest + +from testscenarios import TestWithScenarios + +from test_soledad.util import BaseSoledadTest +from test_soledad.util import make_soledad_document_for_test +from test_soledad.util import make_token_soledad_app +from test_soledad.util import make_token_http_database_for_test +from test_soledad.util import copy_token_http_database_for_test +from test_soledad.u1db_tests import test_backends + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_backends`. +# ----------------------------------------------------------------------------- + +@pytest.mark.usefixtures('method_tmpdir') +class SoledadTests( + TestWithScenarios, test_backends.AllDatabaseTests, BaseSoledadTest): + + def setUp(self): + TestWithScenarios.setUp(self) + test_backends.AllDatabaseTests.setUp(self) + BaseSoledadTest.setUp(self) + + scenarios = [ + ('token_http', { + 'make_database_for_test': make_token_http_database_for_test, + 'copy_database_for_test': copy_token_http_database_for_test, + 'make_document_for_test': make_soledad_document_for_test, + 'make_app_with_state': make_token_soledad_app, + }) + ] diff --git a/tests/client/test_attachments.py b/tests/client/test_attachments.py new file mode 100644 index 00000000..2df5b90d --- /dev/null +++ b/tests/client/test_attachments.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# test_attachments.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for document attachments. +""" + +import pytest + +from io import BytesIO +from mock import Mock + +from twisted.internet import defer +from test_soledad.util import BaseSoledadTest + + +from leap.soledad.client import AttachmentStates + + +def mock_response(doc): + doc._manager._client.get = Mock( + return_value=defer.succeed(Mock(code=200, json=lambda: []))) + doc._manager._client.put = Mock( + return_value=defer.succeed(Mock(code=200))) + + +@pytest.mark.usefixture('method_tmpdir') +class AttachmentTests(BaseSoledadTest): + + @defer.inlineCallbacks + def test_create_doc_saves_store(self): + doc = yield self._soledad.create_doc({}) + self.assertEqual(self._soledad, doc.store) + + @defer.inlineCallbacks + def test_put_attachment(self): + doc = yield self._soledad.create_doc({}) + mock_response(doc) + yield doc.put_attachment(BytesIO('test')) + local_list = yield doc._manager.local_list() + self.assertIn(doc._blob_id, local_list) + + @defer.inlineCallbacks + def test_get_attachment(self): + doc = yield self._soledad.create_doc({}) + mock_response(doc) + yield doc.put_attachment(BytesIO('test')) + fd = yield doc.get_attachment() + self.assertEqual('test', fd.read()) + + @defer.inlineCallbacks + def test_get_attachment_state(self): + doc = yield self._soledad.create_doc({}) + state = yield doc.get_attachment_state() + self.assertEqual(AttachmentStates.NONE, state) + mock_response(doc) + yield doc.put_attachment(BytesIO('test')) + state = yield doc.get_attachment_state() + self.assertEqual(AttachmentStates.LOCAL, state) + + @defer.inlineCallbacks + def test_is_dirty(self): + doc = yield self._soledad.create_doc({}) + dirty = yield doc.is_dirty() + self.assertFalse(dirty) + doc.content = {'test': True} + dirty = yield doc.is_dirty() + self.assertTrue(dirty) diff --git a/tests/client/test_aux_methods.py b/tests/client/test_aux_methods.py new file mode 100644 index 00000000..1eb676c7 --- /dev/null +++ b/tests/client/test_aux_methods.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# test_soledad.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for general Soledad functionality. +""" +import os + +from pytest import inlineCallbacks + +from leap.soledad.client import Soledad +from leap.soledad.client._db.adbapi import U1DBConnectionPool +from leap.soledad.client._secrets.util import SecretsError + +from test_soledad.util import BaseSoledadTest + + +class AuxMethodsTestCase(BaseSoledadTest): + + def test__init_dirs(self): + sol = self._soledad_instance(prefix='_init_dirs') + local_db_dir = os.path.dirname(sol.local_db_path) + secrets_path = os.path.dirname(sol.secrets_path) + self.assertTrue(os.path.isdir(local_db_dir)) + self.assertTrue(os.path.isdir(secrets_path)) + + def _close_soledad(results): + sol.close() + + d = sol.create_doc({}) + d.addCallback(_close_soledad) + return d + + def test__init_u1db_sqlcipher_backend(self): + sol = self._soledad_instance(prefix='_init_db') + self.assertIsInstance(sol._dbpool, U1DBConnectionPool) + self.assertTrue(os.path.isfile(sol.local_db_path)) + sol.close() + + def test__init_config_with_defaults(self): + """ + Test if configuration defaults point to the correct place. + """ + + class SoledadMock(Soledad): + + def __init__(self): + pass + + # instantiate without initializing so we just test + # _init_config_with_defaults() + sol = SoledadMock() + sol.passphrase = u'' + sol.server_url = '' + sol._init_config_with_defaults() + # assert value of local_db_path + self.assertEquals( + os.path.join(sol.default_prefix, 'soledad.u1db'), + sol.local_db_path) + + def test__init_config_from_params(self): + """ + Test if configuration is correctly read from file. + """ + sol = self._soledad_instance( + 'leap@leap.se', + passphrase=u'123', + secrets_path='value_3', + local_db_path='value_2', + server_url='value_1', + cert_file=None) + self.assertEqual( + os.path.join(self.tempdir, 'value_3'), + sol.secrets_path) + self.assertEqual( + os.path.join(self.tempdir, 'value_2'), + sol.local_db_path) + self.assertEqual('value_1', sol.server_url) + sol.close() + + @inlineCallbacks + def test_change_passphrase(self): + """ + Test if passphrase can be changed. + """ + prefix = '_change_passphrase' + sol = self._soledad_instance( + 'leap@leap.se', + passphrase=u'123', + prefix=prefix, + ) + + doc1 = yield sol.create_doc({'simple': 'doc'}) + sol.change_passphrase(u'654321') + sol.close() + + with self.assertRaises(SecretsError): + self._soledad_instance( + 'leap@leap.se', + passphrase=u'123', + prefix=prefix) + + sol2 = self._soledad_instance( + 'leap@leap.se', + passphrase=u'654321', + prefix=prefix) + doc2 = yield sol2.get_doc(doc1.doc_id) + + self.assertEqual(doc1, doc2) + + sol2.close() + + def test_get_passphrase(self): + """ + Assert passphrase getter works fine. + """ + sol = self._soledad_instance() + self.assertEqual('123', sol.passphrase) + sol.close() diff --git a/tests/client/test_crypto.py b/tests/client/test_crypto.py new file mode 100644 index 00000000..5b647b73 --- /dev/null +++ b/tests/client/test_crypto.py @@ -0,0 +1,384 @@ +# -*- coding: utf-8 -*- +# test_crypto.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for cryptographic related stuff. +""" +import binascii +import base64 +import json +import os + +from io import BytesIO + +import pytest + +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.backends import default_backend +from cryptography.exceptions import InvalidTag + +from leap.soledad.common.document import SoledadDocument +from test_soledad.util import BaseSoledadTest +from leap.soledad.client import _crypto +from leap.soledad.client import _scrypt +from leap.soledad.common.blobs import preamble as _preamble + +from twisted.trial import unittest +from twisted.internet import defer + + +snowden1 = ( + "You can't come up against " + "the world's most powerful intelligence " + "agencies and not accept the risk. " + "If they want to get you, over time " + "they will.") + + +class ScryptTest(unittest.TestCase): + + def test_scrypt(self): + secret = 'supersikret' + salt = 'randomsalt' + key = _scrypt.hash(secret, salt, buflen=32) + expected = ('47996b569ea58d51ccbcc318d710' + 'a537acd28bb7a94615ab8d061d4b2a920f01') + assert binascii.b2a_hex(key) == expected + + +class AESTest(unittest.TestCase): + + def test_chunked_encryption(self): + key = 'A' * 32 + + fd = BytesIO() + aes = _crypto.AESWriter(key, _buffer=fd) + iv = aes.iv + + data = snowden1 + block = 16 + + for i in range(len(data) / block): + chunk = data[i * block:(i + 1) * block] + aes.write(chunk) + aes.end() + + ciphertext_chunked = fd.getvalue() + ciphertext, tag = _aes_encrypt(key, iv, data) + + assert ciphertext_chunked == ciphertext + + def test_decrypt(self): + key = 'A' * 32 + iv = 'A' * 16 + + data = snowden1 + block = 16 + + ciphertext, tag = _aes_encrypt(key, iv, data) + + fd = BytesIO() + aes = _crypto.AESWriter(key, iv, fd, tag=tag) + + for i in range(len(ciphertext) / block): + chunk = ciphertext[i * block:(i + 1) * block] + aes.write(chunk) + aes.end() + + cleartext_chunked = fd.getvalue() + assert cleartext_chunked == data + + +class BlobTestCase(unittest.TestCase): + + class doc_info: + doc_id = 'D-deadbeef' + rev = '397932e0c77f45fcb7c3732930e7e9b2:1' + + def setUp(self): + self.inf = BytesIO(snowden1) + self.blob = _crypto.BlobEncryptor( + self.doc_info, self.inf, + armor=True, + secret='A' * 96) + + @defer.inlineCallbacks + def test_unarmored_blob_encrypt(self): + self.blob.armor = False + encrypted = yield self.blob.encrypt() + + decryptor = _crypto.BlobDecryptor( + self.doc_info, encrypted, armor=False, + secret='A' * 96) + decrypted = yield decryptor.decrypt() + assert decrypted.getvalue() == snowden1 + + @defer.inlineCallbacks + def test_default_armored_blob_encrypt(self): + encrypted = yield self.blob.encrypt() + decode = base64.urlsafe_b64decode + assert map(decode, encrypted.getvalue().split()) + + @defer.inlineCallbacks + def test_blob_encryptor(self): + encrypted = yield self.blob.encrypt() + preamble, ciphertext = encrypted.getvalue().split() + preamble = base64.urlsafe_b64decode(preamble) + ciphertext = base64.urlsafe_b64decode(ciphertext) + ciphertext = ciphertext[:-16] + + assert len(preamble) == _preamble.PACMAN.size + unpacked_data = _preamble.PACMAN.unpack(preamble) + magic, sch, meth, ts, iv, doc_id, rev, _ = unpacked_data + assert magic == _crypto.MAGIC + assert sch == 1 + assert meth == _crypto.ENC_METHOD.aes_256_gcm + assert iv == self.blob.iv + assert doc_id == 'D-deadbeef' + assert rev == self.doc_info.rev + + aes_key = _crypto._get_sym_key_for_doc( + self.doc_info.doc_id, 'A' * 96) + assert ciphertext == _aes_encrypt(aes_key, self.blob.iv, snowden1)[0] + + decrypted = _aes_decrypt(aes_key, self.blob.iv, self.blob.tag, + ciphertext, preamble) + assert str(decrypted) == snowden1 + + @defer.inlineCallbacks + def test_init_with_preamble_alone(self): + ciphertext = yield self.blob.encrypt() + preamble = ciphertext.getvalue().split()[0] + decryptor = _crypto.BlobDecryptor( + self.doc_info, BytesIO(preamble), + start_stream=False, + secret='A' * 96) + assert decryptor._consume_preamble() + + @defer.inlineCallbacks + def test_incremental_blob_decryptor(self): + ciphertext = yield self.blob.encrypt() + preamble, ciphertext = ciphertext.getvalue().split() + ciphertext = base64.urlsafe_b64decode(ciphertext) + + decryptor = _crypto.BlobDecryptor( + self.doc_info, BytesIO(preamble), + start_stream=False, + secret='A' * 96, + tag=ciphertext[-16:]) + ciphertext = BytesIO(ciphertext[:-16]) + chunk = ciphertext.read(10) + while chunk: + decryptor.write(chunk) + chunk = ciphertext.read(10) + decrypted = decryptor._end_stream() + assert decrypted.getvalue() == snowden1 + + @defer.inlineCallbacks + def test_blob_decryptor(self): + ciphertext = yield self.blob.encrypt() + + decryptor = _crypto.BlobDecryptor( + self.doc_info, ciphertext, + secret='A' * 96) + decrypted = yield decryptor.decrypt() + assert decrypted.getvalue() == snowden1 + + @defer.inlineCallbacks + def test_unarmored_blob_decryptor(self): + self.blob.armor = False + ciphertext = yield self.blob.encrypt() + + decryptor = _crypto.BlobDecryptor( + self.doc_info, ciphertext, + armor=False, + secret='A' * 96) + decrypted = yield decryptor.decrypt() + assert decrypted.getvalue() == snowden1 + + @defer.inlineCallbacks + def test_encrypt_and_decrypt(self): + """ + Check that encrypting and decrypting gives same doc. + """ + crypto = _crypto.SoledadCrypto('A' * 96) + payload = {'key': 'someval'} + doc1 = SoledadDocument('id1', '1', json.dumps(payload)) + + encrypted = yield crypto.encrypt_doc(doc1) + assert encrypted != payload + assert 'raw' in encrypted + doc2 = SoledadDocument('id1', '1') + doc2.set_json(encrypted) + assert _crypto.is_symmetrically_encrypted(encrypted) + decrypted = (yield crypto.decrypt_doc(doc2)).getvalue() + assert len(decrypted) != 0 + assert json.loads(decrypted) == payload + + @defer.inlineCallbacks + def test_decrypt_with_wrong_tag_raises(self): + """ + Trying to decrypt a document with wrong MAC should raise. + """ + crypto = _crypto.SoledadCrypto('A' * 96) + payload = {'key': 'someval'} + doc1 = SoledadDocument('id1', '1', json.dumps(payload)) + + encrypted = yield crypto.encrypt_doc(doc1) + encdict = json.loads(encrypted) + preamble, raw = str(encdict['raw']).split() + preamble = base64.urlsafe_b64decode(preamble) + raw = base64.urlsafe_b64decode(raw) + # mess with tag + messed = raw[:-16] + '0' * 16 + + preamble = base64.urlsafe_b64encode(preamble) + newraw = preamble + ' ' + base64.urlsafe_b64encode(str(messed)) + doc2 = SoledadDocument('id1', '1') + doc2.set_json(json.dumps({"raw": str(newraw)})) + + with pytest.raises(_crypto.InvalidBlob): + yield crypto.decrypt_doc(doc2) + + +class SoledadSecretsTestCase(BaseSoledadTest): + + def test_generated_secrets_have_correct_length(self): + expected = self._soledad.secrets.lengths + for name, length in expected.iteritems(): + secret = getattr(self._soledad.secrets, name) + self.assertEqual(length, len(secret)) + + +class SoledadCryptoAESTestCase(BaseSoledadTest): + + def test_encrypt_decrypt_sym(self): + # generate 256-bit key + key = os.urandom(32) + iv, cyphertext = _crypto.encrypt_sym('data', key) + self.assertTrue(cyphertext is not None) + self.assertTrue(cyphertext != '') + self.assertTrue(cyphertext != 'data') + plaintext = _crypto.decrypt_sym(cyphertext, key, iv) + self.assertEqual('data', plaintext) + + def test_decrypt_with_wrong_iv_raises(self): + key = os.urandom(32) + iv, cyphertext = _crypto.encrypt_sym('data', key) + self.assertTrue(cyphertext is not None) + self.assertTrue(cyphertext != '') + self.assertTrue(cyphertext != 'data') + # get a different iv by changing the first byte + rawiv = binascii.a2b_base64(iv) + wrongiv = rawiv + while wrongiv == rawiv: + wrongiv = os.urandom(1) + rawiv[1:] + with pytest.raises(InvalidTag): + _crypto.decrypt_sym( + cyphertext, key, iv=binascii.b2a_base64(wrongiv)) + + def test_decrypt_with_wrong_key_raises(self): + key = os.urandom(32) + iv, cyphertext = _crypto.encrypt_sym('data', key) + self.assertTrue(cyphertext is not None) + self.assertTrue(cyphertext != '') + self.assertTrue(cyphertext != 'data') + wrongkey = os.urandom(32) # 256-bits key + # ensure keys are different in case we are extremely lucky + while wrongkey == key: + wrongkey = os.urandom(32) + with pytest.raises(InvalidTag): + _crypto.decrypt_sym(cyphertext, wrongkey, iv) + + +class PreambleTestCase(unittest.TestCase): + class doc_info: + doc_id = 'D-deadbeef' + rev = '397932e0c77f45fcb7c3732930e7e9b2:1' + + def setUp(self): + self.cleartext = BytesIO(snowden1) + self.blob = _crypto.BlobEncryptor( + self.doc_info, self.cleartext, + secret='A' * 96) + + def test_preamble_starts_with_magic_signature(self): + preamble = self.blob._encode_preamble() + assert preamble.startswith(_crypto.MAGIC) + + def test_preamble_has_cipher_metadata(self): + preamble = self.blob._encode_preamble() + unpacked = _preamble.PACMAN.unpack(preamble) + encryption_scheme, encryption_method = unpacked[1:3] + assert encryption_scheme in _crypto.ENC_SCHEME + assert encryption_method in _crypto.ENC_METHOD + assert unpacked[4] == self.blob.iv + + def test_preamble_has_document_sync_metadata(self): + preamble = self.blob._encode_preamble() + unpacked = _preamble.PACMAN.unpack(preamble) + doc_id, doc_rev = unpacked[5:7] + assert doc_id == self.doc_info.doc_id + assert doc_rev == self.doc_info.rev + + def test_preamble_has_document_size(self): + preamble = self.blob._encode_preamble() + unpacked = _preamble.PACMAN.unpack(preamble) + size = unpacked[7] + assert size == _crypto._ceiling(len(snowden1)) + + @defer.inlineCallbacks + def test_preamble_can_come_without_size(self): + # XXX: This test case is here only to test backwards compatibility! + preamble = self.blob._encode_preamble() + # repack preamble using legacy format, without doc size + unpacked = _preamble.PACMAN.unpack(preamble) + preamble_without_size = _preamble.LEGACY_PACMAN.pack(*unpacked[0:7]) + # encrypt it manually for custom tag + ciphertext, tag = _aes_encrypt(self.blob.sym_key, self.blob.iv, + self.cleartext.getvalue(), + aead=preamble_without_size) + ciphertext = ciphertext + tag + # encode it + ciphertext = base64.urlsafe_b64encode(ciphertext) + preamble_without_size = base64.urlsafe_b64encode(preamble_without_size) + # decrypt it + ciphertext = preamble_without_size + ' ' + ciphertext + cleartext = yield _crypto.BlobDecryptor( + self.doc_info, BytesIO(ciphertext), + secret='A' * 96).decrypt() + assert cleartext.getvalue() == self.cleartext.getvalue() + warnings = self.flushWarnings() + assert len(warnings) == 1 + assert 'legacy preamble without size' in warnings[0]['message'] + + +def _aes_encrypt(key, iv, data, aead=''): + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.GCM(iv), backend=backend) + encryptor = cipher.encryptor() + if aead: + encryptor.authenticate_additional_data(aead) + return encryptor.update(data) + encryptor.finalize(), encryptor.tag + + +def _aes_decrypt(key, iv, tag, data, aead=''): + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.GCM(iv, tag), backend=backend) + decryptor = cipher.decryptor() + if aead: + decryptor.authenticate_additional_data(aead) + return decryptor.update(data) + decryptor.finalize() diff --git a/tests/client/test_deprecated_crypto.py b/tests/client/test_deprecated_crypto.py new file mode 100644 index 00000000..939a2003 --- /dev/null +++ b/tests/client/test_deprecated_crypto.py @@ -0,0 +1,94 @@ +import json +import pytest + +from pytest import inlineCallbacks +from six.moves.urllib.parse import urljoin +from uuid import uuid4 + +from leap.soledad.client import crypto as old_crypto +from leap.soledad.common.couch import CouchDatabase +from leap.soledad.common import crypto as common_crypto + +from test_soledad.u1db_tests import simple_doc +from test_soledad.util import SoledadWithCouchServerMixin +from test_soledad.util import make_token_soledad_app +from test_soledad.u1db_tests import TestCaseWithServer + + +def deprecate_client_crypto(client): + secret = client._crypto.secret + _crypto = old_crypto.SoledadCrypto(secret) + setattr(client._dbsyncer, '_crypto', _crypto) + return client + + +def couch_database(couch_url, uuid): + db = CouchDatabase(couch_url, "user-%s" % (uuid,)) + return db + + +class DeprecatedCryptoTest(SoledadWithCouchServerMixin, TestCaseWithServer): + + def setUp(self): + SoledadWithCouchServerMixin.setUp(self) + TestCaseWithServer.setUp(self) + + def tearDown(self): + SoledadWithCouchServerMixin.tearDown(self) + TestCaseWithServer.tearDown(self) + + @staticmethod + def make_app_with_state(state): + return make_token_soledad_app(state) + + @pytest.mark.needs_couch + @inlineCallbacks + def test_touch_updates_remote_representation(self): + self.startTwistedServer() + user = 'user-' + uuid4().hex + server_url = 'http://%s:%d' % (self.server_address) + client = self._soledad_instance(user=user, server_url=server_url) + deprecated_client = deprecate_client_crypto( + self._soledad_instance(user=user, server_url=server_url)) + + self.make_app() + remote = self.request_state._create_database(replica_uid=client.uuid) + remote = CouchDatabase.open_database( + urljoin(self.couch_url, 'user-' + user), + create=True) + + # ensure remote db is empty + gen, docs = remote.get_all_docs() + assert gen == 0 + assert len(docs) == 0 + + # create a doc with deprecated client and sync + yield deprecated_client.create_doc(json.loads(simple_doc)) + yield deprecated_client.sync() + + # check for doc in remote db + gen, docs = remote.get_all_docs() + assert gen == 1 + assert len(docs) == 1 + doc = docs.pop() + content = doc.content + assert common_crypto.ENC_JSON_KEY in content + assert common_crypto.ENC_SCHEME_KEY in content + assert common_crypto.ENC_METHOD_KEY in content + assert common_crypto.ENC_IV_KEY in content + assert common_crypto.MAC_KEY in content + assert common_crypto.MAC_METHOD_KEY in content + + # "touch" the document with a newer client and synx + _, docs = yield client.get_all_docs() + yield client.put_doc(doc) + yield client.sync() + + # check for newer representation of doc in remote db + gen, docs = remote.get_all_docs() + assert gen == 2 + assert len(docs) == 1 + doc = docs.pop() + content = doc.content + assert len(content) == 1 + assert 'raw' in content diff --git a/tests/client/test_doc.py b/tests/client/test_doc.py new file mode 100644 index 00000000..36479e90 --- /dev/null +++ b/tests/client/test_doc.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# test_soledad_doc.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: soledad docs +""" +import pytest + +from testscenarios import TestWithScenarios + +from test_soledad.u1db_tests import test_document +from test_soledad.util import BaseSoledadTest +from test_soledad.util import make_soledad_document_for_test + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_document`. +# ----------------------------------------------------------------------------- + +@pytest.mark.usefixtures('method_tmpdir') +class TestSoledadDocument( + TestWithScenarios, + test_document.TestDocument, BaseSoledadTest): + + scenarios = ([( + 'leap', { + 'make_document_for_test': make_soledad_document_for_test})]) + + +@pytest.mark.usefixtures('method_tmpdir') +class TestSoledadPyDocument( + TestWithScenarios, + test_document.TestPyDocument, BaseSoledadTest): + + scenarios = ([( + 'leap', { + 'make_document_for_test': make_soledad_document_for_test})]) diff --git a/tests/client/test_http.py b/tests/client/test_http.py new file mode 100644 index 00000000..47df4b4a --- /dev/null +++ b/tests/client/test_http.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# test_http.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: test http database +""" + +from twisted.trial import unittest + +from leap.soledad.client import auth +from leap.soledad.common.l2db.remote import http_database + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_http_database`. +# ----------------------------------------------------------------------------- + +class _HTTPDatabase(http_database.HTTPDatabase, auth.TokenBasedAuth): + + """ + Wraps our token auth implementation. + """ + + def set_token_credentials(self, uuid, token): + auth.TokenBasedAuth.set_token_credentials(self, uuid, token) + + def _sign_request(self, method, url_query, params): + return auth.TokenBasedAuth._sign_request( + self, method, url_query, params) + + +class TestHTTPDatabaseWithCreds(unittest.TestCase): + + def test_get_sync_target_inherits_token_credentials(self): + # this test was from TestDatabaseSimpleOperations but we put it here + # for convenience. + self.db = _HTTPDatabase('dbase') + self.db.set_token_credentials('user-uuid', 'auth-token') + st = self.db.get_sync_target() + self.assertEqual(self.db._creds, st._creds) + + def test_ctr_with_creds(self): + db1 = _HTTPDatabase('http://dbs/db', creds={'token': { + 'uuid': 'user-uuid', + 'token': 'auth-token', + }}) + self.assertIn('token', db1._creds) diff --git a/tests/client/test_https.py b/tests/client/test_https.py new file mode 100644 index 00000000..1b6caed6 --- /dev/null +++ b/tests/client/test_https.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +# test_sync_target.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: https +""" +import pytest + +from testscenarios import TestWithScenarios + +from leap.soledad import client + +from leap.soledad.common.l2db.remote import http_client +from test_soledad.u1db_tests import test_backends +from test_soledad.u1db_tests import test_https +from test_soledad.util import ( + BaseSoledadTest, + make_soledad_document_for_test, + make_soledad_app, + make_token_soledad_app, +) + + +LEAP_SCENARIOS = [ + ('http', { + 'make_database_for_test': test_backends.make_http_database_for_test, + 'copy_database_for_test': test_backends.copy_http_database_for_test, + 'make_document_for_test': make_soledad_document_for_test, + 'make_app_with_state': make_soledad_app}), +] + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_https`. +# ----------------------------------------------------------------------------- + +def token_leap_https_sync_target(test, host, path, cert_file=None): + _, port = test.server.server_address + # source_replica_uid = test._soledad._dbpool.replica_uid + creds = {'token': {'uuid': 'user-uuid', 'token': 'auth-token'}} + if not cert_file: + cert_file = test.cacert_pem + st = client.http_target.SoledadHTTPSyncTarget( + 'https://%s:%d/%s' % (host, port, path), + source_replica_uid='other-id', + creds=creds, + crypto=test._soledad._crypto, + cert_file=cert_file) + return st + + +@pytest.mark.skip +class TestSoledadHTTPSyncTargetHttpsSupport( + TestWithScenarios, + # test_https.TestHttpSyncTargetHttpsSupport, + BaseSoledadTest): + + scenarios = [ + ('token_soledad_https', + { + # 'server_def': test_https.https_server_def, + 'make_app_with_state': make_token_soledad_app, + 'make_document_for_test': make_soledad_document_for_test, + 'sync_target': token_leap_https_sync_target}), + ] + + def setUp(self): + # the parent constructor undoes our SSL monkey patch to ensure tests + # run smoothly with standard u1db. + test_https.TestHttpSyncTargetHttpsSupport.setUp(self) + # so here monkey patch again to test our functionality. + api = client.api + http_client._VerifiedHTTPSConnection = api.VerifiedHTTPSConnection + client.api.SOLEDAD_CERT = http_client.CA_CERTS + + def test_cannot_verify_cert(self): + self.startServer() + # don't print expected traceback server-side + self.server.handle_error = lambda req, cli_addr: None + self.request_state._create_database('test') + remote_target = self.getSyncTarget( + 'localhost', 'test', cert_file=http_client.CA_CERTS) + d = remote_target.record_sync_info('other-id', 2, 'T-id') + + def _assert_raises(result): + from twisted.python.failure import Failure + if isinstance(result, Failure): + from OpenSSL.SSL import Error + error = result.value.message[0].value + if isinstance(error, Error): + msg = error.message[0][2] + self.assertEqual("certificate verify failed", msg) + return + self.fail("certificate verification should have failed.") + + d.addCallbacks(_assert_raises, _assert_raises) + return d + + def test_working(self): + """ + Test that SSL connections work well. + + This test was adapted to patch Soledad's HTTPS connection custom class + with the intended CA certificates. + """ + self.startServer() + db = self.request_state._create_database('test') + remote_target = self.getSyncTarget('localhost', 'test') + d = remote_target.record_sync_info('other-id', 2, 'T-id') + d.addCallback(lambda _: + self.assertEqual( + (2, 'T-id'), + db._get_replica_gen_and_trans_id('other-id') + )) + d.addCallback(lambda _: remote_target.close()) + return d + + def test_host_mismatch(self): + """ + This test is disabled because soledad's twisted-based http agent uses + pyOpenSSL, which will complain if we try to use an IP to connect to + the remote host (see the original test in u1db_tests/test_https.py). + """ + pass diff --git a/tests/client/test_incoming_processing_flow.py b/tests/client/test_incoming_processing_flow.py new file mode 100644 index 00000000..7bc1e3c6 --- /dev/null +++ b/tests/client/test_incoming_processing_flow.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +# test_incoming_processing_flow.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Unit tests for incoming box processing flow. +""" +from mock import Mock, call +from leap.soledad.client import interfaces +from leap.soledad.client.incoming import IncomingBoxProcessingLoop +from twisted.internet import defer +from twisted.trial import unittest +from zope.interface import implementer + + +@implementer(interfaces.IIncomingBoxConsumer) +class GoodConsumer(object): + def __init__(self): + self.name = 'GoodConsumer' + self.processed, self.saved = [], [] + + def process(self, item, item_id, encrypted=True): + self.processed.append(item_id) + return defer.succeed([item_id]) + + def save(self, parts, item_id): + self.saved.append(item_id) + return defer.succeed(None) + + +class ProcessingFailedConsumer(GoodConsumer): + def __init__(self): + self.name = 'ProcessingFailedConsumer' + self.processed, self.saved = [], [] + + def process(self, item, item_id, encrypted=True): + return defer.fail() + + +class SavingFailedConsumer(GoodConsumer): + def __init__(self): + self.name = 'SavingFailedConsumer' + self.processed, self.saved = [], [] + + def save(self, parts, item_id): + return defer.fail() + + +class IncomingBoxProcessingTestCase(unittest.TestCase): + + def setUp(self): + self.box = Mock() + self.loop = IncomingBoxProcessingLoop(self.box) + + def _set_pending_items(self, pending): + self.box.list_pending.return_value = defer.succeed(pending) + pending_iter = iter([defer.succeed(item) for item in pending]) + self.box.fetch_for_processing.side_effect = pending_iter + + @defer.inlineCallbacks + def test_processing_flow_reserves_a_message(self): + self._set_pending_items(['one_item']) + self.loop.add_consumer(GoodConsumer()) + yield self.loop() + self.box.fetch_for_processing.assert_called_once_with('one_item') + + @defer.inlineCallbacks + def test_no_consumers(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + yield self.loop() + self.box.fetch_for_processing.assert_not_called() + self.box.delete.assert_not_called() + + @defer.inlineCallbacks + def test_pending_list_with_multiple_items(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = GoodConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + calls = [call('one'), call('two'), call('three')] + self.box.fetch_for_processing.assert_has_calls(calls) + + @defer.inlineCallbacks + def test_good_consumer_process_all(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = GoodConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + self.assertEquals(items, consumer.processed) + + @defer.inlineCallbacks + def test_good_consumer_saves_all(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = GoodConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + self.assertEquals(items, consumer.saved) + + @defer.inlineCallbacks + def test_multiple_good_consumers_process_all(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = GoodConsumer() + consumer2 = GoodConsumer() + self.loop.add_consumer(consumer) + self.loop.add_consumer(consumer2) + yield self.loop() + self.assertEquals(items, consumer.processed) + self.assertEquals(items, consumer2.processed) + + @defer.inlineCallbacks + def test_good_consumer_marks_as_processed(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = GoodConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + self.box.set_processed.assert_has_calls([call(x) for x in items]) + + @defer.inlineCallbacks + def test_good_consumer_deletes_items(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = GoodConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + self.box.delete.assert_has_calls([call(x) for x in items]) + + @defer.inlineCallbacks + def test_processing_failed_doesnt_mark_as_processed(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = ProcessingFailedConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + self.box.set_processed.assert_not_called() + + @defer.inlineCallbacks + def test_processing_failed_doesnt_delete(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = ProcessingFailedConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + self.box.delete.assert_not_called() + + @defer.inlineCallbacks + def test_processing_failed_marks_as_failed(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = ProcessingFailedConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + self.box.set_failed.assert_has_calls([call(x) for x in items]) + + @defer.inlineCallbacks + def test_saving_failed_marks_as_processed(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = SavingFailedConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + self.box.set_processed.assert_has_calls([call(x) for x in items]) + + @defer.inlineCallbacks + def test_saving_failed_doesnt_delete(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = SavingFailedConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + self.box.delete.assert_not_called() + + @defer.inlineCallbacks + def test_saving_failed_marks_as_failed(self): + items = ['one', 'two', 'three'] + self._set_pending_items(items) + consumer = SavingFailedConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + self.box.set_failed.assert_has_calls([call(x) for x in items]) diff --git a/tests/client/test_recovery_code.py b/tests/client/test_recovery_code.py new file mode 100644 index 00000000..7bbccc41 --- /dev/null +++ b/tests/client/test_recovery_code.py @@ -0,0 +1,38 @@ +# -*- CODing: utf-8 -*- +# test_recovery_code.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for recovery code generation. +""" +import binascii + +from mock import patch +from twisted.trial import unittest +from leap.soledad.client._recovery_code import RecoveryCode + + +class RecoveryCodeTestCase(unittest.TestCase): + + @patch('leap.soledad.client._recovery_code.os.urandom') + def test_generate_recovery_code(self, mock_os_urandom): + generated_random_code = '123456' + mock_os_urandom.return_value = generated_random_code + recovery_code = RecoveryCode() + + code = recovery_code.generate() + + mock_os_urandom.assert_called_with(RecoveryCode.code_length) + self.assertEqual(binascii.hexlify(generated_random_code), code) diff --git a/tests/client/test_secrets.py b/tests/client/test_secrets.py new file mode 100644 index 00000000..7b643cb4 --- /dev/null +++ b/tests/client/test_secrets.py @@ -0,0 +1,166 @@ +# -*- CODing: utf-8 -*- +# test_secrets.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for secrets encryption and decryption. +""" +import scrypt + +from twisted.trial import unittest + +from leap.soledad.client._crypto import ENC_METHOD +from leap.soledad.client._secrets import SecretsCrypto + + +class SecretsCryptoTestCase(unittest.TestCase): + + SECRETS = { + 'remote_secret': 'a' * 512, + 'local_salt': 'b' * 64, + 'local_secret': 'c' * 448 + } + ENCRYPTED_V2 = { + 'cipher': 2, + 'length': 1437, + 'kdf_salt': 'TSgNLeAGFeITeSgNzmYZHh+mzmkZPOqao7CAV/tx3KZCLwsrT0HmWtVK3' + 'TyWHWNgVdeamMZYRuvZavE2sp0DGw==\n', + 'iv': 'TKZQKIlRgdnXFhJf08qswg==', + 'secrets': 'ZNZRi72VDtwZqyuU+uf3yzZt23vCtMS3Ki2bnZyeHUOSGVweJeDadF4oqE' + 'BW87NN00j9E49BzyzLr9SNgwZjPp0wlUm7kt+s8EUfJUdH8nxaQ+9iqGXM' + 'cCHmBM8L8DRN2m3BrPGx7m+QGlN9sbrRpl7fqc46RWcYuTEpm4upjdtI7O' + 'jDd0JG3C0rUzIuKJn9w4rEpX3tLEKXVdZfLvRXS5roR0cauazsDO69E13q' + 'a01vDuY+UJ+buLQ3FluPnnk8QE7ztPVUmRJJ76yAIhjVX9owiwlp9GnUJY' + 'sETRCqdRSTwUcHIkzVR0zAvtxTX7eGTitzf4gCYEC4T9v5N/jHxEfPdx28' + 'MM4KShWN2nFxNFQLQUpMN2OrM7UyUw+DQ3ydqBeBPKPHRN5s05kIK7P/Ra' + 'aLNcrJWa7DopLbgLlei0Jd7S4sjv1ufaRY7v0qJaVkhh/VaCylTSVw1rv5' + 'YzSWcHHcLuC0R8xLadz6T+EpsVYxgPYCS7w5xoE82zwNQzw/EBxLIcyLPl' + 'ipKnr2dttrmm3KXUOT1IdbSbI5elF6yQTAusdqiXuypey+MDqHYWEYWkCn' + 'e9/uGM9FjklDLE0RtPEDxhq64tw6u2Xu7RzDzyQDI8EIoTdU+4zEMTnelZ' + 'fKEwdG58EDxTXfUk6IDcRUupz3YuToSMhIOkqgXnbWl/nrK0O9v4JMhQjI' + 'r+oPICYfFr14kvJXBsfntILTJCxzbqTQcNba3jc8rGqCZ6gM0u4PndwTG2' + 'UiCqPU2HMnWvVGQOXeLdQY+EqqXQiRDi0DrDmkVwFf+27dPXxmZ43C48W3' + 'lMhTKXl0rdBFnOD5jiMh0X6q/KYXonyEtMZMsjT7dFePcCy4wQRhuut+ac' + '/TJWyrr+/IB45E+LZbhV7xCy1dYsbdb52jTRJFpaQ83sj6Iv6SYdiqqXzL' + 'F5JGMyuovTjwAoIIQzpLv36xY2wGGAH1V8c7QmDR2qumXrHD9R68WjBoSY' + '7IFM0TFAGZNun56y/zQ4r8yOMSAId+j4kuRH0fENEi0FJ+SpmSdHfpvBhE' + 'MdGh927E9enEYWmUQMmkxXIw6E+O3cmOWt2hsMbUAikDCpQOnVP2BD55HT' + '6FfbW7ITVwPINHYexmy2Xcm8H5zzGFSp+uYIyPBYDKA+VJ+QQI8bud9K+T' + 'NBybUv9u6LbB6BsLpwLoxMPJu0WsN2HpmLYgrg2ML1huMF1OtaGRuUr2PL' + 'NBaZaL6VOztYrVtQG1+tNyRxn8XQTtx0l6n+EihGVe9Sk5XF6DJA9ZN7uO' + 'svTUFJ5qG3Erf4AmbUJWoOR/NvapBtifidM7gPZZ6NqBs6v72rU1pGy+p7' + 'o84KrmB2MNf3yJ0BvKxPvFmltF3Dc7LB5TN8ycbmFM6hgrLvvhPxiHEnG/' + '8Qcrg0nUXOipFGNgZEU7t7Mz6RJ189Z2Kx1HVGrkAzEgqwZYqijAPlsgzO' + 'bg6DwzwC7stolQWGCDQUtJVlE8FZ/Up8zFYYZKn52WzjmSN4/hHhEvdkck' + 'Nez/JVev6fMcVrgdrTZ+uCwxjN/4xPdgog2HV470ea1bvIkSNOOrhm194M' + '40GmvmBhiSSMjdRQCQtM0t9bUuSQLPDzEiCA9QaLyygtlz9uRR/dXgkEg0' + 'J4YNpZvhE0wbyp4GHytbPaAmrcd7im9+buTuMwhXpZl0stmfkJxVHJSZ8Y' + 'IHakHs3W1fdYyI3wxGpel/9eYO3ISukolwrHXESP65wVNKfBwbqVJzQmts' + 'pyDBOI6DcLKZfE1EVg0+uwQ/5PKZbn0TwlXO1YE3NL3mAply3zQR9hyBrY' + '6f1jkHVD3irIlWkSiPJsP8sW+nrK8c/Ha8F+dua6DTZmg594OIaQj8mPiY' + 'GcIusiARWocR5/MmSjupGOgFx4HtmckTJtAta3XP4elOx04teH/P9Cgr1x' + 'XYf+cEX6gp92L9rTo0FCz3Hw==\n', + 'version': 2, + 'kdf': 'scrypt', + 'kdf_length': 32 + } + + ENCRYPTED_V1 = { + 'version': 1, + 'active_secret': 'secret_id', + 'storage_secrets': { + 'secret_id': { + 'kdf': 'scrypt', + 'secret': 'u31ObvxNU8jB0HgMj3TVwQ==:JQwlYq6sAQmHYS3x2CJzObT9h' + 'j1iiHthvrMh887qedNCcOfJyCA3jpRkc0vjd2Qk/2HSJ+JxM2F' + 'MrPzzx5O34EHlgF2scen34guZRRIf42WpnMy+PrL4cnMlZLgCh' + 'H1Jz6wcIMEpU9LQ8OaCShk1/yJ6qcVHOV4DDt3mTF7ttiqI5cp' + 'msaVtxxYCcpxFiWSeSCEgr0h4/Ih1qHuM6vk+CQjf/zg1f/7HR' + 'imIyNYXit9Fw3YTkxBen1wG3f5L7OAODRTuqnWpkQFOmclx050' + 'k0frKRcX6UWhIOWpW2mqJXnvzDtQQVGzqIdSgGTGtUDGQ7Onnc' + 'NkUlSnuVC7PkDNNRuwit3pCB9YWBWyPAQgs0kLqoV4YcuSctz6' + 'SAf76ozdcK5/SrOzutOfyPag4V3AYKMv6rCKALJ10OnFJ61FL9' + 'kd6JZam7WOlEUXyO7Gdgvz+eKiQMTZXbtO2kAKqel513MedPXC' + 'dzajUe1U2JaGg86UdiDWoPYOiWxnAPwfNJk+1QuNy5NZ7PaMtF' + 'IKT3/Xema2U8mufS0FbvJyK2flP1VUWcCzHKTSqX6+kU7UpoWa' + 'hYa7PlO40El+putTQLBmNaEeaWFngO+XB4TReICHSiCdcAb3pw' + 'sabjtxt+OpK4vbj3yBSfpiZTpVbEjt9U/tUpVp/T2M66lMi3ZC' + 'oHLlhu45Zo0aEq3UmQ/WBXu6EkO2eLYz2br9YQwRbSJ6z5CHmu' + 'hjKBQlpvGNfZYObx5lY4o6Ab4f/N8gyukskjmAFAf7Fr8cEog/' + 'oxmbagoCtUGRYJp2paooqH8L6xXp0Y8+23g7WJaAIr1i4V4aKS' + 'r9x7iUK6prcZTtMJZEHCswkLN/+DU6/FX3YZcOjseC+Qv3P+9v' + 'zIDp/92KJzqVqITGwrsc6ZsglMW37qxs6albtw3lMWSHlkcLbj' + 'Xf/iHPeKnb2WNLdkFNQ1J5OaTJR+E1CrXN+pm1JtB6XaUbaLGV' + 'CGUo13lAPVDtXcPbo64kMrQtQu4m9m8X8t8tfuJmINfwBnrKzk' + 'O6pl+LwimFaFEArV6wcaMxmwi0lM7mt4U1u9OIQjghQ/dEmOyV' + 'dZBnvyG7T/oRuLdUyZ/QGXZMlPQ3lAZ0ONn1Mk4bmKToW8ToE8' + 'ylld3rLlWDjjoQP8mP05Izg3mguLHXUhikUL8MD5NdYyeZJ1XZ' + '0OZ5S9uncurYj2ABWJoVaq/tFCdCEo9bbjWsePei26GZjaM3Fx' + 'RkAICXe/bt6/uLgaPZtO+sdARDuU3DRKMIdgM9NBaIn0kC7Wk4' + 'bnYShZ/rbhVt2/ds5XinnDBZsxSR3s553DixJ9v6w9Db++9Stw' + '4DgePd9lLy+6WuVBlKmcNflx9zg7US0AOarX2UNiQ==', + 'kdf_length': 32, + 'kdf_salt': 'MYH68QH48nRFMWH44piFWqBnKtU8KCz6Ajh24otrvzJlqPgB' + 'v6bvFJjRvjRp/0/v1j2nt40RZ6H5hfoKmore0g==\n', + 'length': 1024, + 'cipher': 'aes256', + } + } + } + + def setUp(self): + class Soledad(object): + passphrase = '123' + soledad = Soledad() + self._crypto = SecretsCrypto(soledad) + + def test__get_key(self): + salt = 'abc' + expected = scrypt.hash('123', salt, buflen=32) + key = self._crypto._get_key(salt) + self.assertEqual(expected, key) + + def test_encrypt(self): + info = self._crypto.encrypt(self.SECRETS) + self.assertEqual(8, len(info)) + for key, value in [ + ('kdf', 'scrypt'), + ('kdf_salt', None), + ('kdf_length', None), + ('cipher', ENC_METHOD.aes_256_gcm), + ('length', None), + ('iv', None), + ('secrets', None), + ('version', 2)]: + self.assertTrue(key in info) + if value: + self.assertEqual(info[key], value) + + def test__decrypt_v2(self): + encrypted = self.ENCRYPTED_V2 + decrypted = self._crypto.decrypt(encrypted) + self.assertEqual(decrypted, self.SECRETS) + + def test__decrypt_v1(self): + encrypted = self.ENCRYPTED_V1 + decrypted = self._crypto.decrypt(encrypted) + self.assertEqual(decrypted, self.SECRETS) + + def test__no_version_defaults_to_v1(self): + encrypted = dict(self.ENCRYPTED_V1) + del encrypted['version'] + decrypted = self._crypto.decrypt(encrypted) + self.assertEqual(decrypted, self.SECRETS) + self.assertEqual(encrypted['version'], 1) diff --git a/tests/client/test_shared_db.py b/tests/client/test_shared_db.py new file mode 100644 index 00000000..b045e524 --- /dev/null +++ b/tests/client/test_shared_db.py @@ -0,0 +1,40 @@ +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client.shared_db import SoledadSharedDatabase + +from test_soledad.util import BaseSoledadTest + + +class SoledadSharedDBTestCase(BaseSoledadTest): + + """ + These tests ensure the functionalities of the shared recovery database. + """ + + def setUp(self): + BaseSoledadTest.setUp(self) + self._shared_db = SoledadSharedDatabase( + 'https://provider/', document_factory=SoledadDocument, + creds=None) + + def tearDown(self): + BaseSoledadTest.tearDown(self) + + def test__get_remote_doc(self): + """ + Ensure the shared db is queried with the correct doc_id. + """ + doc_id = self._soledad.secrets.storage._remote_doc_id() + self._soledad.secrets.storage._get_remote_doc() + self._soledad.secrets.storage._shared_db.get_doc.assert_called_with( + doc_id) + + def test_save_remote(self): + """ + Ensure recovery document is put into shared recover db. + """ + doc_id = self._soledad.secrets.storage._remote_doc_id() + storage = self._soledad.secrets.storage + storage.save_remote({'content': 'blah'}) + storage._shared_db.get_doc.assert_called_with(doc_id) + storage._shared_db.put_doc.assert_called_with(self._doc_put) + self.assertTrue(self._doc_put.doc_id == doc_id) diff --git a/tests/client/test_signals.py b/tests/client/test_signals.py new file mode 100644 index 00000000..c7609a74 --- /dev/null +++ b/tests/client/test_signals.py @@ -0,0 +1,149 @@ +from mock import Mock +from twisted.internet import defer + +from leap import soledad +from leap.common.events import catalog +from leap.soledad.common.document import SoledadDocument + +from test_soledad.util import ADDRESS +from test_soledad.util import BaseSoledadTest + + +class SoledadSignalingTestCase(BaseSoledadTest): + + """ + These tests ensure signals are correctly emmited by Soledad. + """ + + EVENTS_SERVER_PORT = 8090 + + def setUp(self): + # mock signaling + soledad.client.signal = Mock() + soledad.client._secrets.util.events.emit_async = Mock() + # run parent's setUp + BaseSoledadTest.setUp(self) + + def tearDown(self): + BaseSoledadTest.tearDown(self) + + def _pop_mock_call(self, mocked): + mocked.call_args_list.pop() + mocked.mock_calls.pop() + mocked.call_args = mocked.call_args_list[-1] + + def test_stage3_bootstrap_signals(self): + """ + Test that a fresh soledad emits all bootstrap signals. + + Signals are: + - downloading keys / done downloading keys. + - creating keys / done creating keys. + - downloading keys / done downloading keys. + - uploading keys / done uploading keys. + """ + soledad.client._secrets.util.events.emit_async.reset_mock() + # get a fresh instance so it emits all bootstrap signals + sol = self._soledad_instance( + secrets_path='alternative_stage3.json', + local_db_path='alternative_stage3.u1db') + # reverse call order so we can verify in the order the signals were + # expected + soledad.client._secrets.util.events.emit_async.mock_calls.reverse() + soledad.client._secrets.util.events.emit_async.call_args = \ + soledad.client._secrets.util.events.emit_async.call_args_list[0] + soledad.client._secrets.util.events.emit_async.call_args_list.reverse() + + user_data = {'userid': ADDRESS, 'uuid': ADDRESS} + + def _assert(*args, **kwargs): + mocked = soledad.client._secrets.util.events.emit_async + mocked.assert_called_with(*args) + pop = kwargs.get('pop') + if pop or pop is None: + self._pop_mock_call(mocked) + + _assert(catalog.SOLEDAD_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_CREATING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_CREATING_KEYS, user_data) + _assert(catalog.SOLEDAD_UPLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data, pop=False) + + sol.close() + + def test_stage2_bootstrap_signals(self): + """ + Test that if there are keys in server, soledad will download them and + emit corresponding signals. + """ + # get existing instance so we have access to keys + sol = self._soledad_instance() + # create a document with secrets + doc = SoledadDocument(doc_id=sol.secrets.storage._remote_doc_id()) + doc.content = sol.secrets.crypto.encrypt(sol.secrets._secrets) + sol.close() + # reset mock + soledad.client._secrets.util.events.emit_async.reset_mock() + # get a fresh instance so it emits all bootstrap signals + shared_db = self.get_default_shared_mock(get_doc_return_value=doc) + sol = self._soledad_instance( + secrets_path='alternative_stage2.json', + local_db_path='alternative_stage2.u1db', + shared_db_class=shared_db) + # reverse call order so we can verify in the order the signals were + # expected + mocked = soledad.client._secrets.util.events.emit_async + mocked.mock_calls.reverse() + mocked.call_args = mocked.call_args_list[0] + mocked.call_args_list.reverse() + + def _assert(*args, **kwargs): + mocked = soledad.client._secrets.util.events.emit_async + mocked.assert_called_with(*args) + pop = kwargs.get('pop') + if pop or pop is None: + self._pop_mock_call(mocked) + + # assert download keys signals + user_data = {'userid': ADDRESS, 'uuid': ADDRESS} + _assert(catalog.SOLEDAD_DOWNLOADING_KEYS, user_data) + _assert(catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data, pop=False) + + sol.close() + + def test_stage1_bootstrap_signals(self): + """ + Test that if soledad already has a local secret, it emits no signals. + """ + soledad.client.signal.reset_mock() + # get an existent instance so it emits only some of bootstrap signals + sol = self._soledad_instance() + self.assertEqual([], soledad.client.signal.mock_calls) + sol.close() + + @defer.inlineCallbacks + def test_sync_signals(self): + """ + Test Soledad emits SOLEDAD_CREATING_KEYS signal. + """ + # get a fresh instance so it emits all bootstrap signals + sol = self._soledad_instance() + soledad.client.signal.reset_mock() + + # mock the actual db sync so soledad does not try to connect to the + # server + d = defer.Deferred() + d.callback(None) + sol._dbsyncer.sync = Mock(return_value=d) + + yield sol.sync() + + # assert the signal has been emitted + soledad.client.events.emit_async.assert_called_with( + catalog.SOLEDAD_DONE_DATA_SYNC, + {'userid': ADDRESS, 'uuid': ADDRESS}, + ) + sol.close() diff --git a/tests/client/test_soledad b/tests/client/test_soledad new file mode 120000 index 00000000..c1a35d32 --- /dev/null +++ b/tests/client/test_soledad @@ -0,0 +1 @@ +../test_soledad \ No newline at end of file diff --git a/tests/client/test_soledad_doc.py b/tests/client/test_soledad_doc.py new file mode 100644 index 00000000..e158d768 --- /dev/null +++ b/tests/client/test_soledad_doc.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# test_soledad_doc.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: soledad docs +""" +from testscenarios import TestWithScenarios + +from test_soledad.u1db_tests import test_document +from test_soledad.util import BaseSoledadTest +from test_soledad.util import make_soledad_document_for_test + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_document`. +# ----------------------------------------------------------------------------- + +class TestSoledadDocument( + TestWithScenarios, + test_document.TestDocument, BaseSoledadTest): + + scenarios = ([( + 'leap', { + 'make_document_for_test': make_soledad_document_for_test})]) + + +class TestSoledadPyDocument( + TestWithScenarios, + test_document.TestPyDocument, BaseSoledadTest): + + scenarios = ([( + 'leap', { + 'make_document_for_test': make_soledad_document_for_test})]) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..d3a39289 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,398 @@ +import glob +import base64 +import json +import os +import pytest +import re +import random +import requests +import signal +import socket +import subprocess +import sys +import time +import urlparse + +from hashlib import sha512 +from six.moves.urllib.parse import urljoin +from uuid import uuid4 + +from leap.soledad.common.couch import CouchDatabase +from leap.soledad.client import Soledad + + +def _select_subdir(subdir, blacklist, items): + + # allow blacklisted subdir if explicited in command line + if subdir and subdir in blacklist: + blacklist.remove(subdir) + + # determine blacklisted subdirs + dirname = os.path.dirname(__file__) + blacklisted_subdirs = map(lambda s: os.path.join(dirname, s), blacklist) + + # determine base path for selected tests + path = dirname + if subdir: + path = os.path.join(dirname, subdir) + + # remove tests from blacklisted subdirs + selected = [] + deselected = [] + for item in items: + filename = item.module.__file__ + blacklisted = any( + map(lambda s: filename.startswith(s), blacklisted_subdirs)) + if blacklisted or not filename.startswith(path): + deselected.append(item) + else: + selected.append(item) + + return selected, deselected + + +def pytest_collection_modifyitems(items, config): + + # mark tests that depend on couchdb + marker = getattr(pytest.mark, 'needs_couch') + for item in items: + if 'soledad/testing/tests/couch/' in item.module.__file__: + item.add_marker(marker) + + # select/deselect tests based on a blacklist and the subdir option given in + # command line + blacklist = ['benchmarks', 'responsiveness'] + subdir = config.getoption('subdir') + selected, deselected = _select_subdir(subdir, blacklist, items) + config.hook.pytest_deselected(items=deselected) + items[:] = selected + + +# +# default options for all tests +# + +DEFAULT_PASSPHRASE = '123' + +DEFAULT_URL = 'http://127.0.0.1:2424' +DEFAULT_PRIVKEY = 'soledad_privkey.pem' +DEFAULT_CERTKEY = 'soledad_certkey.pem' +DEFAULT_TOKEN = 'an-auth-token' + + +def pytest_addoption(parser): + parser.addoption( + "--couch-url", type="string", default="http://127.0.0.1:5984", + help="the url for the couch server to be used during tests") + + # the following options are only used in benchmarks, but has to be defined + # here due to how pytest discovers plugins during startup. + parser.addoption( + "--watch-memory", default=False, action="store_true", + help="whether to monitor memory percentages during test run. " + "**Warning**: enabling this will impact the time taken and the " + "CPU used by the benchmarked code, so use with caution!") + + parser.addoption( + "--soledad-server-url", type="string", default=None, + help="Soledad Server URL. A local server will be started if and only " + "if no URL is passed.") + + # the following option is only used in responsiveness tests, but has to be + # defined here due to how pytest discovers plugins during startup. + parser.addoption( + "--elasticsearch-url", type="string", default=None, + help="the url for posting responsiveness results to elasticsearch") + + parser.addoption( + "--subdir", type="string", default=None, + help="select only tests from a certain subdirectory of ./tests/") + + +def _request(method, url, data=None, do=True): + if do: + method = getattr(requests, method) + method(url, data=data) + else: + cmd = 'curl --netrc -X %s %s' % (method.upper(), url) + if data: + cmd += ' -d "%s"' % json.dumps(data) + print cmd + + +@pytest.fixture +def couch_url(request): + url = request.config.option.couch_url + request.cls.couch_url = url + + +@pytest.fixture +def method_tmpdir(request, tmpdir): + request.instance.tempdir = tmpdir.strpath + + +# +# remote_db fixture: provides an empty database for a given user in a per +# function scope. +# + +class UserDatabase(object): + + def __init__(self, url, uuid, create=True): + self._remote_db_url = urljoin(url, 'user-%s' % uuid) + self._create = create + + def setup(self): + if self._create: + return CouchDatabase.open_database( + url=self._remote_db_url, create=True, replica_uid=None) + else: + _request('put', self._remote_db_url, do=False) + + def teardown(self): + _request('delete', self._remote_db_url, do=self._create) + + +@pytest.fixture() +def remote_db(request): + couch_url = request.config.option.couch_url + + def create(uuid, create=True): + db = UserDatabase(couch_url, uuid, create=create) + request.addfinalizer(db.teardown) + return db.setup() + return create + + +def get_pid(pidfile): + if not os.path.isfile(pidfile): + return 0 + try: + with open(pidfile) as f: + return int(f.read()) + except IOError: + return 0 + + +# +# soledad_server fixture: provides a running soledad server in a per module +# context (same soledad server for all tests in this module). +# + +class SoledadServer(object): + + def __init__(self, tmpdir_factory, couch_url): + tmpdir = tmpdir_factory.mktemp('soledad-server') + self.tmpdir = tmpdir + self._pidfile = os.path.join(tmpdir.strpath, 'soledad-server.pid') + self._logfile = os.path.join(tmpdir.strpath, 'soledad-server.log') + self._couch_url = couch_url + + def start(self): + self._create_conf_file() + # start the server + executable = 'twistd' + if 'VIRTUAL_ENV' not in os.environ: + executable = os.path.join( + os.path.dirname(os.environ['_']), 'twistd') + subprocess.check_call([ + executable, + '--logfile=%s' % self._logfile, + '--pidfile=%s' % self._pidfile, + 'web', + '--class=leap.soledad.server.entrypoints.SoledadEntrypoint', + '--port=tcp:2424' + ]) + + def _create_conf_file(self): + + # come up with name of the configuration file + fname = '/etc/soledad/soledad-server.conf' + if not os.access('/etc', os.W_OK): + fname = os.path.join(self.tmpdir.strpath, 'soledad-server.conf') + + # create the configuration file + dirname = os.path.dirname(fname) + if not os.path.isdir(dirname): + os.mkdir(dirname) + with open(fname, 'w') as f: + blobs_path = os.path.join(str(self.tmpdir), 'blobs') + content = '''[soledad-server] +couch_url = %s +blobs = true +blobs_path = %s''' % (self._couch_url, blobs_path) + f.write(content) + + # update the environment to use that file + os.environ.update({'SOLEDAD_SERVER_CONFIG_FILE': fname}) + + def stop(self): + pid = get_pid(self._pidfile) + os.kill(pid, signal.SIGTERM) + + +@pytest.fixture(scope='module') +def soledad_server(tmpdir_factory, request): + + # avoid starting a server if the url is remote + soledad_url = request.config.option.soledad_server_url + if soledad_url is not None: + return None + + # start a soledad server + couch_url = request.config.option.couch_url + server = SoledadServer(tmpdir_factory, couch_url) + server.start() + request.addfinalizer(server.stop) + return server + + +# +# soledad_dbs fixture: provides all databases needed by soledad server in a per +# module scope (same databases for all tests in this module). +# + +def _token_dbname(): + dbname = 'tokens_' + \ + str(int(time.time() / (30 * 24 * 3600))) + return dbname + + +class SoledadDatabases(object): + + def __init__(self, url, create=True): + self._token_db_url = urljoin(url, _token_dbname()) + self._shared_db_url = urljoin(url, 'shared') + self._create = create + + def setup(self, uuid): + self._create_dbs() + self._add_token(uuid) + + def _create_dbs(self): + _request('put', self._token_db_url, do=self._create) + _request('put', self._shared_db_url, do=self._create) + + def _add_token(self, uuid): + token = sha512(DEFAULT_TOKEN).hexdigest() + content = {'type': 'Token', 'user_id': uuid} + _request('put', self._token_db_url + '/' + token, + data=json.dumps(content), do=self._create) + + def teardown(self): + _request('delete', self._token_db_url, do=self._create) + _request('delete', self._shared_db_url, do=self._create) + + +@pytest.fixture() +def soledad_dbs(request): + couch_url = request.config.option.couch_url + + def create(uuid, create=True): + db = SoledadDatabases(couch_url, create=create) + request.addfinalizer(db.teardown) + return db.setup(uuid) + return create + + +# +# soledad_client fixture: provides a clean soledad client for a test function. +# + +def _get_certfile(url, tmpdir): + + # download the certificate + parsed = urlparse.urlsplit(url) + netloc = re.sub('^[^\.]+\.', '', parsed.netloc) + host, _ = netloc.split(':') + response = requests.get('https://%s/ca.crt' % host, verify=False) + + # store it in a temporary file + cert_file = os.path.join(tmpdir.strpath, 'cert.pem') + with open(cert_file, 'w') as f: + f.write(response.text) + + return cert_file + + +@pytest.fixture() +def soledad_client(tmpdir, soledad_server, remote_db, soledad_dbs, request): + passphrase = DEFAULT_PASSPHRASE + token = DEFAULT_TOKEN + + # default values for local server + server_url = DEFAULT_URL + default_uuid = uuid4().hex + create = True + cert_file = None + + # use values for remote server if server url is passed + url_arg = request.config.option.soledad_server_url + if url_arg: + server_url = url_arg + default_uuid = 'test-user' + create = False + cert_file = _get_certfile(server_url, tmpdir) + + remote_db(default_uuid, create=create) + soledad_dbs(default_uuid, create=create) + + # get a soledad instance + def create(force_fresh_db=False): + secrets_file = '%s.secret' % default_uuid + secrets_path = os.path.join(tmpdir.strpath, secrets_file) + + # in some tests we might want to use the same user and remote database + # but with a clean/empty local database (i.e. download benchmarks), so + # here we provide a way to do that. + idx = 1 + if force_fresh_db: + # find the next index for this user + idx = len(glob.glob('%s/*-*.db' % tmpdir.strpath)) + 1 + db_file = '%s-%d.db' % (default_uuid, idx) + local_db_path = os.path.join(tmpdir.strpath, db_file) + + soledad_client = Soledad( + default_uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + auth_token=token, + with_blobs=True) + request.addfinalizer(soledad_client.close) + return soledad_client + return create + + +# +# pytest-benchmark customizations +# + +# avoid hooking if this is not a benchmarking environment +if 'pytest_benchmark' in sys.modules: + + def pytest_benchmark_update_machine_info(config, machine_info): + """ + Add the host's hostname information to machine_info. + + Get the value from the HOST_HOSTNAME environment variable if it is set, + or from the actual system's hostname otherwise. + """ + hostname = os.environ.get('HOST_HOSTNAME', socket.gethostname()) + machine_info['host'] = hostname + + +# +# benchmark/responsiveness fixtures +# + +@pytest.fixture() +def payload(): + def generate(size): + random.seed(1337) # same seed to avoid different bench results + payload_bytes = bytearray(random.getrandbits(8) for _ in xrange(size)) + # encode as base64 to avoid ascii encode/decode errors + return base64.b64encode(payload_bytes)[:size] # remove b64 overhead + return generate diff --git a/tests/couch/__init__.py b/tests/couch/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/couch/common.py b/tests/couch/common.py new file mode 100644 index 00000000..3c272f6e --- /dev/null +++ b/tests/couch/common.py @@ -0,0 +1,75 @@ +from uuid import uuid4 +from six.moves.urllib.parse import urljoin +from couchdb.client import Server + +from leap.soledad.common import couch +from leap.soledad.common.document import ServerDocument + +from test_soledad import u1db_tests as tests + + +simple_doc = tests.simple_doc +nested_doc = tests.nested_doc + + +def make_couch_database_for_test(test, replica_uid): + dbname = ('test-%s' % uuid4().hex) + db = couch.CouchDatabase.open_database( + urljoin(test.couch_url, dbname), + create=True, + replica_uid=replica_uid or 'test') + test.addCleanup(test.delete_db, dbname) + return db + + +def copy_couch_database_for_test(test, db): + couch_url = test.couch_url + new_dbname = db._dbname + '_copy' + new_db = couch.CouchDatabase.open_database( + urljoin(couch_url, new_dbname), + create=True, + replica_uid=db._replica_uid or 'test') + # copy all docs + session = couch.Session() + old_couch_db = Server(couch_url, session=session)[db._dbname] + new_couch_db = Server(couch_url, session=session)[new_dbname] + for doc_id in old_couch_db: + doc = old_couch_db.get(doc_id) + # bypass u1db_config document + if doc_id == 'u1db_config': + pass + # copy u1db docs + elif 'u1db_rev' in doc: + new_doc = { + '_id': doc['_id'], + 'u1db_rev': doc['u1db_rev'] + } + attachments = [] + if ('u1db_conflicts' in doc): + new_doc['u1db_conflicts'] = doc['u1db_conflicts'] + for c_rev in doc['u1db_conflicts']: + attachments.append('u1db_conflict_%s' % c_rev) + new_couch_db.save(new_doc) + # save conflict data + attachments.append('u1db_content') + for att_name in attachments: + att = old_couch_db.get_attachment(doc_id, att_name) + if (att is not None): + new_couch_db.put_attachment(new_doc, att, + filename=att_name) + elif doc_id.startswith('gen-'): + new_couch_db.save(doc) + # cleanup connections to prevent file descriptor leaking + return new_db + + +def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): + return ServerDocument( + doc_id, rev, content, has_conflicts=has_conflicts) + + +COUCH_SCENARIOS = [ + ('couch', {'make_database_for_test': make_couch_database_for_test, + 'copy_database_for_test': copy_couch_database_for_test, + 'make_document_for_test': make_document_for_test, }), +] diff --git a/tests/couch/test_atomicity.py b/tests/couch/test_atomicity.py new file mode 100644 index 00000000..48e1c01d --- /dev/null +++ b/tests/couch/test_atomicity.py @@ -0,0 +1,370 @@ +# -*- coding: utf-8 -*- +# test_couch_operations_atomicity.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test atomicity of couch operations. +""" +import os +import pytest +import threading + +from six.moves.urllib.parse import urljoin +from twisted.internet import defer +from uuid import uuid4 + +from leap.soledad.client import Soledad +from leap.soledad.common.couch.state import CouchServerState +from leap.soledad.common.couch import CouchDatabase + +from test_soledad.util import ( + make_token_soledad_app, + make_soledad_document_for_test, + soledad_sync_target, +) +from test_soledad.util import CouchDBTestCase +from test_soledad.u1db_tests import TestCaseWithServer + + +REPEAT_TIMES = 20 + + +@pytest.mark.usefixtures('method_tmpdir') +class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): + + @staticmethod + def make_app_after_state(state): + return make_token_soledad_app(state) + + make_document_for_test = make_soledad_document_for_test + + sync_target = soledad_sync_target + + def _soledad_instance(self, user=None, passphrase=u'123', + prefix='', + secrets_path='secrets.json', + local_db_path='soledad.u1db', server_url='', + cert_file=None, auth_token=None): + """ + Instantiate Soledad. + """ + user = user or self.user + + # this callback ensures we save a document which is sent to the shared + # db. + def _put_doc_side_effect(doc): + self._doc_put = doc + + soledad = Soledad( + user, + passphrase, + secrets_path=os.path.join(self.tempdir, prefix, secrets_path), + local_db_path=os.path.join( + self.tempdir, prefix, local_db_path), + server_url=server_url, + cert_file=cert_file, + auth_token=auth_token, + shared_db=self.get_default_shared_mock(_put_doc_side_effect)) + self.addCleanup(soledad.close) + return soledad + + def make_app(self): + self.request_state = CouchServerState(self.couch_url) + return self.make_app_after_state(self.request_state) + + def setUp(self): + TestCaseWithServer.setUp(self) + CouchDBTestCase.setUp(self) + self.user = ('user-%s' % uuid4().hex) + self.db = CouchDatabase.open_database( + urljoin(self.couch_url, 'user-' + self.user), + create=True, + replica_uid='replica') + self.startTwistedServer() + + def tearDown(self): + self.db.delete_database() + self.db.close() + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) + + # + # Sequential tests + # + + def test_correct_transaction_log_after_sequential_puts(self): + """ + Assert that the transaction_log increases accordingly with sequential + puts. + """ + doc = self.db.create_doc({'ops': 0}) + docs = [doc.doc_id] + for i in range(0, REPEAT_TIMES): + self.assertEqual( + i + 1, len(self.db._get_transaction_log())) + doc.content['ops'] += 1 + self.db.put_doc(doc) + docs.append(doc.doc_id) + + # assert length of transaction_log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + REPEAT_TIMES + 1, len(transaction_log)) + + # assert that all entries in the log belong to the same doc + self.assertEqual(REPEAT_TIMES + 1, len(docs)) + for doc_id in docs: + self.assertEqual( + REPEAT_TIMES + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + def test_correct_transaction_log_after_sequential_deletes(self): + """ + Assert that the transaction_log increases accordingly with sequential + puts and deletes. + """ + docs = [] + for i in range(0, REPEAT_TIMES): + doc = self.db.create_doc({'ops': 0}) + self.assertEqual( + 2 * i + 1, len(self.db._get_transaction_log())) + docs.append(doc.doc_id) + self.db.delete_doc(doc) + self.assertEqual( + 2 * i + 2, len(self.db._get_transaction_log())) + + # assert length of transaction_log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + 2 * REPEAT_TIMES, len(transaction_log)) + + # assert that each doc appears twice in the transaction_log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 2, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + @defer.inlineCallbacks + def test_correct_sync_log_after_sequential_syncs(self): + """ + Assert that the sync_log increases accordingly with sequential syncs. + """ + sol = self._soledad_instance( + auth_token='auth-token', + server_url=self.getURL()) + source_replica_uid = sol._dbpool.replica_uid + + def _create_docs(): + deferreds = [] + for i in xrange(0, REPEAT_TIMES): + deferreds.append(sol.create_doc({})) + return defer.gatherResults(deferreds) + + def _assert_transaction_and_sync_logs(results, sync_idx): + # assert sizes of transaction and sync logs + self.assertEqual( + sync_idx * REPEAT_TIMES, + len(self.db._get_transaction_log())) + gen, _ = self.db._get_replica_gen_and_trans_id(source_replica_uid) + self.assertEqual(sync_idx * REPEAT_TIMES, gen) + + def _assert_sync(results, sync_idx): + gen, docs = results + self.assertEqual((sync_idx + 1) * REPEAT_TIMES, gen) + self.assertEqual((sync_idx + 1) * REPEAT_TIMES, len(docs)) + # assert sizes of transaction and sync logs + self.assertEqual((sync_idx + 1) * REPEAT_TIMES, + len(self.db._get_transaction_log())) + target_known_gen, target_known_trans_id = \ + self.db._get_replica_gen_and_trans_id(source_replica_uid) + # assert it has the correct gen and trans_id + conn_key = sol._dbpool._u1dbconnections.keys().pop() + conn = sol._dbpool._u1dbconnections[conn_key] + sol_gen, sol_trans_id = conn._get_generation_info() + self.assertEqual(sol_gen, target_known_gen) + self.assertEqual(sol_trans_id, target_known_trans_id) + + # sync first time and assert success + results = yield _create_docs() + _assert_transaction_and_sync_logs(results, 0) + yield sol.sync() + results = yield sol.get_all_docs() + _assert_sync(results, 0) + + # create more docs, sync second time and assert success + results = yield _create_docs() + _assert_transaction_and_sync_logs(results, 1) + yield sol.sync() + results = yield sol.get_all_docs() + _assert_sync(results, 1) + + # + # Concurrency tests + # + + class _WorkerThread(threading.Thread): + + def __init__(self, params, run_method): + threading.Thread.__init__(self) + self._params = params + self._run_method = run_method + + def run(self): + self._run_method(self) + + def test_correct_transaction_log_after_concurrent_puts(self): + """ + Assert that the transaction_log increases accordingly with concurrent + puts. + """ + pool = threading.BoundedSemaphore(value=1) + threads = [] + docs = [] + + def _run_method(self): + doc = self._params['db'].create_doc({}) + pool.acquire() + self._params['docs'].append(doc.doc_id) + pool.release() + + for i in range(0, REPEAT_TIMES): + thread = self._WorkerThread( + {'docs': docs, 'db': self.db}, + _run_method) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + # assert length of transaction_log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + REPEAT_TIMES, len(transaction_log)) + + # assert all documents are in the log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + def test_correct_transaction_log_after_concurrent_deletes(self): + """ + Assert that the transaction_log increases accordingly with concurrent + puts and deletes. + """ + threads = [] + docs = [] + pool = threading.BoundedSemaphore(value=1) + + # create/delete method that will be run concurrently + def _run_method(self): + doc = self._params['db'].create_doc({}) + pool.acquire() + docs.append(doc.doc_id) + pool.release() + self._params['db'].delete_doc(doc) + + # launch concurrent threads + for i in range(0, REPEAT_TIMES): + thread = self._WorkerThread({'db': self.db}, _run_method) + thread.start() + threads.append(thread) + + # wait for threads to finish + for thread in threads: + thread.join() + + # assert transaction log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + 2 * REPEAT_TIMES, len(transaction_log)) + # assert that each doc appears twice in the transaction_log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 2, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + def test_correct_sync_log_after_concurrent_puts_and_sync(self): + """ + Assert that the sync_log is correct after concurrent syncs. + """ + docs = [] + + sol = self._soledad_instance( + auth_token='auth-token', + server_url=self.getURL()) + + def _save_doc_ids(results): + for doc in results: + docs.append(doc.doc_id) + + # create documents in parallel + deferreds = [] + for i in range(0, REPEAT_TIMES): + d = sol.create_doc({}) + deferreds.append(d) + + # wait for documents creation and sync + d = defer.gatherResults(deferreds) + d.addCallback(_save_doc_ids) + d.addCallback(lambda _: sol.sync()) + + def _assert_logs(results): + transaction_log = self.db._get_transaction_log() + self.assertEqual(REPEAT_TIMES, len(transaction_log)) + # assert all documents are in the remote log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + d.addCallback(_assert_logs) + d.addCallback(lambda _: sol.close()) + + return d + + @defer.inlineCallbacks + def test_concurrent_syncs_do_not_fail(self): + """ + Assert that concurrent attempts to sync end up being executed + sequentially and do not fail. + """ + docs = [] + + sol = self._soledad_instance( + auth_token='auth-token', + server_url=self.getURL()) + + deferreds = [] + for i in xrange(0, REPEAT_TIMES): + d = sol.create_doc({}) + d.addCallback(lambda doc: docs.append(doc.doc_id)) + d.addCallback(lambda _: sol.sync()) + deferreds.append(d) + yield defer.gatherResults(deferreds, consumeErrors=True) + + transaction_log = self.db._get_transaction_log() + self.assertEqual(REPEAT_TIMES, len(transaction_log)) + # assert all documents are in the remote log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) diff --git a/tests/couch/test_backend.py b/tests/couch/test_backend.py new file mode 100644 index 00000000..9dfa22ac --- /dev/null +++ b/tests/couch/test_backend.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +# test_couch.py +# Copyright (C) 2013-2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test ObjectStore and Couch backend bits. +""" + +from uuid import uuid4 +from six.moves.urllib.parse import urljoin +from testscenarios import TestWithScenarios +from twisted.trial import unittest + +from leap.soledad.common import couch + +from test_soledad.util import CouchDBTestCase +from test_soledad.u1db_tests import test_backends + +from .common import COUCH_SCENARIOS + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_common_backend`. +# ----------------------------------------------------------------------------- + +class TestCouchBackendImpl(CouchDBTestCase): + + def test__allocate_doc_id(self): + db = couch.CouchDatabase.open_database( + urljoin(self.couch_url, 'test-%s' % uuid4().hex), + create=True) + doc_id1 = db._allocate_doc_id() + self.assertTrue(doc_id1.startswith('D-')) + self.assertEqual(34, len(doc_id1)) + int(doc_id1[len('D-'):], 16) + self.assertNotEqual(doc_id1, db._allocate_doc_id()) + self.delete_db(db._dbname) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_backends`. +# ----------------------------------------------------------------------------- + +class CouchTests( + TestWithScenarios, test_backends.AllDatabaseTests, CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class CouchBackendTests( + TestWithScenarios, + test_backends.LocalDatabaseTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class CouchValidateGenNTransIdTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateGenNTransIdTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class CouchValidateSourceGenTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateSourceGenTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class CouchWithConflictsTests( + TestWithScenarios, + test_backends.LocalDatabaseWithConflictsTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +# Notice: the CouchDB backend does not have indexing capabilities, so we do +# not test indexing now. + +# class CouchIndexTests(test_backends.DatabaseIndexTests, CouchDBTestCase): +# +# scenarios = COUCH_SCENARIOS +# +# def tearDown(self): +# self.db.delete_database() +# test_backends.DatabaseIndexTests.tearDown(self) + + +class DatabaseNameValidationTest(unittest.TestCase): + + def test_database_name_validation(self): + inject = couch.state.is_db_name_valid("user-deadbeef | cat /secret") + self.assertFalse(inject) + self.assertTrue(couch.state.is_db_name_valid("user-cafe1337")) diff --git a/tests/couch/test_command.py b/tests/couch/test_command.py new file mode 100644 index 00000000..9fb2c153 --- /dev/null +++ b/tests/couch/test_command.py @@ -0,0 +1,31 @@ +from twisted.trial import unittest + +from leap.soledad.common.couch import state as couch_state +from leap.soledad.common.l2db import errors as u1db_errors + +from mock import Mock + + +class CommandBasedDBCreationTest(unittest.TestCase): + + def test_ensure_db_using_custom_command(self): + state = couch_state.CouchServerState( + "url", create_cmd="/bin/echo", check_schema_versions=False) + mock_db = Mock() + mock_db.replica_uid = 'replica_uid' + state.open_database = Mock(return_value=mock_db) + db, replica_uid = state.ensure_database("user-1337") # works + self.assertEquals(mock_db, db) + self.assertEquals(mock_db.replica_uid, replica_uid) + + def test_raises_unauthorized_on_failure(self): + state = couch_state.CouchServerState( + "url", create_cmd="inexistent", check_schema_versions=False) + self.assertRaises(u1db_errors.Unauthorized, + state.ensure_database, "user-1337") + + def test_raises_unauthorized_by_default(self): + state = couch_state.CouchServerState("url", + check_schema_versions=False) + self.assertRaises(u1db_errors.Unauthorized, + state.ensure_database, "user-1337") diff --git a/tests/couch/test_ddocs.py b/tests/couch/test_ddocs.py new file mode 100644 index 00000000..3937f2de --- /dev/null +++ b/tests/couch/test_ddocs.py @@ -0,0 +1,60 @@ +from uuid import uuid4 + +from leap.soledad.common import couch + +from test_soledad.util import CouchDBTestCase + + +class CouchDesignDocsTests(CouchDBTestCase): + + def setUp(self): + CouchDBTestCase.setUp(self) + self.create_db() + + def create_db(self, dbname=None): + if not dbname: + dbname = ('test-%s' % uuid4().hex) + if dbname not in self.couch_server: + self.couch_server.create(dbname) + self.db = couch.CouchDatabase( + (self.couch_url), + dbname) + + def tearDown(self): + self.db.delete_database() + self.db.close() + CouchDBTestCase.tearDown(self) + + def test_ensure_security_doc(self): + """ + Ensure_security creates a _security ddoc to ensure that only soledad + will have the lowest privileged access to an user db. + """ + self.assertFalse(self.db._database.resource.get_json('_security')[2]) + self.db.ensure_security_ddoc() + security_ddoc = self.db._database.resource.get_json('_security')[2] + self.assertIn('admins', security_ddoc) + self.assertFalse(security_ddoc['admins']['names']) + self.assertIn('members', security_ddoc) + self.assertIn('soledad', security_ddoc['members']['names']) + + def test_ensure_security_from_configuration(self): + """ + Given a configuration, follow it to create the security document + """ + configuration = {'members': ['user1', 'user2'], + 'members_roles': ['role1', 'role2'], + 'admins': ['admin'], + 'admins_roles': ['administrators'] + } + self.db.ensure_security_ddoc(configuration) + + security_ddoc = self.db._database.resource.get_json('_security')[2] + self.assertEquals(configuration['admins'], + security_ddoc['admins']['names']) + self.assertEquals(configuration['admins_roles'], + security_ddoc['admins']['roles']) + self.assertEquals(configuration['members'], + security_ddoc['members']['names']) + self.assertEquals(configuration['members_roles'], + security_ddoc['members']['roles']) diff --git a/tests/couch/test_soledad b/tests/couch/test_soledad new file mode 120000 index 00000000..c1a35d32 --- /dev/null +++ b/tests/couch/test_soledad @@ -0,0 +1 @@ +../test_soledad \ No newline at end of file diff --git a/tests/couch/test_state.py b/tests/couch/test_state.py new file mode 100644 index 00000000..e5ac3704 --- /dev/null +++ b/tests/couch/test_state.py @@ -0,0 +1,32 @@ +import pytest +from leap.soledad.common.couch import CONFIG_DOC_ID +from leap.soledad.common.couch import SCHEMA_VERSION +from leap.soledad.common.couch import SCHEMA_VERSION_KEY +from leap.soledad.common.couch.state import CouchServerState +from uuid import uuid4 + +from leap.soledad.common.errors import WrongCouchSchemaVersionError +from leap.soledad.common.errors import MissingCouchConfigDocumentError +from test_soledad.util import CouchDBTestCase + + +class CouchDesignDocsTests(CouchDBTestCase): + + def setUp(self): + CouchDBTestCase.setUp(self) + self.db = self.couch_server.create('user-' + uuid4().hex) + self.addCleanup(self.delete_db, self.db.name) + + def test_wrong_couch_version_raises(self): + wrong_schema_version = SCHEMA_VERSION + 1 + self.db.create( + {'_id': CONFIG_DOC_ID, SCHEMA_VERSION_KEY: wrong_schema_version}) + with pytest.raises(WrongCouchSchemaVersionError): + CouchServerState(self.couch_url, create_cmd='/bin/echo', + check_schema_versions=True) + + def test_missing_config_doc_raises(self): + self.db.create({}) + with pytest.raises(MissingCouchConfigDocumentError): + CouchServerState(self.couch_url, create_cmd='/bin/echo', + check_schema_versions=True) diff --git a/tests/couch/test_sync.py b/tests/couch/test_sync.py new file mode 100644 index 00000000..c353518e --- /dev/null +++ b/tests/couch/test_sync.py @@ -0,0 +1,700 @@ +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db import errors as u1db_errors + +from testscenarios import TestWithScenarios + +from test_soledad import u1db_tests as tests +from test_soledad.util import CouchDBTestCase +from test_soledad.util import sync_via_synchronizer +from test_soledad.u1db_tests import DatabaseBaseTests + +from .common import simple_doc +from .common import COUCH_SCENARIOS + + +sync_scenarios = [] +for name, scenario in COUCH_SCENARIOS: + scenario = dict(scenario) + scenario['do_sync'] = sync_via_synchronizer + sync_scenarios.append((name, scenario)) + scenario = dict(scenario) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sync`. +# ----------------------------------------------------------------------------- + +class CouchBackendSyncTests( + TestWithScenarios, + DatabaseBaseTests, + CouchDBTestCase): + + scenarios = sync_scenarios + + def create_database(self, replica_uid, sync_role=None): + if replica_uid == 'test' and sync_role is None: + # created up the chain by base class but unused + return None + db = self.create_database_for_role(replica_uid, sync_role) + if sync_role: + self._use_tracking[db] = (replica_uid, sync_role) + return db + + def create_database_for_role(self, replica_uid, sync_role): + # hook point for reuse + return DatabaseBaseTests.create_database(self, replica_uid) + + def copy_database(self, db, sync_role=None): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + db_copy = self.copy_database_for_test(self, db) + name, orig_sync_role = self._use_tracking[db] + self._use_tracking[db_copy] = ( + name + '(copy)', sync_role or orig_sync_role) + return db_copy + + def sync(self, db_from, db_to, trace_hook=None, + trace_hook_shallow=None): + from_name, from_sync_role = self._use_tracking[db_from] + to_name, to_sync_role = self._use_tracking[db_to] + if from_sync_role not in ('source', 'both'): + raise Exception("%s marked for %s use but used as source" % + (from_name, from_sync_role)) + if to_sync_role not in ('target', 'both'): + raise Exception("%s marked for %s use but used as target" % + (to_name, to_sync_role)) + return self.do_sync(self, db_from, db_to, trace_hook, + trace_hook_shallow) + + def setUp(self): + self.db = None + self.db1 = None + self.db2 = None + self.db3 = None + self.db1_copy = None + self.db2_copy = None + self._use_tracking = {} + DatabaseBaseTests.setUp(self) + + def tearDown(self): + for db in [ + self.db, self.db1, self.db2, + self.db3, self.db1_copy, self.db2_copy + ]: + if db is not None: + self.delete_db(db._dbname) + db.close() + DatabaseBaseTests.tearDown(self) + + def assertLastExchangeLog(self, db, expected): + log = getattr(db, '_last_exchange_log', None) + if log is None: + return + self.assertEqual(expected, log) + + def test_sync_tracks_db_generation_of_other(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertEqual( + (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 0}}) + + def test_sync_autoresolves(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc, doc_id='doc') + rev1 = doc1.rev + doc2 = self.db2.create_doc_from_json(simple_doc, doc_id='doc') + rev2 = doc2.rev + self.sync(self.db1, self.db2) + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) + v = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) + + def test_sync_autoresolves_moar(self): + # here we test that when a database that has a conflicted document is + # the source of a sync, and the target database has a revision of the + # conflicted document that is newer than the source database's, and + # that target's database's document's content is the same as the + # source's document's conflict's, the source's document's conflict gets + # autoresolved, and the source's document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # `-------> + # a3b2 a1b2 (autoresolved) + # `-------> + # a3b2 a3b2 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + # because the conflict is on the source, sync it another time + self.sync(self.db1, self.db2) + # make sure db2 now has the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_autoresolves_moar_backwards(self): + # here we test that when a database that has a conflicted document is + # the target of a sync, and the source database has a revision of the + # conflicted document that is newer than the target database's, and + # that source's database's document's content is the same as the + # target's document's conflict's, the target's document's conflict gets + # autoresolved, and the document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # <-------' + # a3b2 a3b2 (autoresolved and propagated) + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + revc = self.db1.get_doc_conflicts('doc')[-1].rev + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db2, self.db1) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + vecc = vectorclock.VectorClockRev(revc) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + self.assertTrue(vec3.is_newer(vecc)) + # make sure db2 now has the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_autoresolves_moar_backwards_three(self): + # same as autoresolves_moar_backwards, but with three databases (note + # all the syncs go in the same direction -- this is a more natural + # scenario): + # + # A B C + # a1 - - + # `-------> + # a1 a1 - + # `-------> + # a1 a1 a1 + # v v + # a2 a1b1 a1 + # `-------------------> + # a2 a1b1 a2 + # `-------> + # a2+a1b1 a2 + # v + # a2 a2+a1b1 a2c1 (same as a1b1) + # `-------------------> + # a2c1 a2+a1b1 a2c1 + # `-------> + # a2b2c1 a2b2c1 a2c1 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + self.db3 = self.create_database('test3', 'target') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + self.sync(self.db2, self.db3) + for db, content in [(self.db2, '{"hi": 42}'), + (self.db1, '{}'), + ]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db3) + self.sync(self.db2, self.db3) + # db2 and db3 now both have a doc of {}, but db2 has a + # conflict + doc = self.db2.get_doc('doc') + self.assertTrue(doc.has_conflicts) + revc = self.db2.get_doc_conflicts('doc')[-1].rev + self.assertEqual('{}', doc.get_json()) + self.assertEqual(self.db3.get_doc('doc').get_json(), doc.get_json()) + self.assertEqual(self.db3.get_doc('doc').rev, doc.rev) + # set db3 to have a doc of {hi:42} (same as db2 before the conflict) + doc = self.db3.get_doc('doc') + doc.set_json('{"hi": 42}') + self.db3.put_doc(doc) + rev3 = doc.rev + # sync it across to db1 + self.sync(self.db1, self.db3) + # db1 now has hi:42, with a rev that is newer than db2's doc + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertFalse(doc.has_conflicts) + self.assertEqual('{"hi": 42}', doc.get_json()) + VCR = vectorclock.VectorClockRev + self.assertTrue(VCR(rev1).is_newer(VCR(self.db2.get_doc('doc').rev))) + # so sync it to db2 + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db2.get_doc('doc') + self.assertFalse(doc.has_conflicts) + # db2's revision of the document is strictly newer than db1's before + # the sync, and db3's before that sync way back when + self.assertTrue(VCR(doc.rev).is_newer(VCR(rev1))) + self.assertTrue(VCR(doc.rev).is_newer(VCR(rev3))) + self.assertTrue(VCR(doc.rev).is_newer(VCR(revc))) + # make sure both dbs now have the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_puts_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(simple_doc) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_pulls_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(simple_doc) + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertGetDoc(self.db1, doc.doc_id, doc.rev, simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) + + def test_sync_pulling_doesnt_update_other_if_changed(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(simple_doc) + # After the local side has sent its list of docs, before we start + # receiving the "targets" response, we update the local database with a + # new record. + # When we finish synchronizing, we can notice that something locally + # was updated, and we cannot tell c2 our new updated generation + + def before_get_docs(state): + if state != 'before get_docs': + return + self.db1.create_doc_from_json(simple_doc) + + self.assertEqual(0, self.sync(self.db1, self.db2, + trace_hook=before_get_docs)) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + # c2 should not have gotten a '_record_sync_info' call, because the + # local database had been updated more than just by the messages + # returned from c2. + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + + def test_sync_doesnt_update_other_if_nothing_pulled(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(simple_doc) + + def no_record_sync_info(state): + if state != 'record_sync_info': + return + self.fail('SyncTarget.record_sync_info was called') + self.assertEqual(1, self.sync(self.db1, self.db2, + trace_hook_shallow=no_record_sync_info)) + self.assertEqual( + 1, + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) + + def test_sync_ignores_convergence(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(simple_doc) + self.db3 = self.create_database('test3', 'target') + self.assertEqual(1, self.sync(self.db1, self.db3)) + self.assertEqual(0, self.sync(self.db2, self.db3)) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_ignores_superseded(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(simple_doc) + doc_rev1 = doc.rev + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.sync(self.db2, self.db3) + new_content = '{"key": "altval"}' + doc.set_json(new_content) + self.db1.put_doc(doc) + doc_rev2 = doc.rev + self.sync(self.db2, self.db1) + self.assertLastExchangeLog( + self.db1, + {'receive': {'docs': [(doc.doc_id, doc_rev1)], + 'source_uid': 'test2', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc_rev2)], + 'last_gen': 2}}) + self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) + + def test_sync_sees_remote_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + doc1_rev = doc1.rev + new_doc = '{"key": "altval"}' + doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) + doc2_rev = doc2.rev + self.assertTransactionLog([doc1.doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc_id, doc1_rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [(doc_id, doc2_rev)], + 'last_gen': 1}}) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) + self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) + + def test_sync_sees_remote_delete_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + self.sync(self.db1, self.db2) + doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) + new_doc = '{"key": "altval"}' + doc1.set_json(new_doc) + self.db1.put_doc(doc1) + self.db2.delete_doc(doc2) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc_id, doc1.rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [(doc_id, doc2.rev)], + 'last_gen': 2}}) + self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) + self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, doc2.rev, None, False) + + def test_sync_local_race_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(simple_doc) + doc_id = doc.doc_id + doc1_rev = doc.rev + self.sync(self.db1, self.db2) + content1 = '{"key": "localval"}' + content2 = '{"key": "altval"}' + doc.set_json(content2) + self.db2.put_doc(doc) + doc2_rev2 = doc.rev + triggered = [] + + def after_whatschanged(state): + if state != 'after whats_changed': + return + triggered.append(True) + doc = self.make_document(doc_id, doc1_rev, content1) + self.db1.put_doc(doc) + + self.sync(self.db1, self.db2, trace_hook=after_whatschanged) + self.assertEqual([True], triggered) + self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) + + def test_sync_propagates_deletes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + self.sync(self.db1, self.db2) + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.db1.delete_doc(doc1) + deleted_rev = doc1.rev + self.sync(self.db1, self.db2) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db1, doc_id, deleted_rev, None, False) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, deleted_rev, None, False) + self.sync(self.db2, self.db3) + self.assertLastExchangeLog( + self.db3, + {'receive': {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test2', + 'source_gen': 2, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db3, doc_id, deleted_rev, None, False) + + def test_sync_propagates_deletes_2(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') + self.sync(self.db1, self.db2) + doc1_2 = self.db2.get_doc('the-doc') + self.db2.delete_doc(doc1_2) + self.sync(self.db1, self.db2) + self.assertGetDocIncludeDeleted( + self.db1, 'the-doc', doc1_2.rev, None, False) + + def test_sync_propagates_resolution(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + self.db3 = self.create_database('test3', 'both') + self.sync(self.db2, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db2._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.sync(self.db3, self.db1) + # update on 2 + doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') + self.db2.put_doc(doc2) + self.sync(self.db2, self.db3) + self.assertEqual(self.db3.get_doc('the-doc').rev, doc2.rev) + # update on 1 + doc1.set_json('{"a": 3}') + self.db1.put_doc(doc1) + # conflicts + self.sync(self.db2, self.db1) + self.sync(self.db3, self.db1) + self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) + self.assertTrue(self.db3.get_doc('the-doc').has_conflicts) + # resolve + conflicts = self.db2.get_doc_conflicts('the-doc') + doc4 = self.make_document('the-doc', None, '{"a": 4}') + revs = [doc.rev for doc in conflicts] + self.db2.resolve_doc(doc4, revs) + doc2 = self.db2.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc2.get_json()) + self.assertFalse(doc2.has_conflicts) + self.sync(self.db2, self.db3) + doc3 = self.db3.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc3.get_json()) + self.assertFalse(doc3.has_conflicts) + + def test_sync_supersedes_conflicts(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.create_database('test3', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') + self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') + self.sync(self.db3, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.sync(self.db3, self.db2) + self.assertEqual( + self.db2._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + doc1.set_json('{"a": 2}') + self.db1.put_doc(doc1) + self.sync(self.db3, self.db1) + # original doc1 should have been removed from conflicts + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + + def test_sync_stops_after_get_sync_info(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc) + self.sync(self.db1, self.db2) + + def put_hook(state): + self.fail("Tracehook triggered for %s" % (state,)) + + self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) + + def test_sync_detects_identical_replica_uid(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test1', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.assertRaises( + u1db_errors.InvalidReplicaUID, self.sync, self.db1, self.db2) + + def test_sync_detects_rollback_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) + + def test_sync_detects_rollback_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) + + def test_sync_detects_diverged_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, self.db3, self.db2) + + def test_sync_detects_diverged_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db2) + self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, self.db1, self.db3) + + def test_sync_detects_rollback_and_divergence_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, + self.db1_copy, self.db2) + + def test_sync_detects_rollback_and_divergence_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, + self.db1, self.db2_copy) + + def test_optional_sync_preserve_json(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + cont1 = '{"a": 2}' + cont2 = '{"b": 3}' + self.db1.create_doc_from_json(cont1, doc_id="1") + self.db2.create_doc_from_json(cont2, doc_id="2") + self.sync(self.db1, self.db2) + self.assertEqual(cont1, self.db2.get_doc("1").get_json()) + self.assertEqual(cont2, self.db1.get_doc("2").get_json()) diff --git a/tests/couch/test_sync_target.py b/tests/couch/test_sync_target.py new file mode 100644 index 00000000..0370a6d1 --- /dev/null +++ b/tests/couch/test_sync_target.py @@ -0,0 +1,343 @@ +import json + +from leap.soledad.common.l2db import SyncTarget +from leap.soledad.common.l2db import errors as u1db_errors + +from testscenarios import TestWithScenarios + +from test_soledad import u1db_tests as tests +from test_soledad.util import CouchDBTestCase +from test_soledad.util import make_local_db_and_target +from test_soledad.u1db_tests import DatabaseBaseTests + +from .common import simple_doc +from .common import nested_doc +from .common import COUCH_SCENARIOS + + +target_scenarios = [ + ('local', {'create_db_and_target': make_local_db_and_target}), ] + + +class CouchBackendSyncTargetTests( + TestWithScenarios, + DatabaseBaseTests, + CouchDBTestCase): + + # TODO: implement _set_trace_hook(_shallow) in CouchSyncTarget so + # skipped tests can be succesfully executed. + + # whitebox true means self.db is the actual local db object + # against which the sync is performed + whitebox = True + + scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios)) + + def set_trace_hook(self, callback, shallow=False): + setter = (self.st._set_trace_hook if not shallow else + self.st._set_trace_hook_shallow) + try: + setter(callback) + except NotImplementedError: + self.skipTest("%s does not implement _set_trace_hook" + % (self.st.__class__.__name__,)) + + def setUp(self): + CouchDBTestCase.setUp(self) + # other stuff + self.db, self.st = self.create_db_and_target(self) + self.other_changes = [] + + def tearDown(self): + self.db.close() + CouchDBTestCase.tearDown(self) + + def receive_doc(self, doc, gen, trans_id): + self.other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + def test_sync_exchange_returns_many_new_docs(self): + # This test was replicated to allow dictionaries to be compared after + # JSON expansion (because one dictionary may have many different + # serialized representations). + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + self.assertEqual(2, new_gen) + self.assertEqual( + [(doc.doc_id, doc.rev, json.loads(simple_doc), 1), + (doc2.doc_id, doc2.rev, json.loads(nested_doc), 2)], + [c[:-3] + (json.loads(c[-3]), c[-2]) for c in self.other_changes]) + if self.whitebox: + self.assertEqual( + self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': + [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) + + def test_get_sync_target(self): + self.assertIsNot(None, self.st) + + def test_get_sync_info(self): + self.assertEqual( + ('test', 0, '', 0, ''), self.st.get_sync_info('other')) + + def test_create_doc_updates_sync_info(self): + self.assertEqual( + ('test', 0, '', 0, ''), self.st.get_sync_info('other')) + self.db.create_doc_from_json(simple_doc) + self.assertEqual(1, self.st.get_sync_info('other')[1]) + + def test_record_sync_info(self): + self.st.record_sync_info('replica', 10, 'T-transid') + self.assertEqual( + ('test', 0, '', 10, 'T-transid'), self.st.get_sync_info('replica')) + + def test_sync_exchange(self): + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', simple_doc), 10, + 'T-sid')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) + self.assertTransactionLog(['doc-id'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, last_trans_id)) + self.assertEqual(10, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_deleted(self): + doc = self.db.create_doc_from_json('{}') + edit_rev = 'replica:1|' + doc.rev + docs_by_gen = [ + (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, edit_rev, None, False) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(10, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_push_many(self): + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', simple_doc), 10, 'T-1'), + (self.make_document('doc-id2', 'replica:1', nested_doc), 11, + 'T-2')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) + self.assertGetDoc(self.db, 'doc-id2', 'replica:1', nested_doc, False) + self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(11, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_refuses_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_ignores_convergence(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + gen, txid = self.db._get_generation_info() + docs_by_gen = [ + (self.make_document(doc.doc_id, doc.rev, simple_doc), 10, 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=gen, + last_known_trans_id=txid, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual(([], 1), (self.other_changes, new_gen)) + + def test_sync_exchange_returns_new_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_returns_deleted_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) + self.assertEqual(2, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_getting_newer_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_with_concurrent_updates_of_synced_doc(self): + expected = [] + + def before_whatschanged_cb(state): + if state != 'before whats_changed': + return + cont = '{"key": "cuncurrent"}' + conc_rev = self.db.put_doc( + self.make_document(doc.doc_id, 'test:1|z:2', cont)) + expected.append((doc.doc_id, conc_rev, cont, 3)) + + self.set_trace_hook(before_whatschanged_cb) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(expected, [c[:-1] for c in self.other_changes]) + self.assertEqual(3, new_gen) + + def test_sync_exchange_with_concurrent_updates(self): + + def after_whatschanged_cb(state): + if state != 'after whats_changed': + return + self.db.create_doc_from_json('{"new": "doc"}') + + self.set_trace_hook(after_whatschanged_cb) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_converged_handling(self): + doc = self.db.create_doc_from_json(simple_doc) + docs_by_gen = [ + (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), + (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, + 'T-bar')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_detect_incomplete_exchange(self): + def before_get_docs_explode(state): + if state != 'before get_docs': + return + raise u1db_errors.U1DBError("fail") + self.set_trace_hook(before_get_docs_explode) + # suppress traceback printing in the wsgiref server + # self.patch(simple_server.ServerHandler, + # 'log_exception', lambda h, exc_info: None) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertRaises( + (u1db_errors.U1DBError, u1db_errors.BrokenSyncStream), + self.st.sync_exchange, [], 'other-replica', + last_known_generation=0, last_known_trans_id=None, + return_doc_cb=self.receive_doc) + + def test_sync_exchange_doc_ids(self): + sync_exchange_doc_ids = getattr(self.st, 'sync_exchange_doc_ids', None) + if sync_exchange_doc_ids is None: + self.skipTest("sync_exchange_doc_ids not implemented") + db2 = self.create_database('test2') + doc = db2.create_doc_from_json(simple_doc) + new_gen, trans_id = sync_exchange_doc_ids( + db2, [(doc.doc_id, 10, 'T-sid')], 0, None, + return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + self.assertTransactionLog([doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(10, self.st.get_sync_info(db2._replica_uid)[3]) + + def test__set_trace_hook(self): + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb) + self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ], + called) + + def test__set_trace_hook_shallow(self): + st_trace_shallow = self.st._set_trace_hook_shallow + target_st_trace_shallow = SyncTarget._set_trace_hook_shallow + same_meth = st_trace_shallow == self.st._set_trace_hook + same_fun = st_trace_shallow.im_func == target_st_trace_shallow.im_func + if (same_meth or same_fun): + # shallow same as full + expected = ['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ] + else: + expected = ['sync_exchange', 'record_sync_info'] + + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb, shallow=True) + self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(expected, called) diff --git a/tests/pipes/test_pipes.py b/tests/pipes/test_pipes.py new file mode 100644 index 00000000..42ed81ac --- /dev/null +++ b/tests/pipes/test_pipes.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# test_pipes.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for streaming components. +""" +from twisted.trial import unittest +from leap.soledad.client._pipes import TruncatedTailPipe +from leap.soledad.client._pipes import PreamblePipe +from io import BytesIO + + +class TruncatedTailPipeTestCase(unittest.TestCase): + + def test_tail_truncating_pipe(self): + pipe = TruncatedTailPipe(tail_size=20) + payload = 'A' * 100 + 'B' * 20 + for data in payload: + pipe.write(data) + result = pipe.close() + assert result.getvalue() == 'A' * 100 + + +class PreamblePipeTestCase(unittest.TestCase): + + def test_preamble_pipe(self): + remaining = BytesIO() + preamble = BytesIO() + + def callback(dataBuffer): + preamble.write(dataBuffer.getvalue()) + return remaining + pipe = PreamblePipe(callback) + payload = 'A' * 100 + ' ' + 'B' * 20 + for data in payload: + pipe.write(data) + assert remaining.getvalue() == 'B' * 20 + assert preamble.getvalue() == 'A' * 100 diff --git a/tests/responsiveness/conftest.py b/tests/responsiveness/conftest.py new file mode 100644 index 00000000..a46aea44 --- /dev/null +++ b/tests/responsiveness/conftest.py @@ -0,0 +1,20 @@ +import pytest + +import elastic +import watchdog as wd + + +def _post_results(dog, request): + elastic.post(dog.seconds_blocked, request) + + +@pytest.fixture +def watchdog(request): + dog = wd.Watchdog() + dog_d = dog.start() + request.addfinalizer(lambda: _post_results(dog, request)) + + def _run(deferred_fun): + deferred_fun().addCallback(lambda _: dog.stop()) + return dog_d + return _run diff --git a/tests/responsiveness/elastic.py b/tests/responsiveness/elastic.py new file mode 100644 index 00000000..fed1506b --- /dev/null +++ b/tests/responsiveness/elastic.py @@ -0,0 +1,47 @@ +import datetime +import elasticsearch + +from pytest_benchmark.plugin import pytest_benchmark_generate_machine_info +from pytest_benchmark.utils import get_commit_info, get_tag, get_machine_id +from pytest_benchmark.storage.elasticsearch import BenchmarkJSONSerializer + + +def post(seconds_blocked, request,): + body, doc_id = get_doc(seconds_blocked, request) + url = request.config.getoption("elasticsearch_url") + if url: + es = elasticsearch.Elasticsearch( + hosts=[url], + serializer=BenchmarkJSONSerializer()) + es.index( + index='responsiveness', + doc_type='responsiveness', + id=doc_id, + body=body) + else: + print body + + +def get_doc(seconds_blocked, request): + fullname = request.node._nodeid + name = request.node.name + group = None + marker = request.node.get_marker("responsivness") + if marker: + group = marker.kwargs.get("group") + + doc = { + "datetime": datetime.datetime.utcnow().isoformat(), + "commit_info": get_commit_info(), + "fullname": fullname, + "name": name, + "group": group, + "machine_info": pytest_benchmark_generate_machine_info(), + } + + # generate a doc id like the one used by pytest-benchmark + machine_id = get_machine_id() + tag = get_tag() + doc_id = machine_id + "_" + tag + "_" + fullname + + return doc, doc_id diff --git a/tests/responsiveness/test_responsiveness.py b/tests/responsiveness/test_responsiveness.py new file mode 100644 index 00000000..b3e2c56a --- /dev/null +++ b/tests/responsiveness/test_responsiveness.py @@ -0,0 +1,50 @@ +import pytest + +from twisted.internet import defer + + +@pytest.inlineCallbacks +def load_up(client, amount, payload): + # create a bunch of local documents + deferreds = [] + for i in xrange(amount): + deferreds.append(client.create_doc({'content': payload})) + yield defer.gatherResults(deferreds) + + +def create_upload(amount, size): + + @pytest.mark.responsiveness + @pytest.inlineCallbacks + def _test(soledad_client, payload, watchdog): + + client = soledad_client() + yield load_up(client, amount, payload(size)) + yield watchdog(client.sync) + + return _test + + +test_responsiveness_upload_10_1000k = create_upload(10, 1000 * 1000) +test_responsiveness_upload_100_100k = create_upload(100, 100 * 1000) +test_responsiveness_upload_1000_10k = create_upload(1000, 10 * 1000) + + +def create_download(downloads, size): + + @pytest.mark.responsiveness + @pytest.inlineCallbacks + def _test(soledad_client, payload, watchdog): + client = soledad_client() + yield load_up(client, downloads, payload(size)) + yield client.sync() + + clean_client = soledad_client(force_fresh_db=True) + yield watchdog(clean_client.sync) + + return _test + + +test_responsiveness_download_10_1000k = create_download(10, 1000 * 1000) +test_responsiveness_download_100_100k = create_download(100, 100 * 1000) +test_responsiveness_download_1000_10k = create_download(1000, 10 * 1000) diff --git a/tests/responsiveness/watchdog.py b/tests/responsiveness/watchdog.py new file mode 100644 index 00000000..88f4fa67 --- /dev/null +++ b/tests/responsiveness/watchdog.py @@ -0,0 +1,53 @@ +from twisted.internet import defer, reactor +from twisted.internet.task import LoopingCall +from twisted.internet.threads import deferToThread + + +class Watchdog(object): + + DEBUG = False + + def __init__(self, delay=0.01): + self.delay = delay + self.loop_call = LoopingCall.withCount(self.watch) + self.blocked = 0 + self.checks = [] + self.d = None + + def start(self): + self.debug("\n[watchdog] starting") + self.loop_call.start(self.delay) + self.d = defer.Deferred() + return self.d + + def watch(self, count): + self.debug("[watchdog] watching (%d)" % count) + if (self.loop_call.running): + self.checks.append(deferToThread(self._check, count)) + + def _check(self, count): + # self.debug("[watchdog] _checking (%d)" % count) + if count > 1: + self.blocked += count + + def stop(self): + # delay the actual stop so we make sure at least one check watch will + # run in the reactor. + reactor.callLater(2 * self.delay, self._stop) + + @defer.inlineCallbacks + def _stop(self): + if not self.loop_call.running: + return + + self.loop_call.stop() + yield defer.gatherResults(self.checks) + self.d.callback(None) + + @property + def seconds_blocked(self): + return self.blocked * self.delay + + def debug(self, s): + if self.DEBUG: + print(s) diff --git a/tests/server/__init__.py b/tests/server/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/server/test__resource.py b/tests/server/test__resource.py new file mode 100644 index 00000000..a43ac19f --- /dev/null +++ b/tests/server/test__resource.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +# test__resource.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for Soledad server main resource. +""" +from twisted.trial import unittest +from twisted.web.test.test_web import DummyRequest +from twisted.web.wsgi import WSGIResource +from twisted.web.resource import getChildForRequest +from twisted.internet import reactor + +from leap.soledad.server._resource import PublicResource +from leap.soledad.server._resource import LocalResource +from leap.soledad.server._server_info import ServerInfo +from leap.soledad.server._blobs import BlobsResource +from leap.soledad.server._incoming import IncomingResource +from leap.soledad.server.gzip_middleware import GzipMiddleware + + +_pool = reactor.getThreadPool() + + +class PublicResourceTestCase(unittest.TestCase): + + def test_get_root(self): + blobs_resource = None # doesn't matter + resource = PublicResource( + blobs_resource=blobs_resource, sync_pool=_pool) + request = DummyRequest(['']) + child = getChildForRequest(resource, request) + self.assertIsInstance(child, ServerInfo) + + def test_get_blobs_enabled(self): + blobs_resource = BlobsResource("filesystem", '/tmp') + resource = PublicResource( + blobs_resource=blobs_resource, sync_pool=_pool) + request = DummyRequest(['blobs']) + child = getChildForRequest(resource, request) + self.assertIsInstance(child, BlobsResource) + + def test_get_blobs_disabled(self): + blobs_resource = None + resource = PublicResource( + blobs_resource=blobs_resource, sync_pool=_pool) + request = DummyRequest(['blobs']) + child = getChildForRequest(resource, request) + # if blobs is disabled, the request should be routed to sync + self.assertIsInstance(child, WSGIResource) + self.assertIsInstance(child._application, GzipMiddleware) + + def test_get_sync(self): + blobs_resource = None # doesn't matter + resource = PublicResource( + blobs_resource=blobs_resource, sync_pool=_pool) + request = DummyRequest(['user-db', 'sync-from', 'source-id']) + child = getChildForRequest(resource, request) + self.assertIsInstance(child, WSGIResource) + self.assertIsInstance(child._application, GzipMiddleware) + + def test_no_incoming_on_public_resource(self): + resource = PublicResource(None, sync_pool=_pool) + request = DummyRequest(['incoming']) + child = getChildForRequest(resource, request) + # WSGIResource is returned if a path is unknown + self.assertIsInstance(child, WSGIResource) + + def test_get_incoming(self): + resource = LocalResource() + request = DummyRequest(['incoming']) + child = getChildForRequest(resource, request) + self.assertIsInstance(child, IncomingResource) diff --git a/tests/server/test__server_info.py b/tests/server/test__server_info.py new file mode 100644 index 00000000..40567ef1 --- /dev/null +++ b/tests/server/test__server_info.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# test__server_info.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Tests for Soledad server information announcement. +""" +import json + +from twisted.trial import unittest +from twisted.web.test.test_web import DummyRequest + +from leap.soledad.server._server_info import ServerInfo + + +class ServerInfoTestCase(unittest.TestCase): + + def test_blobs_enabled(self): + resource = ServerInfo(True) + response = resource.render(DummyRequest([''])) + _info = json.loads(response) + self.assertEquals(_info['blobs'], True) + self.assertTrue(isinstance(_info['version'], basestring)) + + def test_blobs_disabled(self): + resource = ServerInfo(False) + response = resource.render(DummyRequest([''])) + _info = json.loads(response) + self.assertEquals(_info['blobs'], False) + self.assertTrue(isinstance(_info['version'], basestring)) diff --git a/tests/server/test_auth.py b/tests/server/test_auth.py new file mode 100644 index 00000000..78cf20ab --- /dev/null +++ b/tests/server/test_auth.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# test_auth.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for auth pieces. +""" +import os +import collections +import pytest + +from contextlib import contextmanager + +from twisted.cred.credentials import UsernamePassword +from twisted.cred.error import UnauthorizedLogin +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest +from twisted.web.resource import IResource +from twisted.web.test import test_httpauth + +import leap.soledad.server.auth as auth_module +from leap.soledad.server.auth import SoledadRealm +from leap.soledad.server.auth import CouchDBTokenChecker +from leap.soledad.server.auth import FileTokenChecker +from leap.soledad.server.auth import TokenCredentialFactory +from leap.soledad.server._resource import PublicResource + + +class SoledadRealmTestCase(unittest.TestCase): + + def test_returned_resource(self): + # we have to pass a pool to the realm , otherwise tests will hang + conf = {'blobs': False} + pool = reactor.getThreadPool() + realm = SoledadRealm(conf=conf, sync_pool=pool) + iface, avatar, logout = realm.requestAvatar('any', None, IResource) + self.assertIsInstance(avatar, PublicResource) + self.assertIsNone(logout()) + + +class DummyServer(object): + """ + I fake the `couchdb.client.Server` GET api and always return the token + given on my creation. + """ + + def __init__(self, token): + self._token = token + + def get(self, _): + return self._token + + +@contextmanager +def dummy_server(token): + yield collections.defaultdict(lambda: DummyServer(token)) + + +class CouchDBTokenCheckerTestCase(unittest.TestCase): + + @inlineCallbacks + def test_good_creds(self): + # set up a dummy server which always return a *valid* token document + token = {'user_id': 'user', 'type': 'Token'} + server = dummy_server(token) + # setup the checker with the custom server + checker = CouchDBTokenChecker() + auth_module.couch_server = lambda url: server + # assert the checker *can* verify the creds + creds = UsernamePassword('user', 'pass') + avatarId = yield checker.requestAvatarId(creds) + self.assertEqual('user', avatarId) + + @inlineCallbacks + def test_bad_creds(self): + # set up a dummy server which always return an *invalid* token document + token = None + server = dummy_server(token) + # setup the checker with the custom server + checker = CouchDBTokenChecker() + auth_module.couch_server = lambda url: server + # assert the checker *cannot* verify the creds + creds = UsernamePassword('user', '') + with self.assertRaises(UnauthorizedLogin): + yield checker.requestAvatarId(creds) + + +class FileTokenCheckerTestCase(unittest.TestCase): + + @inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_good_creds(self): + auth_file_path = os.path.join(self.tempdir, 'auth.file') + with open(auth_file_path, 'w') as tempfile: + tempfile.write('goodservice:goodtoken') + # setup the checker with the auth tokens file + conf = {'services_tokens_file': auth_file_path} + checker = FileTokenChecker(conf) + # assert the checker *can* verify the creds + creds = UsernamePassword('goodservice', 'goodtoken') + avatarId = yield checker.requestAvatarId(creds) + self.assertEqual('goodservice', avatarId) + + @inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_bad_creds(self): + auth_file_path = os.path.join(self.tempdir, 'auth.file') + with open(auth_file_path, 'w') as tempfile: + tempfile.write('service:token') + # setup the checker with the auth tokens file + conf = {'services_tokens_file': auth_file_path} + checker = FileTokenChecker(conf) + # assert the checker *cannot* verify the creds + creds = UsernamePassword('service', 'wrongtoken') + with self.assertRaises(UnauthorizedLogin): + yield checker.requestAvatarId(creds) + + +class TokenCredentialFactoryTestcase( + test_httpauth.RequestMixin, test_httpauth.BasicAuthTestsMixin, + unittest.TestCase): + + def setUp(self): + test_httpauth.BasicAuthTestsMixin.setUp(self) + self.credentialFactory = TokenCredentialFactory() diff --git a/tests/server/test_blobs_resource_validation.py b/tests/server/test_blobs_resource_validation.py new file mode 100644 index 00000000..9f6dfc2f --- /dev/null +++ b/tests/server/test_blobs_resource_validation.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# test_blobs_resource_validation.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for invalid user or blob_id on blobs resource +""" +import pytest +from twisted.trial import unittest +from twisted.web.test.test_web import DummyRequest +from leap.soledad.server import _blobs as server_blobs + + +class BlobServerTestCase(unittest.TestCase): + + @pytest.mark.usefixtures("method_tmpdir") + def setUp(self): + self.resource = server_blobs.BlobsResource("filesystem", self.tempdir) + + @pytest.mark.usefixtures("method_tmpdir") + def test_valid_arguments(self): + request = DummyRequest(['v4l1d-us3r', 'v4l1d-bl0b-1d']) + self.assertTrue(self.resource._validate(request)) + + @pytest.mark.usefixtures("method_tmpdir") + def test_invalid_user_get(self): + request = DummyRequest(['invalid user', 'valid-blob-id']) + request.path = '/blobs/' + with pytest.raises(Exception): + self.resource.render_GET(request) + + @pytest.mark.usefixtures("method_tmpdir") + def test_invalid_user_put(self): + request = DummyRequest(['invalid user', 'valid-blob-id']) + request.path = '/blobs/' + with pytest.raises(Exception): + self.resource.render_PUT(request) + + @pytest.mark.usefixtures("method_tmpdir") + def test_invalid_blob_id_get(self): + request = DummyRequest(['valid-user', 'invalid blob id']) + request.path = '/blobs/' + with pytest.raises(Exception): + self.resource.render_GET(request) + + @pytest.mark.usefixtures("method_tmpdir") + def test_invalid_blob_id_put(self): + request = DummyRequest(['valid-user', 'invalid blob id']) + request.path = '/blobs/' + with pytest.raises(Exception): + self.resource.render_PUT(request) diff --git a/tests/server/test_blobs_server.py b/tests/server/test_blobs_server.py new file mode 100644 index 00000000..9eddf108 --- /dev/null +++ b/tests/server/test_blobs_server.py @@ -0,0 +1,286 @@ +# -*- coding: utf-8 -*- +# test_blobs_server.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Integration tests for blobs server +""" +import os +import pytest +from uuid import uuid4 +from io import BytesIO +from twisted.trial import unittest +from twisted.web.server import Site +from twisted.internet import reactor +from twisted.internet import defer +from treq._utils import set_global_pool + +from leap.soledad.common.blobs import Flags +from leap.soledad.server import _blobs as server_blobs +from leap.soledad.client._db.blobs import BlobManager +from leap.soledad.client._db.blobs import BlobAlreadyExistsError +from leap.soledad.client._db.blobs import InvalidFlagsError +from leap.soledad.client._db.blobs import SoledadError + + +class BlobServerTestCase(unittest.TestCase): + + def setUp(self): + root = server_blobs.BlobsResource("filesystem", self.tempdir) + site = Site(root) + self.port = reactor.listenTCP(0, site, interface='127.0.0.1') + self.host = self.port.getHost() + self.uri = 'http://%s:%s/' % (self.host.host, self.host.port) + self.secret = 'A' * 96 + set_global_pool(None) + + def tearDown(self): + self.port.stopListening() + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_upload_download(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + fd = BytesIO("save me") + yield manager._encrypt_and_upload('blob_id', fd) + blob, size = yield manager._download_and_decrypt('blob_id') + self.assertEquals(blob.getvalue(), "save me") + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_set_get_flags(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + fd = BytesIO("flag me") + yield manager._encrypt_and_upload('blob_id', fd) + yield manager.set_flags('blob_id', [Flags.PROCESSING]) + flags = yield manager.get_flags('blob_id') + self.assertEquals([Flags.PROCESSING], flags) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_set_flags_raises_if_no_blob_found(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + with pytest.raises(SoledadError): + yield manager.set_flags('missing_id', [Flags.PENDING]) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_list_filter_flag(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + fd = BytesIO("flag me") + yield manager._encrypt_and_upload('blob_id', fd) + yield manager.set_flags('blob_id', [Flags.PROCESSING]) + blobs_list = yield manager.remote_list(filter_flag=Flags.PENDING) + self.assertEquals([], blobs_list) + blobs_list = yield manager.remote_list(filter_flag=Flags.PROCESSING) + self.assertEquals(['blob_id'], blobs_list) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_list_filter_flag_order_by_date(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + yield manager._encrypt_and_upload('blob_id1', BytesIO("x")) + yield manager._encrypt_and_upload('blob_id2', BytesIO("x")) + yield manager._encrypt_and_upload('blob_id3', BytesIO("x")) + yield manager.set_flags('blob_id1', [Flags.PROCESSING]) + yield manager.set_flags('blob_id2', [Flags.PROCESSING]) + yield manager.set_flags('blob_id3', [Flags.PROCESSING]) + blobs_list = yield manager.remote_list(filter_flag=Flags.PROCESSING, + order_by='+date') + expected_list = ['blob_id1', 'blob_id2', 'blob_id3'] + self.assertEquals(expected_list, blobs_list) + blobs_list = yield manager.remote_list(filter_flag=Flags.PROCESSING, + order_by='-date') + self.assertEquals(list(reversed(expected_list)), blobs_list) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_cant_set_invalid_flags(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + fd = BytesIO("flag me") + yield manager._encrypt_and_upload('blob_id', fd) + with pytest.raises(InvalidFlagsError): + yield manager.set_flags('blob_id', ['invalid']) + flags = yield manager.get_flags('blob_id') + self.assertEquals([], flags) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_get_empty_flags(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + fd = BytesIO("flag me") + yield manager._encrypt_and_upload('blob_id', fd) + flags = yield manager.get_flags('blob_id') + self.assertEquals([], flags) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_flags_ignored_by_listing(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + fd = BytesIO("flag me") + yield manager._encrypt_and_upload('blob_id', fd) + yield manager.set_flags('blob_id', [Flags.PROCESSING]) + blobs_list = yield manager.remote_list() + self.assertEquals(['blob_id'], blobs_list) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_upload_changes_remote_list(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + yield manager._encrypt_and_upload('blob_id1', BytesIO("1")) + yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) + blobs_list = yield manager.remote_list() + self.assertEquals(set(['blob_id1', 'blob_id2']), set(blobs_list)) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_list_orders_by_date(self): + user_uid = uuid4().hex + manager = BlobManager('', self.uri, self.secret, + self.secret, user_uid) + yield manager._encrypt_and_upload('blob_id1', BytesIO("1")) + yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) + blobs_list = yield manager.remote_list(order_by='date') + self.assertEquals(['blob_id1', 'blob_id2'], blobs_list) + parts = [user_uid, 'default', 'b', 'blo', 'blob_i', 'blob_id1'] + self.__touch(self.tempdir, *parts) + blobs_list = yield manager.remote_list(order_by='+date') + self.assertEquals(['blob_id2', 'blob_id1'], blobs_list) + blobs_list = yield manager.remote_list(order_by='-date') + self.assertEquals(['blob_id1', 'blob_id2'], blobs_list) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_count(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + deferreds = [] + for i in range(10): + deferreds.append(manager._encrypt_and_upload(str(i), BytesIO("1"))) + yield defer.gatherResults(deferreds) + + result = yield manager.count() + self.assertEquals({"count": len(deferreds)}, result) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_list_restricted_by_namespace(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + namespace = 'incoming' + yield manager._encrypt_and_upload('blob_id1', BytesIO("1"), + namespace=namespace) + yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) + blobs_list = yield manager.remote_list(namespace=namespace) + self.assertEquals(['blob_id1'], blobs_list) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_list_default_doesnt_list_other_namespaces(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + namespace = 'incoming' + yield manager._encrypt_and_upload('blob_id1', BytesIO("1"), + namespace=namespace) + yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) + blobs_list = yield manager.remote_list() + self.assertEquals(['blob_id2'], blobs_list) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_download_from_namespace(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + namespace, blob_id, content = 'incoming', 'blob_id1', 'test' + yield manager._encrypt_and_upload(blob_id, BytesIO(content), + namespace=namespace) + got_blob = yield manager._download_and_decrypt(blob_id, namespace) + self.assertEquals(content, got_blob[0].getvalue()) + + def __touch(self, *args): + path = os.path.join(*args) + with open(path, 'a'): + os.utime(path, None) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_upload_deny_duplicates(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + fd = BytesIO("save me") + yield manager._encrypt_and_upload('blob_id', fd) + fd = BytesIO("save me") + with pytest.raises(BlobAlreadyExistsError): + yield manager._encrypt_and_upload('blob_id', fd) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_send_missing(self): + manager = BlobManager(self.tempdir, self.uri, self.secret, + self.secret, uuid4().hex) + self.addCleanup(manager.close) + blob_id = 'local_only_blob_id' + yield manager.local.put(blob_id, BytesIO("X"), size=1) + yield manager.send_missing() + result = yield manager._download_and_decrypt(blob_id) + self.assertIsNotNone(result) + self.assertEquals(result[0].getvalue(), "X") + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_fetch_missing(self): + manager = BlobManager(self.tempdir, self.uri, self.secret, + self.secret, uuid4().hex) + self.addCleanup(manager.close) + blob_id = 'remote_only_blob_id' + yield manager._encrypt_and_upload(blob_id, BytesIO("X")) + yield manager.fetch_missing() + result = yield manager.local.get(blob_id) + self.assertIsNotNone(result) + self.assertEquals(result.getvalue(), "X") + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_upload_then_delete_updates_list(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + yield manager._encrypt_and_upload('blob_id1', BytesIO("1")) + yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) + yield manager._delete_from_remote('blob_id1') + blobs_list = yield manager.remote_list() + self.assertEquals(set(['blob_id2']), set(blobs_list)) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_upload_then_delete_updates_list_using_namespace(self): + manager = BlobManager('', self.uri, self.secret, + self.secret, uuid4().hex) + namespace = 'special_archives' + yield manager._encrypt_and_upload('blob_id1', BytesIO("1"), + namespace=namespace) + yield manager._encrypt_and_upload('blob_id2', BytesIO("2"), + namespace=namespace) + yield manager._delete_from_remote('blob_id1', namespace=namespace) + blobs_list = yield manager.remote_list(namespace=namespace) + self.assertEquals(set(['blob_id2']), set(blobs_list)) diff --git a/tests/server/test_config.py b/tests/server/test_config.py new file mode 100644 index 00000000..dfb09f4c --- /dev/null +++ b/tests/server/test_config.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# test_config.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for server configuration. +""" + +from twisted.trial import unittest +from pkg_resources import resource_filename + +from leap.soledad.server._config import _load_config +from leap.soledad.server._config import CONFIG_DEFAULTS + + +class ConfigurationParsingTest(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + + def test_use_defaults_on_failure(self): + config = _load_config('this file will never exist') + expected = CONFIG_DEFAULTS + self.assertEquals(expected, config) + + def test_security_values_configuration(self): + # given + config_path = resource_filename('test_soledad', + 'fixture_soledad.conf') + # when + config = _load_config(config_path) + + # then + expected = {'members': ['user1', 'user2'], + 'members_roles': ['role1', 'role2'], + 'admins': ['user3', 'user4'], + 'admins_roles': ['role3', 'role3']} + self.assertDictEqual(expected, config['database-security']) + + def test_server_values_configuration(self): + # given + config_path = resource_filename('test_soledad', + 'fixture_soledad.conf') + # when + config = _load_config(config_path) + + # then + expected = {'couch_url': + 'http://soledad:passwd@localhost:5984', + 'create_cmd': + 'sudo -u soledad-admin /usr/bin/soledad-create-userdb', + 'admin_netrc': + '/etc/couchdb/couchdb-soledad-admin.netrc', + 'batching': False, + 'blobs': False, + 'services_tokens_file': '/etc/soledad/services.tokens', + 'blobs_path': '/var/lib/soledad/blobs'} + self.assertDictEqual(expected, config['soledad-server']) diff --git a/tests/server/test_incoming_flow_integration.py b/tests/server/test_incoming_flow_integration.py new file mode 100644 index 00000000..b492534f --- /dev/null +++ b/tests/server/test_incoming_flow_integration.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# test_incoming_flow_integration.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Integration tests for the complete flow of IncomingBox feature +""" +import pytest +from uuid import uuid4 +from twisted.trial import unittest +from twisted.web.server import Site +from twisted.internet import reactor +from twisted.internet import defer +from twisted.web.resource import Resource +from zope.interface import implementer + +from leap.soledad.client.incoming import IncomingBoxProcessingLoop +from leap.soledad.client.incoming import IncomingBox +from leap.soledad.server import _blobs as server_blobs +from leap.soledad.client._db.blobs import BlobManager +from leap.soledad.server._incoming import IncomingResource +from leap.soledad.server._blobs import BlobsServerState +from leap.soledad.client import interfaces + + +@implementer(interfaces.IIncomingBoxConsumer) +class GoodConsumer(object): + def __init__(self): + self.name = 'GoodConsumer' + self.processed, self.saved = [], [] + + def process(self, item, item_id, encrypted=True): + self.processed.append(item_id) + return defer.succeed([item_id]) + + def save(self, parts, item_id): + self.saved.append(item_id) + return defer.succeed(None) + + +class IncomingFlowIntegrationTestCase(unittest.TestCase): + + def setUp(self): + root = Resource() + state = BlobsServerState('filesystem', blobs_path=self.tempdir) + incoming_resource = IncomingResource(state) + blobs_resource = server_blobs.BlobsResource("filesystem", self.tempdir) + root.putChild('blobs', blobs_resource) + root.putChild('incoming', incoming_resource) + site = Site(root) + self.port = reactor.listenTCP(0, site, interface='127.0.0.1') + self.host = self.port.getHost() + self.uri = 'http://%s:%s/' % (self.host.host, self.host.port) + self.blobs_uri = self.uri + 'blobs/' + self.incoming_uri = self.uri + 'incoming' + self.user_id = 'user-' + uuid4().hex + self.secret = 'A' * 96 + self.blob_manager = BlobManager(self.tempdir, self.blobs_uri, + self.secret, self.secret, + self.user_id) + self.box = IncomingBox(self.blob_manager, 'MX') + self.loop = IncomingBoxProcessingLoop(self.box) + # FIXME: We use blob_manager client only to avoid DelayedCalls + # Somehow treq being used here keeps a connection pool open + self.client = self.blob_manager._client + + def fill(self, messages): + deferreds = [] + for message_id, message in messages: + uri = '%s/%s/%s' % (self.incoming_uri, self.user_id, message_id) + deferreds.append(self.blob_manager._client.put(uri, data=message)) + return defer.gatherResults(deferreds) + + def tearDown(self): + self.port.stopListening() + self.blob_manager.close() + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_consume_a_incoming_message(self): + yield self.fill([('msg1', 'blob')]) + consumer = GoodConsumer() + self.loop.add_consumer(consumer) + yield self.loop() + self.assertIn('msg1', consumer.processed) diff --git a/tests/server/test_incoming_resource.py b/tests/server/test_incoming_resource.py new file mode 100644 index 00000000..0d4918b9 --- /dev/null +++ b/tests/server/test_incoming_resource.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# test_incoming_resource.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Unit tests for incoming API resource +""" +from twisted.trial import unittest +from twisted.web.test.test_web import DummyRequest +from leap.soledad.server._incoming import IncomingResource +from leap.soledad.server._incoming import IncomingFormatter +from leap.soledad.common.crypto import EncryptionSchemes +from io import BytesIO +from uuid import uuid4 +from mock import Mock + + +class IncomingResourceTestCase(unittest.TestCase): + + def setUp(self): + self.couchdb = Mock() + self.backend_factory = Mock() + self.backend_factory.open_database.return_value = self.couchdb + self.resource = IncomingResource(self.backend_factory) + self.user_uuid = uuid4().hex + + def test_save_document(self): + formatter = IncomingFormatter() + doc_id, scheme = uuid4().hex, EncryptionSchemes.PUBKEY + content = 'Incoming content' + request = DummyRequest([self.user_uuid, doc_id]) + request.content = BytesIO(content) + self.resource.render_PUT(request) + + open_database = self.backend_factory.open_database + open_database.assert_called_once_with(self.user_uuid) + self.couchdb.put_doc.assert_called_once() + doc = self.couchdb.put_doc.call_args[0][0] + self.assertEquals(doc_id, doc.doc_id) + self.assertEquals(formatter.format(content, scheme), doc.content) + + def test_formatter(self): + formatter = IncomingFormatter() + formatted = formatter.format('content', EncryptionSchemes.PUBKEY) + self.assertEquals(formatted['_enc_scheme'], EncryptionSchemes.PUBKEY) diff --git a/tests/server/test_incoming_server.py b/tests/server/test_incoming_server.py new file mode 100644 index 00000000..241bc581 --- /dev/null +++ b/tests/server/test_incoming_server.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# test_incoming_server.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Integration tests for incoming API +""" +import pytest +import json +from io import BytesIO +from uuid import uuid4 +from twisted.web.test.test_web import DummyRequest +from twisted.web.server import Site +from twisted.internet import reactor +from twisted.internet import defer +import treq + +from leap.soledad.server._incoming import IncomingResource +from leap.soledad.server._blobs import BlobsServerState +from leap.soledad.server._incoming import IncomingFormatter +from leap.soledad.common.crypto import EncryptionSchemes +from leap.soledad.common.blobs import Flags +from test_soledad.util import CouchServerStateForTests +from test_soledad.util import CouchDBTestCase + + +class IncomingOnCouchServerTestCase(CouchDBTestCase): + + def setUp(self): + self.port = None + + def tearDown(self): + if self.port: + self.port.stopListening() + + def prepare(self, backend): + self.user_id = 'user-' + uuid4().hex + if backend == 'couch': + self.state = CouchServerStateForTests(self.couch_url) + self.state.ensure_database(self.user_id) + else: + self.state = BlobsServerState(backend) + root = IncomingResource(self.state) + site = Site(root) + self.port = reactor.listenTCP(0, site, interface='127.0.0.1') + self.host = self.port.getHost() + self.uri = 'http://%s:%s/' % (self.host.host, self.host.port) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_put_incoming_creates_a_document_using_couch(self): + self.prepare('couch') + user_id, doc_id = self.user_id, uuid4().hex + content, scheme = 'Hi', EncryptionSchemes.PUBKEY + formatter = IncomingFormatter() + incoming_endpoint = self.uri + '%s/%s' % (user_id, doc_id) + yield treq.put(incoming_endpoint, BytesIO(content), persistent=False) + db = self.state.open_database(user_id) + + doc = db.get_doc(doc_id) + self.assertEquals(doc.content, formatter.format(content, scheme)) + + @defer.inlineCallbacks + @pytest.mark.usefixtures("method_tmpdir") + def test_put_incoming_creates_a_blob_using_filesystem(self): + self.prepare('filesystem') + user_id, doc_id = self.user_id, uuid4().hex + content = 'Hi' + formatter = IncomingFormatter() + incoming_endpoint = self.uri + '%s/%s' % (user_id, doc_id) + yield treq.put(incoming_endpoint, BytesIO(content), persistent=False) + + db = self.state.open_database(user_id) + request = DummyRequest([user_id, doc_id]) + yield db.read_blob(user_id, doc_id, request, 'MX') + flags = db.get_flags(user_id, doc_id, request, 'MX') + flags = json.loads(flags) + expected = formatter.preamble(content, doc_id) + ' ' + content + self.assertEquals(expected, request.written[0]) + self.assertIn(Flags.PENDING, flags) diff --git a/tests/server/test_server.py b/tests/server/test_server.py new file mode 100644 index 00000000..25f0cc2d --- /dev/null +++ b/tests/server/test_server.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- +# test_server.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for server-related functionality. +""" +import binascii +import os +import pytest + +from six.moves.urllib.parse import urljoin +from uuid import uuid4 + +from twisted.internet import defer + +from leap.soledad.common.couch.state import CouchServerState +from leap.soledad.common.couch import CouchDatabase +from test_soledad.u1db_tests import TestCaseWithServer +from test_soledad.util import CouchDBTestCase +from test_soledad.util import ( + make_token_soledad_app, + make_soledad_document_for_test, + soledad_sync_target, +) + +from leap.soledad.client import _crypto +from leap.soledad.client import Soledad + + +@pytest.mark.needs_couch +@pytest.mark.usefixtures("method_tmpdir") +class EncryptedSyncTestCase( + CouchDBTestCase, TestCaseWithServer): + + """ + Tests for encrypted sync using Soledad server backed by a couch database. + """ + + # increase twisted.trial's timeout because large files syncing might take + # some time to finish. + timeout = 500 + + @staticmethod + def make_app_with_state(state): + return make_token_soledad_app(state) + + make_document_for_test = make_soledad_document_for_test + + sync_target = soledad_sync_target + + def _soledad_instance(self, user=None, passphrase=u'123', + prefix='', + secrets_path='secrets.json', + local_db_path='soledad.u1db', + server_url='', + cert_file=None, auth_token=None): + """ + Instantiate Soledad. + """ + + # this callback ensures we save a document which is sent to the shared + # db. + def _put_doc_side_effect(doc): + self._doc_put = doc + + if not server_url: + # attempt to find the soledad server url + server_address = None + server = getattr(self, 'server', None) + if server: + server_address = getattr(self.server, 'server_address', None) + else: + host = self.port.getHost() + server_address = (host.host, host.port) + if server_address: + server_url = 'http://%s:%d' % (server_address) + + return Soledad( + user, + passphrase, + secrets_path=os.path.join(self.tempdir, prefix, secrets_path), + local_db_path=os.path.join( + self.tempdir, prefix, local_db_path), + server_url=server_url, + cert_file=cert_file, + auth_token=auth_token, + shared_db=self.get_default_shared_mock(_put_doc_side_effect)) + + def make_app(self): + self.request_state = CouchServerState(self.couch_url) + return self.make_app_with_state(self.request_state) + + def setUp(self): + CouchDBTestCase.setUp(self) + TestCaseWithServer.setUp(self) + + def tearDown(self): + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) + + def _test_encrypted_sym_sync(self, passphrase=u'123', doc_size=2, + number_of_docs=1): + """ + Test the complete syncing chain between two soledad dbs using a + Soledad server backed by a couch database. + """ + self.startTwistedServer() + user = 'user-' + uuid4().hex + + # this will store all docs ids to avoid get_all_docs + created_ids = [] + + # instantiate soledad and create a document + sol1 = self._soledad_instance( + user=user, + # token is verified in test_target.make_token_soledad_app + auth_token='auth-token', + passphrase=passphrase) + + # instantiate another soledad using the same secret as the previous + # one (so we can correctly verify the mac of the synced document) + sol2 = self._soledad_instance( + user=user, + prefix='x', + auth_token='auth-token', + secrets_path=sol1.secrets_path, + passphrase=passphrase) + + # ensure remote db exists before syncing + db = CouchDatabase.open_database( + urljoin(self.couch_url, 'user-' + user), + create=True) + + def _db1AssertEmptyDocList(results): + _, doclist = results + self.assertEqual([], doclist) + + def _db1CreateDocs(results): + deferreds = [] + for i in xrange(number_of_docs): + content = binascii.hexlify(os.urandom(doc_size / 2)) + d = sol1.create_doc({'data': content}) + d.addCallback(created_ids.append) + deferreds.append(d) + return defer.DeferredList(deferreds) + + def _db1AssertDocsSyncedToServer(results): + self.assertEqual(number_of_docs, len(created_ids)) + for soldoc in created_ids: + couchdoc = db.get_doc(soldoc.doc_id) + self.assertTrue(couchdoc) + # assert document structure in couch server + self.assertEqual(soldoc.doc_id, couchdoc.doc_id) + self.assertEqual(soldoc.rev, couchdoc.rev) + couch_content = couchdoc.content.keys() + self.assertEqual(['raw'], couch_content) + content = couchdoc.get_json() + self.assertTrue(_crypto.is_symmetrically_encrypted(content)) + + d = sol1.get_all_docs() + d.addCallback(_db1AssertEmptyDocList) + d.addCallback(_db1CreateDocs) + d.addCallback(lambda _: sol1.sync()) + d.addCallback(_db1AssertDocsSyncedToServer) + + def _db2AssertEmptyDocList(results): + _, doclist = results + self.assertEqual([], doclist) + + def _getAllDocsFromBothDbs(results): + d1 = sol1.get_all_docs() + d2 = sol2.get_all_docs() + return defer.DeferredList([d1, d2]) + + d.addCallback(lambda _: sol2.get_all_docs()) + d.addCallback(_db2AssertEmptyDocList) + d.addCallback(lambda _: sol2.sync()) + d.addCallback(_getAllDocsFromBothDbs) + + def _assertDocSyncedFromDb1ToDb2(results): + r1, r2 = results + _, (gen1, doclist1) = r1 + _, (gen2, doclist2) = r2 + self.assertEqual(number_of_docs, gen1) + self.assertEqual(number_of_docs, gen2) + self.assertEqual(number_of_docs, len(doclist1)) + self.assertEqual(number_of_docs, len(doclist2)) + self.assertEqual(doclist1[0], doclist2[0]) + + d.addCallback(_assertDocSyncedFromDb1ToDb2) + + def _cleanUp(results): + db.delete_database() + db.close() + sol1.close() + sol2.close() + + d.addCallback(_cleanUp) + + return d + + def test_encrypted_sym_sync(self): + return self._test_encrypted_sym_sync() + + def test_encrypted_sym_sync_with_unicode_passphrase(self): + """ + Test the complete syncing chain between two soledad dbs using a + Soledad server backed by a couch database, using an unicode + passphrase. + """ + return self._test_encrypted_sym_sync(passphrase=u'ãáàäéàëíìïóòöõúùüñç') + + def test_sync_many_small_files(self): + """ + Test if Soledad can sync many smallfiles. + """ + return self._test_encrypted_sym_sync(doc_size=2, number_of_docs=100) diff --git a/tests/server/test_session.py b/tests/server/test_session.py new file mode 100644 index 00000000..3dbd2740 --- /dev/null +++ b/tests/server/test_session.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +# test_session.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for server session entrypoint. +""" +from twisted.trial import unittest + +from twisted.cred import portal +from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse +from twisted.cred.credentials import IUsernamePassword +from twisted.web.resource import getChildForRequest +from twisted.web.static import Data +from twisted.web.test.requesthelper import DummyRequest +from twisted.web.test.test_httpauth import b64encode +from twisted.web.test.test_httpauth import Realm +from twisted.web._auth.wrapper import UnauthorizedResource + +from leap.soledad.server.session import SoledadSession + + +class SoledadSessionTestCase(unittest.TestCase): + """ + Tests adapted from for + L{twisted.web.test.test_httpauth.HTTPAuthSessionWrapper}. + """ + + def makeRequest(self, *args, **kwargs): + request = DummyRequest(*args, **kwargs) + request.path = '/' + return request + + def setUp(self): + self.username = b'foo bar' + self.password = b'bar baz' + self.avatarContent = b"contents of the avatar resource itself" + self.childName = b"foo-child" + self.childContent = b"contents of the foo child of the avatar" + self.checker = InMemoryUsernamePasswordDatabaseDontUse() + self.checker.addUser(self.username, self.password) + self.avatar = Data(self.avatarContent, 'text/plain') + self.avatar.putChild( + self.childName, Data(self.childContent, 'text/plain')) + self.avatars = {self.username: self.avatar} + self.realm = Realm(self.avatars.get) + self.portal = portal.Portal(self.realm, [self.checker]) + self.wrapper = SoledadSession(self.portal) + + def _authorizedTokenLogin(self, request): + authorization = b64encode( + self.username + b':' + self.password) + request.requestHeaders.addRawHeader(b'authorization', + b'Token ' + authorization) + return getChildForRequest(self.wrapper, request) + + def test_getChildWithDefault(self): + request = self.makeRequest([self.childName]) + child = getChildForRequest(self.wrapper, request) + d = request.notifyFinish() + + def cbFinished(result): + self.assertEqual(request.responseCode, 401) + + d.addCallback(cbFinished) + request.render(child) + return d + + def _invalidAuthorizationTest(self, response): + request = self.makeRequest([self.childName]) + request.requestHeaders.addRawHeader(b'authorization', response) + child = getChildForRequest(self.wrapper, request) + d = request.notifyFinish() + + def cbFinished(result): + self.assertEqual(request.responseCode, 401) + + d.addCallback(cbFinished) + request.render(child) + return d + + def test_getChildWithDefaultUnauthorizedUser(self): + return self._invalidAuthorizationTest( + b'Basic ' + b64encode(b'foo:bar')) + + def test_getChildWithDefaultUnauthorizedPassword(self): + return self._invalidAuthorizationTest( + b'Basic ' + b64encode(self.username + b':bar')) + + def test_getChildWithDefaultUnrecognizedScheme(self): + return self._invalidAuthorizationTest(b'Quux foo bar baz') + + def test_getChildWithDefaultAuthorized(self): + request = self.makeRequest([self.childName]) + child = self._authorizedTokenLogin(request) + d = request.notifyFinish() + + def cbFinished(ignored): + self.assertEqual(request.written, [self.childContent]) + + d.addCallback(cbFinished) + request.render(child) + return d + + def test_renderAuthorized(self): + # Request it exactly, not any of its children. + request = self.makeRequest([]) + child = self._authorizedTokenLogin(request) + d = request.notifyFinish() + + def cbFinished(ignored): + self.assertEqual(request.written, [self.avatarContent]) + + d.addCallback(cbFinished) + request.render(child) + return d + + def test_decodeRaises(self): + request = self.makeRequest([self.childName]) + request.requestHeaders.addRawHeader(b'authorization', + b'Token decode should fail') + child = getChildForRequest(self.wrapper, request) + self.assertIsInstance(child, UnauthorizedResource) + + def test_parseResponse(self): + basicAuthorization = b'Basic abcdef123456' + self.assertEqual( + self.wrapper._parseHeader(basicAuthorization), + None) + tokenAuthorization = b'Token abcdef123456' + self.assertEqual( + self.wrapper._parseHeader(tokenAuthorization), + b'abcdef123456') + + def test_unexpectedDecodeError(self): + + class UnexpectedException(Exception): + pass + + class BadFactory(object): + scheme = b'bad' + + def getChallenge(self, client): + return {} + + def decode(self, response, request): + print("decode raised") + raise UnexpectedException() + + self.wrapper._credentialFactory = BadFactory() + request = self.makeRequest([self.childName]) + request.requestHeaders.addRawHeader(b'authorization', b'Bad abc') + child = getChildForRequest(self.wrapper, request) + request.render(child) + self.assertEqual(request.responseCode, 500) + errors = self.flushLoggedErrors(UnexpectedException) + self.assertEqual(len(errors), 1) + + def test_unexpectedLoginError(self): + class UnexpectedException(Exception): + pass + + class BrokenChecker(object): + credentialInterfaces = (IUsernamePassword,) + + def requestAvatarId(self, credentials): + raise UnexpectedException() + + self.portal.registerChecker(BrokenChecker()) + request = self.makeRequest([self.childName]) + child = self._authorizedTokenLogin(request) + request.render(child) + self.assertEqual(request.responseCode, 500) + self.assertEqual(len(self.flushLoggedErrors(UnexpectedException)), 1) + + def test_cantAccessOtherUserPathByDefault(self): + request = self.makeRequest([]) + # valid url_mapper path, but for another user + request.path = '/blobs/another-user/' + child = self._authorizedTokenLogin(request) + + request.render(child) + self.assertEqual(request.responseCode, 500) diff --git a/tests/server/test_shared_db.py b/tests/server/test_shared_db.py new file mode 100644 index 00000000..96af6dff --- /dev/null +++ b/tests/server/test_shared_db.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# test_shared_db.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the shared db on server side. +""" + + +import pytest + +from twisted.trial import unittest + +from leap.soledad.client.shared_db import SoledadSharedDatabase +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.l2db.errors import RevisionConflict + + +@pytest.mark.needs_couch +class SharedDbTests(unittest.TestCase): + """ + """ + + URL = 'http://127.0.0.1:2424/shared' + CREDS = {'token': {'uuid': 'an-uuid', 'token': 'an-auth-token'}} + + @pytest.fixture(autouse=True) + def soledad_client(self, soledad_server, soledad_dbs): + soledad_dbs('an-uuid') + self._db = SoledadSharedDatabase.open_database(self.URL, self.CREDS) + + @pytest.mark.thisone + def test_doc_update_succeeds(self): + doc_id = 'some-random-doc' + self.assertIsNone(self._db.get_doc(doc_id)) + # create a document in shared db + doc = SoledadDocument(doc_id=doc_id) + self._db.put_doc(doc) + # update that document + expected = {'new': 'content'} + doc.content = expected + self._db.put_doc(doc) + # ensure expected content was saved + doc = self._db.get_doc(doc_id) + self.assertEqual(expected, doc.content) + + @pytest.mark.thisone + def test_doc_update_fails_with_wrong_rev(self): + # create a document in shared db + doc_id = 'some-random-doc' + self.assertIsNone(self._db.get_doc(doc_id)) + # create a document in shared db + doc = SoledadDocument(doc_id=doc_id) + self._db.put_doc(doc) + # try to update document without including revision of old version + doc.rev = 'wrong-rev' + self.assertRaises(RevisionConflict, self._db.put_doc, doc) diff --git a/tests/server/test_soledad b/tests/server/test_soledad new file mode 120000 index 00000000..c1a35d32 --- /dev/null +++ b/tests/server/test_soledad @@ -0,0 +1 @@ +../test_soledad \ No newline at end of file diff --git a/tests/server/test_tac.py b/tests/server/test_tac.py new file mode 100644 index 00000000..7bb50e35 --- /dev/null +++ b/tests/server/test_tac.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# test_tac.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the localhost/public APIs using .tac file. +See docs/auth.rst +""" + + +import os +import signal +import socket +import pytest +import treq + +from pkg_resources import resource_filename +from twisted.trial import unittest +from twisted.internet import defer, reactor +from twisted.internet.protocol import ProcessProtocol +from twisted.web.client import Agent + + +TAC_FILE_PATH = resource_filename('leap.soledad.server', 'server.tac') + + +class TacServerTestCase(unittest.TestCase): + + def test_tac_file_exists(self): + msg = "server.tac used on this test case was expected to be at %s" + self.assertTrue(os.path.isfile(TAC_FILE_PATH), msg % TAC_FILE_PATH) + + @defer.inlineCallbacks + def test_local_public_default_ports_on_server_tac(self): + yield self._spawnServer() + result = yield self._get('http://localhost:2525/incoming') + fail_msg = "Localhost endpoint must require authentication!" + self.assertEquals(401, result.code, fail_msg) + + public_endpoint_url = 'http://%s:2424/' % self._get_public_ip() + result = yield self._get(public_endpoint_url) + self.assertEquals(200, result.code, "server info not accessible") + + result = yield self._get(public_endpoint_url + 'other') + self.assertEquals(401, result.code, "public server lacks auth!") + + public_using_local_port_url = 'http://%s:2525/' % self._get_public_ip() + with pytest.raises(Exception): + yield self._get(public_using_local_port_url) + + def _spawnServer(self): + protocol = ProcessProtocol() + env = os.environ.get('VIRTUAL_ENV', '/usr') + executable = os.path.join(env, 'bin', 'twistd') + no_pid_argument = '--pidfile=' + args = [executable, no_pid_argument, '-noy', TAC_FILE_PATH] + env = {'DEBUG_SERVER': 'yes'} + t = reactor.spawnProcess(protocol, executable, args, env=env) + self.addCleanup(os.kill, t.pid, signal.SIGKILL) + self.addCleanup(t.loseConnection) + return self._sleep(1) # it takes a while to start server + + def _sleep(self, time): + d = defer.Deferred() + reactor.callLater(time, d.callback, True) + return d + + def _get(self, *args, **kwargs): + kwargs['agent'] = Agent(reactor) + return treq.get(*args, **kwargs) + + def _get_public_ip(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + return s.getsockname()[0] diff --git a/tests/server/test_url_mapper.py b/tests/server/test_url_mapper.py new file mode 100644 index 00000000..a04e7593 --- /dev/null +++ b/tests/server/test_url_mapper.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +# test_url_mapper.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for server-related functionality. +""" +import pytest + +from twisted.trial import unittest +from uuid import uuid4 + +from leap.soledad.server.url_mapper import URLMapper + + +class URLMapperTestCase(unittest.TestCase): + """ + Test if the URLMapper behaves as expected. + + The following table lists the authorized actions among all possible + u1db remote actions: + + URL path | Authorized actions + -------------------------------------------------- + / | GET + /shared-db | GET + /shared-db/docs | - + /shared-db/doc/{id} | - + /shared-db/sync-from/{source} | - + /user-db | - + /user-db/docs | - + /user-db/doc/{id} | - + /user-db/sync-from/{source} | GET, PUT, POST + """ + + def setUp(self): + self._uuid = uuid4().hex + self._urlmap = URLMapper() + self._dbname = 'user-%s' % self._uuid + + @pytest.mark.needs_couch + def test_root_authorized(self): + match = self._urlmap.match('/', 'GET') + self.assertIsNotNone(match) + + def test_shared_authorized(self): + self.assertIsNotNone(self._urlmap.match('/shared', 'GET')) + + def test_shared_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared', 'POST')) + + def test_shared_docs_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/docs', 'GET')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared/docs', 'POST')) + + def test_shared_doc_authorized(self): + match = self._urlmap.match('/shared/doc/x', 'GET') + self.assertIsNotNone(match) + self.assertEqual('x', match.get('id')) + + match = self._urlmap.match('/shared/doc/x', 'PUT') + self.assertIsNotNone(match) + self.assertEqual('x', match.get('id')) + + match = self._urlmap.match('/shared/doc/x', 'DELETE') + self.assertIsNotNone(match) + self.assertEqual('x', match.get('id')) + + def test_shared_doc_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/doc/x', 'POST')) + + def test_shared_sync_unauthorized(self): + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'GET')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'PUT')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'DELETE')) + self.assertIsNone(self._urlmap.match('/shared/sync-from/x', 'POST')) + + def test_user_db_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s' % dbname, 'POST')) + + def test_user_db_docs_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s/docs' % dbname, 'POST')) + + def test_user_db_doc_unauthorized(self): + dbname = self._dbname + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'GET')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'PUT')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'DELETE')) + self.assertIsNone(self._urlmap.match('/%s/doc/x' % dbname, 'POST')) + + def test_user_db_sync_authorized(self): + uuid = self._uuid + dbname = self._dbname + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'GET') + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) + + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'PUT') + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) + + match = self._urlmap.match('/%s/sync-from/x' % dbname, 'POST') + self.assertEqual(uuid, match.get('uuid')) + self.assertEqual('x', match.get('source_replica_uid')) + + def test_user_db_sync_unauthorized(self): + dbname = self._dbname + self.assertIsNone( + self._urlmap.match('/%s/sync-from/x' % dbname, 'DELETE')) diff --git a/tests/sqlcipher/hacker_crackdown.txt b/tests/sqlcipher/hacker_crackdown.txt new file mode 100644 index 00000000..a01eb509 --- /dev/null +++ b/tests/sqlcipher/hacker_crackdown.txt @@ -0,0 +1,13005 @@ +The Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + +** This is a COPYRIGHTED Project Gutenberg eBook, Details Below ** +** Please follow the copyright guidelines in this file. ** + +Title: Hacker Crackdown + Law and Disorder on the Electronic Frontier + +Author: Bruce Sterling + +Posting Date: February 9, 2012 [EBook #101] +Release Date: January, 1994 + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** + + + + + + + + + + + + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + +CONTENTS + + +Preface to the Electronic Release of The Hacker Crackdown + +Chronology of the Hacker Crackdown + + +Introduction + + +Part 1: CRASHING THE SYSTEM +A Brief History of Telephony +Bell's Golden Vaporware +Universal Service +Wild Boys and Wire Women +The Electronic Communities +The Ungentle Giant +The Breakup +In Defense of the System +The Crash Post-Mortem +Landslides in Cyberspace + + +Part 2: THE DIGITAL UNDERGROUND +Steal This Phone +Phreaking and Hacking +The View From Under the Floorboards +Boards: Core of the Underground +Phile Phun +The Rake's Progress +Strongholds of the Elite +Sting Boards +Hot Potatoes +War on the Legion +Terminus +Phile 9-1-1 +War Games +Real Cyberpunk + + +Part 3: LAW AND ORDER +Crooked Boards +The World's Biggest Hacker Bust +Teach Them a Lesson +The U.S. Secret Service +The Secret Service Battles the Boodlers +A Walk Downtown +FCIC: The Cutting-Edge Mess +Cyberspace Rangers +FLETC: Training the Hacker-Trackers + + +Part 4: THE CIVIL LIBERTARIANS +NuPrometheus + FBI = Grateful Dead +Whole Earth + Computer Revolution = WELL +Phiber Runs Underground and Acid Spikes the Well +The Trial of Knight Lightning +Shadowhawk Plummets to Earth +Kyrie in the Confessional +$79,499 +A Scholar Investigates +Computers, Freedom, and Privacy + + +Electronic Afterword to The Hacker Crackdown, Halloween 1993 + + + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + + +Preface to the Electronic Release of The Hacker Crackdown + + +January 1, 1994--Austin, Texas + + +Hi, I'm Bruce Sterling, the author of this electronic book. + +Out in the traditional world of print, The Hacker Crackdown +is ISBN 0-553-08058-X, and is formally catalogued by +the Library of Congress as "1. Computer crimes--United States. +2. Telephone--United States--Corrupt practices. +3. Programming (Electronic computers)--United States--Corrupt practices." + +`Corrupt practices,' I always get a kick out of that description. +Librarians are very ingenious people. + +The paperback is ISBN 0-553-56370-X. If you go +and buy a print version of The Hacker Crackdown, +an action I encourage heartily, you may notice that +in the front of the book, beneath the copyright notice-- +"Copyright (C) 1992 by Bruce Sterling"-- +it has this little block of printed legal +boilerplate from the publisher. It says, and I quote: + + "No part of this book may be reproduced or transmitted in any form +or by any means, electronic or mechanical, including photocopying, +recording, or by any information storage and retrieval system, +without permission in writing from the publisher. +For information address: Bantam Books." + +This is a pretty good disclaimer, as such disclaimers go. +I collect intellectual-property disclaimers, and I've seen dozens of them, +and this one is at least pretty straightforward. In this narrow +and particular case, however, it isn't quite accurate. +Bantam Books puts that disclaimer on every book they publish, +but Bantam Books does not, in fact, own the electronic rights to this book. +I do, because of certain extensive contract maneuverings my agent and I +went through before this book was written. I want to give those electronic +publishing rights away through certain not-for-profit channels, +and I've convinced Bantam that this is a good idea. + +Since Bantam has seen fit to peacably agree to this scheme of mine, +Bantam Books is not going to fuss about this. Provided you don't try +to sell the book, they are not going to bother you for what you do with +the electronic copy of this book. If you want to check this out personally, +you can ask them; they're at 1540 Broadway NY NY 10036. However, if you were +so foolish as to print this book and start retailing it for money in violation +of my copyright and the commercial interests of Bantam Books, then Bantam, +a part of the gigantic Bertelsmann multinational publishing combine, +would roust some of their heavy-duty attorneys out of hibernation +and crush you like a bug. This is only to be expected. +I didn't write this book so that you could make money out of it. +If anybody is gonna make money out of this book, +it's gonna be me and my publisher. + +My publisher deserves to make money out of this book. +Not only did the folks at Bantam Books commission me +to write the book, and pay me a hefty sum to do so, but +they bravely printed, in text, an electronic document the +reproduction of which was once alleged to be a federal felony. +Bantam Books and their numerous attorneys were very brave +and forthright about this book. Furthermore, my former editor +at Bantam Books, Betsy Mitchell, genuinely cared about this project, +and worked hard on it, and had a lot of wise things to say +about the manuscript. Betsy deserves genuine credit for this book, +credit that editors too rarely get. + +The critics were very kind to The Hacker Crackdown, +and commercially the book has done well. On the other hand, +I didn't write this book in order to squeeze every last nickel +and dime out of the mitts of impoverished sixteen-year-old +cyberpunk high-school-students. Teenagers don't have any money-- +(no, not even enough for the six-dollar Hacker Crackdown paperback, +with its attractive bright-red cover and useful index). +That's a major reason why teenagers sometimes succumb to the temptation +to do things they shouldn't, such as swiping my books out of libraries. +Kids: this one is all yours, all right? Go give the print version back. +*8-) + +Well-meaning, public-spirited civil libertarians don't have much money, +either. And it seems almost criminal to snatch cash out of the hands of +America's direly underpaid electronic law enforcement community. + +If you're a computer cop, a hacker, or an electronic civil +liberties activist, you are the target audience for this book. +I wrote this book because I wanted to help you, and help other people +understand you and your unique, uhm, problems. I wrote this book +to aid your activities, and to contribute to the public discussion +of important political issues. In giving the text away in this +fashion, I am directly contributing to the book's ultimate aim: +to help civilize cyberspace. + +Information WANTS to be free. And the information inside +this book longs for freedom with a peculiar intensity. +I genuinely believe that the natural habitat of this book +is inside an electronic network. That may not be the easiest +direct method to generate revenue for the book's author, +but that doesn't matter; this is where this book belongs +by its nature. I've written other books--plenty of other books-- +and I'll write more and I am writing more, but this one is special. +I am making The Hacker Crackdown available electronically +as widely as I can conveniently manage, and if you like the book, +and think it is useful, then I urge you to do the same with it. + +You can copy this electronic book. Copy the heck out of it, +be my guest, and give those copies to anybody who wants them. +The nascent world of cyberspace is full of sysadmins, teachers, +trainers, cybrarians, netgurus, and various species of cybernetic activist. +If you're one of those people, I know about you, and I know the hassle +you go through to try to help people learn about the electronic frontier. +I hope that possessing this book in electronic form will lessen your troubles. +Granted, this treatment of our electronic social spectrum is not the ultimate +in academic rigor. And politically, it has something to offend +and trouble almost everyone. But hey, I'm told it's readable, +and at least the price is right. + +You can upload the book onto bulletin board systems, or Internet nodes, +or electronic discussion groups. Go right ahead and do that, I am giving +you express permission right now. Enjoy yourself. + +You can put the book on disks and give the disks away, +as long as you don't take any money for it. + +But this book is not public domain. You can't copyright it in +your own name. I own the copyright. Attempts to pirate this book +and make money from selling it may involve you in a serious litigative snarl. +Believe me, for the pittance you might wring out of such an action, +it's really not worth it. This book don't "belong" to you. +In an odd but very genuine way, I feel it doesn't "belong" to me, either. +It's a book about the people of cyberspace, and distributing it in this way +is the best way I know to actually make this information available, +freely and easily, to all the people of cyberspace--including people +far outside the borders of the United States, who otherwise may never +have a chance to see any edition of the book, and who may perhaps learn +something useful from this strange story of distant, obscure, but portentous +events in so-called "American cyberspace." + +This electronic book is now literary freeware. It now belongs to the +emergent realm of alternative information economics. You have no right +to make this electronic book part of the conventional flow of commerce. +Let it be part of the flow of knowledge: there's a difference. +I've divided the book into four sections, so that it is less ungainly +for upload and download; if there's a section of particular relevance +to you and your colleagues, feel free to reproduce that one and skip the rest. + +[Project Gutenberg has reassembled the file, with Sterling's permission.] + +Just make more when you need them, and give them to whoever might want them. + +Now have fun. + +Bruce Sterling--bruces@well.sf.ca.us + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + + + + +CHRONOLOGY OF THE HACKER CRACKDOWN + + +1865 U.S. Secret Service (USSS) founded. + +1876 Alexander Graham Bell invents telephone. + +1878 First teenage males flung off phone system by enraged authorities. + +1939 "Futurian" science-fiction group raided by Secret Service. + +1971 Yippie phone phreaks start YIPL/TAP magazine. + +1972 RAMPARTS magazine seized in blue-box rip-off scandal. + +1978 Ward Christenson and Randy Suess create first personal + computer bulletin board system. + +1982 William Gibson coins term "cyberspace." + +1982 "414 Gang" raided. + +1983-1983 AT&T dismantled in divestiture. + +1984 Congress passes Comprehensive Crime Control Act giving USSS + jurisdiction over credit card fraud and computer fraud. + +1984 "Legion of Doom" formed. + +1984. 2600: THE HACKER QUARTERLY founded. + +1984. WHOLE EARTH SOFTWARE CATALOG published. + +1985. First police "sting" bulletin board systems established. + +1985. Whole Earth 'Lectronic Link computer conference (WELL) goes on-line. + +1986 Computer Fraud and Abuse Act passed. + +1986 Electronic Communications Privacy Act passed. + +1987 Chicago prosecutors form Computer Fraud and Abuse Task Force. + + +1988 + +July. Secret Service covertly videotapes "SummerCon" hacker convention. + +September. "Prophet" cracks BellSouth AIMSX computer network + and downloads E911 Document to his own computer and to Jolnet. + +September. AT&T Corporate Information Security informed of Prophet's action. + +October. Bellcore Security informed of Prophet's action. + + +1989 + +January. Prophet uploads E911 Document to Knight Lightning. + +February 25. Knight Lightning publishes E911 Document in PHRACK + electronic newsletter. + +May. Chicago Task Force raids and arrests "Kyrie." + +June. "NuPrometheus League" distributes Apple Computer proprietary software. + +June 13. Florida probation office crossed with phone-sex line + in switching-station stunt. + +July. "Fry Guy" raided by USSS and Chicago Computer Fraud + and Abuse Task Force. + +July. Secret Service raids "Prophet," "Leftist," and "Urvile" in Georgia. + + +1990 + +January 15. Martin Luther King Day Crash strikes AT&T long-distance + network nationwide. + +January 18-19. Chicago Task Force raids Knight Lightning in St. Louis. + +January 24. USSS and New York State Police raid "Phiber Optik," + "Acid Phreak," and "Scorpion" in New York City. + +February 1. USSS raids "Terminus" in Maryland. + +February 3. Chicago Task Force raids Richard Andrews' home. + +February 6. Chicago Task Force raids Richard Andrews' business. + +February 6. USSS arrests Terminus, Prophet, Leftist, and Urvile. + +February 9. Chicago Task Force arrests Knight Lightning. + +February 20. AT&T Security shuts down public-access + "attctc" computer in Dallas. + +February 21. Chicago Task Force raids Robert Izenberg in Austin. + +March 1. Chicago Task Force raids Steve Jackson Games, Inc., + "Mentor," and "Erik Bloodaxe" in Austin. + +May 7,8,9. + +USSS and Arizona Organized Crime and Racketeering Bureau conduct +"Operation Sundevil" raids in Cincinnatti, Detroit, Los Angeles, +Miami, Newark, Phoenix, Pittsburgh, Richmond, Tucson, San Diego, +San Jose, and San Francisco. + +May. FBI interviews John Perry Barlow re NuPrometheus case. + +June. Mitch Kapor and Barlow found Electronic Frontier Foundation; + Barlow publishes CRIME AND PUZZLEMENT manifesto. + +July 24-27. Trial of Knight Lightning. + +1991 + +February. CPSR Roundtable in Washington, D.C. + +March 25-28. Computers, Freedom and Privacy conference in San Francisco. + +May 1. Electronic Frontier Foundation, Steve Jackson, + and others file suit against members of Chicago Task Force. + +July 1-2. Switching station phone software crash affects + Washington, Los Angeles, Pittsburgh, San Francisco. + +September 17. AT&T phone crash affects New York City and three airports. + + + + +Introduction + +This is a book about cops, and wild teenage whiz-kids, and lawyers, +and hairy-eyed anarchists, and industrial technicians, and hippies, +and high-tech millionaires, and game hobbyists, and computer security +experts, and Secret Service agents, and grifters, and thieves. + +This book is about the electronic frontier of the 1990s. +It concerns activities that take place inside computers +and over telephone lines. + +A science fiction writer coined the useful term "cyberspace" in 1982, +but the territory in question, the electronic frontier, is about +a hundred and thirty years old. Cyberspace is the "place" where +a telephone conversation appears to occur. Not inside your actual phone, +the plastic device on your desk. Not inside the other person's phone, +in some other city. THE PLACE BETWEEN the phones. The indefinite +place OUT THERE, where the two of you, two human beings, +actually meet and communicate. + +Although it is not exactly "real," "cyberspace" is a genuine place. +Things happen there that have very genuine consequences. This "place" +is not "real," but it is serious, it is earnest. Tens of thousands +of people have dedicated their lives to it, to the public service +of public communication by wire and electronics. + +People have worked on this "frontier" for generations now. +Some people became rich and famous from their efforts there. +Some just played in it, as hobbyists. Others soberly pondered it, +and wrote about it, and regulated it, and negotiated over it in +international forums, and sued one another about it, in gigantic, +epic court battles that lasted for years. And almost since +the beginning, some people have committed crimes in this place. + +But in the past twenty years, this electrical "space," +which was once thin and dark and one-dimensional--little more +than a narrow speaking-tube, stretching from phone to phone-- +has flung itself open like a gigantic jack-in-the-box. +Light has flooded upon it, the eerie light of the glowing computer screen. +This dark electric netherworld has become a vast flowering electronic landscape. +Since the 1960s, the world of the telephone has cross-bred itself +with computers and television, and though there is still no substance +to cyberspace, nothing you can handle, it has a strange kind +of physicality now. It makes good sense today to talk of cyberspace +as a place all its own. + +Because people live in it now. Not just a few people, +not just a few technicians and eccentrics, but thousands +of people, quite normal people. And not just for a little while, +either, but for hours straight, over weeks, and months, +and years. Cyberspace today is a "Net," a "Matrix," +international in scope and growing swiftly and steadily. +It's growing in size, and wealth, and political importance. + +People are making entire careers in modern cyberspace. +Scientists and technicians, of course; they've been there +for twenty years now. But increasingly, cyberspace +is filling with journalists and doctors and lawyers +and artists and clerks. Civil servants make their +careers there now, "on-line" in vast government data-banks; +and so do spies, industrial, political, and just plain snoops; +and so do police, at least a few of them. And there are children +living there now. + +People have met there and been married there. +There are entire living communities in cyberspace today; +chattering, gossiping, planning, conferring and scheming, +leaving one another voice-mail and electronic mail, +giving one another big weightless chunks of valuable data, +both legitimate and illegitimate. They busily pass one another +computer software and the occasional festering computer virus. + +We do not really understand how to live in cyberspace yet. +We are feeling our way into it, blundering about. +That is not surprising. Our lives in the physical world, +the "real" world, are also far from perfect, despite a lot more practice. +Human lives, real lives, are imperfect by their nature, and there are +human beings in cyberspace. The way we live in cyberspace is +a funhouse mirror of the way we live in the real world. +We take both our advantages and our troubles with us. + +This book is about trouble in cyberspace. +Specifically, this book is about certain strange events in +the year 1990, an unprecedented and startling year for the +the growing world of computerized communications. + +In 1990 there came a nationwide crackdown on illicit +computer hackers, with arrests, criminal charges, +one dramatic show-trial, several guilty pleas, and +huge confiscations of data and equipment all over the USA. + +The Hacker Crackdown of 1990 was larger, better organized, +more deliberate, and more resolute than any previous effort +in the brave new world of computer crime. The U.S. Secret Service, +private telephone security, and state and local law enforcement groups +across the country all joined forces in a determined attempt to break +the back of America's electronic underground. It was a fascinating +effort, with very mixed results. + +The Hacker Crackdown had another unprecedented effect; +it spurred the creation, within "the computer community," +of the Electronic Frontier Foundation, a new and very odd +interest group, fiercely dedicated to the establishment +and preservation of electronic civil liberties. The crackdown, +remarkable in itself, has created a melee of debate over electronic crime, +punishment, freedom of the press, and issues of search and seizure. +Politics has entered cyberspace. Where people go, politics follow. + +This is the story of the people of cyberspace. + + + +PART ONE: Crashing the System + +On January 15, 1990, AT&T's long-distance telephone switching system crashed. + +This was a strange, dire, huge event. Sixty thousand people lost +their telephone service completely. During the nine long hours +of frantic effort that it took to restore service, some seventy million +telephone calls went uncompleted. + +Losses of service, known as "outages" in the telco trade, +are a known and accepted hazard of the telephone business. +Hurricanes hit, and phone cables get snapped by the thousands. +Earthquakes wrench through buried fiber-optic lines. +Switching stations catch fire and burn to the ground. +These things do happen. There are contingency plans for them, +and decades of experience in dealing with them. +But the Crash of January 15 was unprecedented. +It was unbelievably huge, and it occurred for +no apparent physical reason. + +The crash started on a Monday afternoon in a single +switching-station in Manhattan. But, unlike any merely +physical damage, it spread and spread. Station after +station across America collapsed in a chain reaction, +until fully half of AT&T's network had gone haywire +and the remaining half was hard-put to handle the overflow. + +Within nine hours, AT&T software engineers more or less +understood what had caused the crash. Replicating the +problem exactly, poring over software line by line, +took them a couple of weeks. But because it was hard +to understand technically, the full truth of the matter +and its implications were not widely and thoroughly aired +and explained. The root cause of the crash remained obscure, +surrounded by rumor and fear. + +The crash was a grave corporate embarrassment. +The "culprit" was a bug in AT&T's own software--not the +sort of admission the telecommunications giant wanted +to make, especially in the face of increasing competition. +Still, the truth WAS told, in the baffling technical terms +necessary to explain it. + +Somehow the explanation failed to persuade +American law enforcement officials and even telephone +corporate security personnel. These people were not +technical experts or software wizards, and they had their +own suspicions about the cause of this disaster. + +The police and telco security had important sources +of information denied to mere software engineers. +They had informants in the computer underground and +years of experience in dealing with high-tech rascality +that seemed to grow ever more sophisticated. +For years they had been expecting a direct and +savage attack against the American national telephone system. +And with the Crash of January 15--the first month of a +new, high-tech decade--their predictions, fears, +and suspicions seemed at last to have entered the real world. +A world where the telephone system had not merely crashed, +but, quite likely, BEEN crashed--by "hackers." + +The crash created a large dark cloud of suspicion +that would color certain people's assumptions and actions +for months. The fact that it took place in the realm of +software was suspicious on its face. The fact that it +occurred on Martin Luther King Day, still the most +politically touchy of American holidays, made it more +suspicious yet. + +The Crash of January 15 gave the Hacker Crackdown +its sense of edge and its sweaty urgency. It made people, +powerful people in positions of public authority, +willing to believe the worst. And, most fatally, +it helped to give investigators a willingness +to take extreme measures and the determination +to preserve almost total secrecy. + +An obscure software fault in an aging switching system +in New York was to lead to a chain reaction of legal +and constitutional trouble all across the country. + +# + +Like the crash in the telephone system, this chain reaction +was ready and waiting to happen. During the 1980s, +the American legal system was extensively patched +to deal with the novel issues of computer crime. +There was, for instance, the Electronic Communications +Privacy Act of 1986 (eloquently described as "a stinking mess" +by a prominent law enforcement official). And there was the +draconian Computer Fraud and Abuse Act of 1986, passed unanimously +by the United States Senate, which later would reveal +a large number of flaws. Extensive, well-meant efforts +had been made to keep the legal system up to date. +But in the day-to-day grind of the real world, +even the most elegant software tends to crumble +and suddenly reveal its hidden bugs. + +Like the advancing telephone system, the American legal system +was certainly not ruined by its temporary crash; but for those +caught under the weight of the collapsing system, life became +a series of blackouts and anomalies. + +In order to understand why these weird events occurred, +both in the world of technology and in the world of law, +it's not enough to understand the merely technical problems. +We will get to those; but first and foremost, we must try +to understand the telephone, and the business of telephones, +and the community of human beings that telephones have created. + +# + +Technologies have life cycles, like cities do, +like institutions do, like laws and governments do. + +The first stage of any technology is the Question +Mark, often known as the "Golden Vaporware" stage. +At this early point, the technology is only a phantom, +a mere gleam in the inventor's eye. One such inventor +was a speech teacher and electrical tinkerer named +Alexander Graham Bell. + +Bell's early inventions, while ingenious, failed to move the world. +In 1863, the teenage Bell and his brother Melville made an artificial +talking mechanism out of wood, rubber, gutta-percha, and tin. +This weird device had a rubber-covered "tongue" made of movable +wooden segments, with vibrating rubber "vocal cords," and +rubber "lips" and "cheeks." While Melville puffed a bellows +into a tin tube, imitating the lungs, young Alec Bell would +manipulate the "lips," "teeth," and "tongue," causing the thing +to emit high-pitched falsetto gibberish. + +Another would-be technical breakthrough was the Bell "phonautograph" +of 1874, actually made out of a human cadaver's ear. Clamped into place +on a tripod, this grisly gadget drew sound-wave images on smoked glass +through a thin straw glued to its vibrating earbones. + +By 1875, Bell had learned to produce audible sounds--ugly shrieks +and squawks--by using magnets, diaphragms, and electrical current. + +Most "Golden Vaporware" technologies go nowhere. + +But the second stage of technology is the Rising Star, +or, the "Goofy Prototype," stage. The telephone, Bell's +most ambitious gadget yet, reached this stage on March +10, 1876. On that great day, Alexander Graham Bell +became the first person to transmit intelligible human +speech electrically. As it happened, young Professor Bell, +industriously tinkering in his Boston lab, had spattered +his trousers with acid. His assistant, Mr. Watson, +heard his cry for help--over Bell's experimental +audio-telegraph. This was an event without precedent. + +Technologies in their "Goofy Prototype" stage rarely +work very well. They're experimental, and therefore +half- baked and rather frazzled. The prototype may +be attractive and novel, and it does look as if it ought +to be good for something-or-other. But nobody, including +the inventor, is quite sure what. Inventors, and speculators, +and pundits may have very firm ideas about its potential +use, but those ideas are often very wrong. + +The natural habitat of the Goofy Prototype is in trade shows +and in the popular press. Infant technologies need publicity +and investment money like a tottering calf need milk. +This was very true of Bell's machine. To raise research and +development money, Bell toured with his device as a stage attraction. + +Contemporary press reports of the stage debut of the telephone +showed pleased astonishment mixed with considerable dread. +Bell's stage telephone was a large wooden box with a crude +speaker-nozzle, the whole contraption about the size and shape +of an overgrown Brownie camera. Its buzzing steel soundplate, +pumped up by powerful electromagnets, was loud enough to fill +an auditorium. Bell's assistant Mr. Watson, who could manage +on the keyboards fairly well, kicked in by playing the organ +from distant rooms, and, later, distant cities. This feat was +considered marvellous, but very eerie indeed. + +Bell's original notion for the telephone, an idea promoted +for a couple of years, was that it would become a mass medium. +We might recognize Bell's idea today as something close to modern +"cable radio." Telephones at a central source would transmit music, +Sunday sermons, and important public speeches to a paying network +of wired-up subscribers. + +At the time, most people thought this notion made good sense. +In fact, Bell's idea was workable. In Hungary, this philosophy +of the telephone was successfully put into everyday practice. +In Budapest, for decades, from 1893 until after World War I, +there was a government-run information service called +"Telefon Hirmondo-." Hirmondo- was a centralized source +of news and entertainment and culture, including stock reports, +plays, concerts, and novels read aloud. At certain hours +of the day, the phone would ring, you would plug in +a loudspeaker for the use of the family, and Telefon +Hirmondo- would be on the air--or rather, on the phone. + +Hirmondo- is dead tech today, but Hirmondo- might be considered +a spiritual ancestor of the modern telephone-accessed computer +data services, such as CompuServe, GEnie or Prodigy. +The principle behind Hirmondo- is also not too far from computer +"bulletin- board systems" or BBS's, which arrived in the late 1970s, +spread rapidly across America, and will figure largely in this book. + +We are used to using telephones for individual person-to-person speech, +because we are used to the Bell system. But this was just one possibility +among many. Communication networks are very flexible and protean, +especially when their hardware becomes sufficiently advanced. +They can be put to all kinds of uses. And they have been-- +and they will be. + +Bell's telephone was bound for glory, but this was a combination +of political decisions, canny infighting in court, inspired industrial +leadership, receptive local conditions and outright good luck. +Much the same is true of communications systems today. + +As Bell and his backers struggled to install their newfangled system +in the real world of nineteenth-century New England, they had to fight +against skepticism and industrial rivalry. There was already a strong +electrical communications network present in America: the telegraph. +The head of the Western Union telegraph system dismissed Bell's prototype +as "an electrical toy" and refused to buy the rights to Bell's patent. +The telephone, it seemed, might be all right as a parlor entertainment-- +but not for serious business. + +Telegrams, unlike mere telephones, left a permanent physical record +of their messages. Telegrams, unlike telephones, could be answered +whenever the recipient had time and convenience. And the telegram +had a much longer distance-range than Bell's early telephone. +These factors made telegraphy seem a much more sound and businesslike +technology--at least to some. + +The telegraph system was huge, and well-entrenched. +In 1876, the United States had 214,000 miles of telegraph wire, +and 8500 telegraph offices. There were specialized telegraphs +for businesses and stock traders, government, police and fire departments. +And Bell's "toy" was best known as a stage-magic musical device. + +The third stage of technology is known as the "Cash Cow" stage. +In the "cash cow" stage, a technology finds its place in the world, +and matures, and becomes settled and productive. After a year or so, +Alexander Graham Bell and his capitalist backers concluded that +eerie music piped from nineteenth-century cyberspace was not the real +selling-point of his invention. Instead, the telephone was about speech-- +individual, personal speech, the human voice, human conversation and +human interaction. The telephone was not to be managed from any centralized +broadcast center. It was to be a personal, intimate technology. + +When you picked up a telephone, you were not absorbing +the cold output of a machine--you were speaking to another human being. +Once people realized this, their instinctive dread of the telephone +as an eerie, unnatural device, swiftly vanished. A "telephone call" +was not a "call" from a "telephone" itself, but a call from another +human being, someone you would generally know and recognize. +The real point was not what the machine could do for you (or to you), +but what you yourself, a person and citizen, could do THROUGH the machine. +This decision on the part of the young Bell Company was absolutely vital. + +The first telephone networks went up around Boston--mostly among +the technically curious and the well-to-do (much the same segment +of the American populace that, a hundred years later, would be +buying personal computers). Entrenched backers of the telegraph +continued to scoff. + +But in January 1878, a disaster made the telephone famous. +A train crashed in Tarriffville, Connecticut. Forward-looking +doctors in the nearby city of Hartford had had Bell's +"speaking telephone" installed. An alert local druggist +was able to telephone an entire community of local doctors, +who rushed to the site to give aid. The disaster, as disasters do, +aroused intense press coverage. The phone had proven its usefulness +in the real world. + +After Tarriffville, the telephone network spread like crabgrass. +By 1890 it was all over New England. By '93, out to Chicago. +By '97, into Minnesota, Nebraska and Texas. By 1904 it was +all over the continent. + +The telephone had become a mature technology. Professor Bell +(now generally known as "Dr. Bell" despite his lack of a formal degree) +became quite wealthy. He lost interest in the tedious day-to-day business +muddle of the booming telephone network, and gratefully returned +his attention to creatively hacking-around in his various laboratories, +which were now much larger, better-ventilated, and gratifyingly +better-equipped. Bell was never to have another great inventive success, +though his speculations and prototypes anticipated fiber-optic transmission, +manned flight, sonar, hydrofoil ships, tetrahedral construction, and +Montessori education. The "decibel," the standard scientific measure +of sound intensity, was named after Bell. + +Not all Bell's vaporware notions were inspired. He was fascinated +by human eugenics. He also spent many years developing a weird personal +system of astrophysics in which gravity did not exist. + +Bell was a definite eccentric. He was something of a hypochondriac, +and throughout his life he habitually stayed up until four A.M., +refusing to rise before noon. But Bell had accomplished a great feat; +he was an idol of millions and his influence, wealth, and great +personal charm, combined with his eccentricity, made him something +of a loose cannon on deck. Bell maintained a thriving scientific +salon in his winter mansion in Washington, D.C., which gave him +considerable backstage influence in governmental and scientific circles. +He was a major financial backer of the the magazines Science and +National Geographic, both still flourishing today as important organs +of the American scientific establishment. + +Bell's companion Thomas Watson, similarly wealthy and similarly odd, +became the ardent political disciple of a 19th-century science-fiction writer +and would-be social reformer, Edward Bellamy. Watson also trod the boards +briefly as a Shakespearian actor. + +There would never be another Alexander Graham Bell, +but in years to come there would be surprising numbers +of people like him. Bell was a prototype of the +high-tech entrepreneur. High-tech entrepreneurs will +play a very prominent role in this book: not merely as +technicians and businessmen, but as pioneers of the +technical frontier, who can carry the power and prestige +they derive from high-technology into the political and +social arena. + +Like later entrepreneurs, Bell was fierce in defense of +his own technological territory. As the telephone began to +flourish, Bell was soon involved in violent lawsuits in the +defense of his patents. Bell's Boston lawyers were +excellent, however, and Bell himself, as an elocution +teacher and gifted public speaker, was a devastatingly +effective legal witness. In the eighteen years of Bell's patents, +the Bell company was involved in six hundred separate lawsuits. +The legal records printed filled 149 volumes. The Bell Company +won every single suit. + +After Bell's exclusive patents expired, rival telephone +companies sprang up all over America. Bell's company, +American Bell Telephone, was soon in deep trouble. +In 1907, American Bell Telephone fell into the hands of the +rather sinister J.P. Morgan financial cartel, robber-baron +speculators who dominated Wall Street. + +At this point, history might have taken a different turn. +American might well have been served forever by a patchwork +of locally owned telephone companies. Many state politicians +and local businessmen considered this an excellent solution. + +But the new Bell holding company, American Telephone and Telegraph +or AT&T, put in a new man at the helm, a visionary industrialist +named Theodore Vail. Vail, a former Post Office manager, +understood large organizations and had an innate feeling +for the nature of large-scale communications. Vail quickly +saw to it that AT&T seized the technological edge once again. +The Pupin and Campbell "loading coil," and the deForest +"audion," are both extinct technology today, but in 1913 +they gave Vail's company the best LONG-DISTANCE lines +ever built. By controlling long-distance--the links +between, and over, and above the smaller local phone +companies--AT&T swiftly gained the whip-hand over them, +and was soon devouring them right and left. + +Vail plowed the profits back into research and development, +starting the Bell tradition of huge-scale and brilliant +industrial research. + +Technically and financially, AT&T gradually steamrollered +the opposition. Independent telephone companies never +became entirely extinct, and hundreds of them flourish today. +But Vail's AT&T became the supreme communications company. +At one point, Vail's AT&T bought Western Union itself, +the very company that had derided Bell's telephone as a "toy." +Vail thoroughly reformed Western Union's hidebound business +along his modern principles; but when the federal government +grew anxious at this centralization of power, Vail politely +gave Western Union back. + +This centralizing process was not unique. Very similar +events had happened in American steel, oil, and railroads. +But AT&T, unlike the other companies, was to remain supreme. +The monopoly robber-barons of those other industries +were humbled and shattered by government trust-busting. + +Vail, the former Post Office official, was quite willing +to accommodate the US government; in fact he would +forge an active alliance with it. AT&T would become +almost a wing of the American government, almost +another Post Office--though not quite. AT&T would +willingly submit to federal regulation, but in return, +it would use the government's regulators as its own police, +who would keep out competitors and assure the Bell +system's profits and preeminence. + +This was the second birth--the political birth--of the +American telephone system. Vail's arrangement was to +persist, with vast success, for many decades, until 1982. +His system was an odd kind of American industrial socialism. +It was born at about the same time as Leninist Communism, +and it lasted almost as long--and, it must be admitted, +to considerably better effect. + +Vail's system worked. Except perhaps for aerospace, +there has been no technology more thoroughly dominated +by Americans than the telephone. The telephone was +seen from the beginning as a quintessentially American +technology. Bell's policy, and the policy of Theodore Vail, +was a profoundly democratic policy of UNIVERSAL ACCESS. +Vail's famous corporate slogan, "One Policy, One System, +Universal Service," was a political slogan, with a very +American ring to it. + +The American telephone was not to become the specialized tool +of government or business, but a general public utility. +At first, it was true, only the wealthy could afford +private telephones, and Bell's company pursued the +business markets primarily. The American phone system +was a capitalist effort, meant to make money; it was not a charity. +But from the first, almost all communities with telephone service +had public telephones. And many stores--especially drugstores-- +offered public use of their phones. You might not own a telephone-- +but you could always get into the system, if you really needed to. + +There was nothing inevitable about this decision to make telephones +"public" and "universal." Vail's system involved a profound act +of trust in the public. This decision was a political one, +informed by the basic values of the American republic. +The situation might have been very different; +and in other countries, under other systems, +it certainly was. + +Joseph Stalin, for instance, vetoed plans for a Soviet +phone system soon after the Bolshevik revolution. +Stalin was certain that publicly accessible telephones +would become instruments of anti-Soviet counterrevolution +and conspiracy. (He was probably right.) When telephones +did arrive in the Soviet Union, they would be instruments +of Party authority, and always heavily tapped. (Alexander +Solzhenitsyn's prison-camp novel The First Circle +describes efforts to develop a phone system more suited +to Stalinist purposes.) + +France, with its tradition of rational centralized government, +had fought bitterly even against the electric telegraph, +which seemed to the French entirely too anarchical and frivolous. +For decades, nineteenth-century France communicated via the +"visual telegraph," a nation-spanning, government-owned semaphore +system of huge stone towers that signalled from hilltops, +across vast distances, with big windmill-like arms. +In 1846, one Dr. Barbay, a semaphore enthusiast, +memorably uttered an early version of what might be called +"the security expert's argument" against the open media. + +"No, the electric telegraph is not a sound invention. +It will always be at the mercy of the slightest disruption, +wild youths, drunkards, bums, etc. . . . The electric telegraph +meets those destructive elements with only a few meters of wire +over which supervision is impossible. A single man could, +without being seen, cut the telegraph wires leading to Paris, +and in twenty-four hours cut in ten different places the wires +of the same line, without being arrested. The visual telegraph, +on the contrary, has its towers, its high walls, its gates +well-guarded from inside by strong armed men. Yes, I declare, +substitution of the electric telegraph for the visual one +is a dreadful measure, a truly idiotic act." + +Dr. Barbay and his high-security stone machines +were eventually unsuccessful, but his argument-- +that communication exists for the safety and convenience +of the state, and must be carefully protected from the wild +boys and the gutter rabble who might want to crash the +system--would be heard again and again. + +When the French telephone system finally did arrive, +its snarled inadequacy was to be notorious. Devotees +of the American Bell System often recommended a trip +to France, for skeptics. + +In Edwardian Britain, issues of class and privacy +were a ball-and-chain for telephonic progress. It was +considered outrageous that anyone--any wild fool off +the street--could simply barge bellowing into one's office +or home, preceded only by the ringing of a telephone bell. +In Britain, phones were tolerated for the use of business, +but private phones tended be stuffed away into closets, +smoking rooms, or servants' quarters. Telephone operators +were resented in Britain because they did not seem to +"know their place." And no one of breeding would print +a telephone number on a business card; this seemed a crass +attempt to make the acquaintance of strangers. + +But phone access in America was to become a popular right; +something like universal suffrage, only more so. +American women could not yet vote when the phone system +came through; yet from the beginning American women +doted on the telephone. This "feminization" of the +American telephone was often commented on by foreigners. +Phones in America were not censored or stiff or formalized; +they were social, private, intimate, and domestic. +In America, Mother's Day is by far the busiest day +of the year for the phone network. + +The early telephone companies, and especially AT&T, +were among the foremost employers of American women. +They employed the daughters of the American middle-class +in great armies: in 1891, eight thousand women; by 1946, +almost a quarter of a million. Women seemed to enjoy +telephone work; it was respectable, it was steady, +it paid fairly well as women's work went, and--not least-- +it seemed a genuine contribution to the social good +of the community. Women found Vail's ideal of public +service attractive. This was especially true in rural areas, +where women operators, running extensive rural party-lines, +enjoyed considerable social power. The operator knew everyone +on the party-line, and everyone knew her. + +Although Bell himself was an ardent suffragist, the +telephone company did not employ women for the sake of +advancing female liberation. AT&T did this for sound +commercial reasons. The first telephone operators of +the Bell system were not women, but teenage American boys. +They were telegraphic messenger boys (a group about to +be rendered technically obsolescent), who swept up +around the phone office, dunned customers for bills, +and made phone connections on the switchboard, +all on the cheap. + +Within the very first year of operation, 1878, +Bell's company learned a sharp lesson about combining +teenage boys and telephone switchboards. Putting +teenage boys in charge of the phone system brought swift +and consistent disaster. Bell's chief engineer described them +as "Wild Indians." The boys were openly rude to customers. +They talked back to subscribers, saucing off, +uttering facetious remarks, and generally giving lip. +The rascals took Saint Patrick's Day off without permission. +And worst of all they played clever tricks with +the switchboard plugs: disconnecting calls, crossing lines +so that customers found themselves talking to strangers, +and so forth. + +This combination of power, technical mastery, and effective +anonymity seemed to act like catnip on teenage boys. + +This wild-kid-on-the-wires phenomenon was not confined to +the USA; from the beginning, the same was true of the British +phone system. An early British commentator kindly remarked: +"No doubt boys in their teens found the work not a little irksome, +and it is also highly probable that under the early conditions +of employment the adventurous and inquisitive spirits of which +the average healthy boy of that age is possessed, were not always +conducive to the best attention being given to the wants +of the telephone subscribers." + +So the boys were flung off the system--or at least, +deprived of control of the switchboard. But the +"adventurous and inquisitive spirits" of the teenage boys +would be heard from in the world of telephony, again and again. + +The fourth stage in the technological life-cycle is death: +"the Dog," dead tech. The telephone has so far avoided this fate. +On the contrary, it is thriving, still spreading, still evolving, +and at increasing speed. + +The telephone has achieved a rare and exalted state for a +technological artifact: it has become a HOUSEHOLD OBJECT. +The telephone, like the clock, like pen and paper, +like kitchen utensils and running water, has become +a technology that is visible only by its absence. +The telephone is technologically transparent. +The global telephone system is the largest and most +complex machine in the world, yet it is easy to use. +More remarkable yet, the telephone is almost entirely +physically safe for the user. + +For the average citizen in the 1870s, the telephone +was weirder, more shocking, more "high-tech" and +harder to comprehend, than the most outrageous stunts +of advanced computing for us Americans in the 1990s. +In trying to understand what is happening to us today, +with our bulletin-board systems, direct overseas dialling, +fiber-optic transmissions, computer viruses, hacking stunts, +and a vivid tangle of new laws and new crimes, it is important +to realize that our society has been through a similar challenge before-- +and that, all in all, we did rather well by it. + +Bell's stage telephone seemed bizarre at first. But the +sensations of weirdness vanished quickly, once people began +to hear the familiar voices of relatives and friends, +in their own homes on their own telephones. The telephone +changed from a fearsome high-tech totem to an everyday pillar +of human community. + +This has also happened, and is still happening, +to computer networks. Computer networks such as +NSFnet, BITnet, USENET, JANET, are technically +advanced, intimidating, and much harder to use than +telephones. Even the popular, commercial computer +networks, such as GEnie, Prodigy, and CompuServe, +cause much head-scratching and have been described +as "user-hateful." Nevertheless they too are changing +from fancy high-tech items into everyday sources +of human community. + +The words "community" and "communication" have +the same root. Wherever you put a communications +network, you put a community as well. And whenever +you TAKE AWAY that network--confiscate it, outlaw it, +crash it, raise its price beyond affordability-- +then you hurt that community. + +Communities will fight to defend themselves. People will fight harder +and more bitterly to defend their communities, than they will fight +to defend their own individual selves. And this is very true +of the "electronic community" that arose around computer networks +in the 1980s--or rather, the VARIOUS electronic communities, +in telephony, law enforcement, computing, and the digital +underground that, by the year 1990, were raiding, rallying, +arresting, suing, jailing, fining and issuing angry manifestos. + +None of the events of 1990 were entirely new. +Nothing happened in 1990 that did not have some kind +of earlier and more understandable precedent. What gave +the Hacker Crackdown its new sense of gravity and +importance was the feeling--the COMMUNITY feeling-- +that the political stakes had been raised; that trouble +in cyberspace was no longer mere mischief or inconclusive +skirmishing, but a genuine fight over genuine issues, +a fight for community survival and the shape of the future. + +These electronic communities, having flourished throughout +the 1980s, were becoming aware of themselves, and increasingly, +becoming aware of other, rival communities. Worries were +sprouting up right and left, with complaints, rumors, +uneasy speculations. But it would take a catalyst, a shock, +to make the new world evident. Like Bell's great publicity break, +the Tarriffville Rail Disaster of January 1878, +it would take a cause celebre. + +That cause was the AT&T Crash of January 15, 1990. +After the Crash, the wounded and anxious telephone +community would come out fighting hard. + +# + +The community of telephone technicians, engineers, operators +and researchers is the oldest community in cyberspace. +These are the veterans, the most developed group, +the richest, the most respectable, in most ways the most powerful. +Whole generations have come and gone since Alexander Graham Bell's day, +but the community he founded survives; people work for the phone system +today whose great-grandparents worked for the phone system. +Its specialty magazines, such as Telephony, AT&T Technical Journal, +Telephone Engineer and Management, are decades old; +they make computer publications like Macworld and PC Week +look like amateur johnny-come-latelies. + +And the phone companies take no back seat in high-technology, either. +Other companies' industrial researchers may have won new markets; +but the researchers of Bell Labs have won SEVEN NOBEL PRIZES. +One potent device that Bell Labs originated, the transistor, +has created entire GROUPS of industries. Bell Labs are +world-famous for generating "a patent a day," and have even +made vital discoveries in astronomy, physics and cosmology. + +Throughout its seventy-year history, "Ma Bell" was not so much +a company as a way of life. Until the cataclysmic divestiture +of the 1980s, Ma Bell was perhaps the ultimate maternalist mega-employer. +The AT&T corporate image was the "gentle giant," "the voice with a smile," +a vaguely socialist-realist world of cleanshaven linemen in shiny helmets +and blandly pretty phone-girls in headsets and nylons. Bell System +employees were famous as rock-ribbed Kiwanis and Rotary members, +Little-League enthusiasts, school-board people. + +During the long heyday of Ma Bell, the Bell employee corps +were nurtured top-to-bottom on a corporate ethos of public service. +There was good money in Bell, but Bell was not ABOUT money; +Bell used public relations, but never mere marketeering. +People went into the Bell System for a good life, +and they had a good life. But it was not mere money +that led Bell people out in the midst of storms and earthquakes +to fight with toppled phone-poles, to wade in flooded manholes, +to pull the red-eyed graveyard-shift over collapsing switching-systems. +The Bell ethic was the electrical equivalent of the postman's: +neither rain, nor snow, nor gloom of night would stop these couriers. + +It is easy to be cynical about this, as it is easy to be +cynical about any political or social system; but cynicism +does not change the fact that thousands of people took +these ideals very seriously. And some still do. + +The Bell ethos was about public service; and that was +gratifying; but it was also about private POWER, and that +was gratifying too. As a corporation, Bell was very special. +Bell was privileged. Bell had snuggled up close to the state. +In fact, Bell was as close to government as you could get in +America and still make a whole lot of legitimate money. + +But unlike other companies, Bell was above and beyond +the vulgar commercial fray. Through its regional operating companies, +Bell was omnipresent, local, and intimate, all over America; +but the central ivory towers at its corporate heart were the +tallest and the ivoriest around. + +There were other phone companies in America, to be sure; +the so-called independents. Rural cooperatives, mostly; +small fry, mostly tolerated, sometimes warred upon. +For many decades, "independent" American phone companies +lived in fear and loathing of the official Bell monopoly +(or the "Bell Octopus," as Ma Bell's nineteenth-century +enemies described her in many angry newspaper manifestos). +Some few of these independent entrepreneurs, while legally +in the wrong, fought so bitterly against the Octopus +that their illegal phone networks were cast into the street +by Bell agents and publicly burned. + +The pure technical sweetness of the Bell System gave its operators, +inventors and engineers a deeply satisfying sense of power and mastery. +They had devoted their lives to improving this vast nation-spanning machine; +over years, whole human lives, they had watched it improve and grow. +It was like a great technological temple. They were an elite, +and they knew it--even if others did not; in fact, they felt +even more powerful BECAUSE others did not understand. + +The deep attraction of this sensation of elite technical power +should never be underestimated. "Technical power" is not for everybody; +for many people it simply has no charm at all. But for some people, +it becomes the core of their lives. For a few, it is overwhelming, +obsessive; it becomes something close to an addiction. People--especially +clever teenage boys whose lives are otherwise mostly powerless and put-upon +--love this sensation of secret power, and are willing to do all sorts +of amazing things to achieve it. The technical POWER of electronics +has motivated many strange acts detailed in this book, which would +otherwise be inexplicable. + +So Bell had power beyond mere capitalism. The Bell service ethos worked, +and was often propagandized, in a rather saccharine fashion. Over the decades, +people slowly grew tired of this. And then, openly impatient with it. +By the early 1980s, Ma Bell was to find herself with scarcely a real friend +in the world. Vail's industrial socialism had become hopelessly +out-of-fashion politically. Bell would be punished for that. +And that punishment would fall harshly upon the people of the +telephone community. + +# + +In 1983, Ma Bell was dismantled by federal court action. +The pieces of Bell are now separate corporate entities. +The core of the company became AT&T Communications, +and also AT&T Industries (formerly Western Electric, +Bell's manufacturing arm). AT&T Bell Labs became Bell +Communications Research, Bellcore. Then there are the +Regional Bell Operating Companies, or RBOCs, pronounced "arbocks." + +Bell was a titan and even these regional chunks are gigantic enterprises: +Fortune 50 companies with plenty of wealth and power behind them. +But the clean lines of "One Policy, One System, Universal Service" +have been shattered, apparently forever. + +The "One Policy" of the early Reagan Administration was to +shatter a system that smacked of noncompetitive socialism. +Since that time, there has been no real telephone "policy" +on the federal level. Despite the breakup, the remnants +of Bell have never been set free to compete in the open marketplace. + +The RBOCs are still very heavily regulated, but not from the top. +Instead, they struggle politically, economically and legally, +in what seems an endless turmoil, in a patchwork of overlapping federal +and state jurisdictions. Increasingly, like other major American corporations, +the RBOCs are becoming multinational, acquiring important commercial interests +in Europe, Latin America, and the Pacific Rim. But this, too, adds to their +legal and political predicament. + +The people of what used to be Ma Bell are not happy about their fate. +They feel ill-used. They might have been grudgingly willing to make +a full transition to the free market; to become just companies amid +other companies. But this never happened. Instead, AT&T and the RBOCS +("the Baby Bells") feel themselves wrenched from side to side by state +regulators, by Congress, by the FCC, and especially by the federal court +of Judge Harold Greene, the magistrate who ordered the Bell breakup +and who has been the de facto czar of American telecommunications +ever since 1983. + +Bell people feel that they exist in a kind of paralegal limbo today. +They don't understand what's demanded of them. If it's "service," +why aren't they treated like a public service? And if it's money, +then why aren't they free to compete for it? No one seems to know, +really. Those who claim to know keep changing their minds. +Nobody in authority seems willing to grasp the nettle for once and all. + +Telephone people from other countries are amazed by the +American telephone system today. Not that it works so well; +for nowadays even the French telephone system works, more or less. +They are amazed that the American telephone system STILL works +AT ALL, under these strange conditions. + +Bell's "One System" of long-distance service is now only about +eighty percent of a system, with the remainder held by Sprint, MCI, +and the midget long-distance companies. Ugly wars over dubious +corporate practices such as "slamming" (an underhanded method +of snitching clients from rivals) break out with some regularity +in the realm of long-distance service. The battle to break Bell's +long-distance monopoly was long and ugly, and since the breakup +the battlefield has not become much prettier. AT&T's famous +shame-and-blame advertisements, which emphasized the shoddy work +and purported ethical shadiness of their competitors, were much +remarked on for their studied psychological cruelty. + +There is much bad blood in this industry, and much +long-treasured resentment. AT&T's post-breakup +corporate logo, a striped sphere, is known in the +industry as the "Death Star" (a reference from the movie +Star Wars, in which the "Death Star" was the spherical +high- tech fortress of the harsh-breathing imperial ultra-baddie, +Darth Vader.) Even AT&T employees are less than thrilled +by the Death Star. A popular (though banned) T-shirt among +AT&T employees bears the old-fashioned Bell logo of the Bell System, +plus the newfangled striped sphere, with the before-and-after comments: +"This is your brain--This is your brain on drugs!" AT&T made a very +well-financed and determined effort to break into the personal +computer market; it was disastrous, and telco computer experts +are derisively known by their competitors as "the pole-climbers." +AT&T and the Baby Bell arbocks still seem to have few friends. + +Under conditions of sharp commercial competition, a crash like +that of January 15, 1990 was a major embarrassment to AT&T. +It was a direct blow against their much-treasured reputation +for reliability. Within days of the crash AT&T's +Chief Executive Officer, Bob Allen, officially apologized, +in terms of deeply pained humility: + +"AT&T had a major service disruption last Monday. +We didn't live up to our own standards of quality, +and we didn't live up to yours. It's as simple as that. +And that's not acceptable to us. Or to you. . . . +We understand how much people have come to depend +upon AT&T service, so our AT&T Bell Laboratories scientists +and our network engineers are doing everything possible +to guard against a recurrence. . . . We know there's no way +to make up for the inconvenience this problem may have caused you." + +Mr Allen's "open letter to customers" was printed in lavish ads +all over the country: in the Wall Street Journal, USA Today, +New York Times, Los Angeles Times, Chicago Tribune, +Philadelphia Inquirer, San Francisco Chronicle Examiner, +Boston Globe, Dallas Morning News, Detroit Free Press, +Washington Post, Houston Chronicle, Cleveland Plain Dealer, +Atlanta Journal Constitution, Minneapolis Star Tribune, +St. Paul Pioneer Press Dispatch, Seattle Times/Post Intelligencer, +Tacoma News Tribune, Miami Herald, Pittsburgh Press, +St. Louis Post Dispatch, Denver Post, Phoenix Republic Gazette +and Tampa Tribune. + +In another press release, AT&T went to some pains to suggest +that this "software glitch" might have happened just as easily to MCI, +although, in fact, it hadn't. (MCI's switching software was quite different +from AT&T's--though not necessarily any safer.) AT&T also announced +their plans to offer a rebate of service on Valentine's Day to make up +for the loss during the Crash. + +"Every technical resource available, including Bell Labs +scientists and engineers, has been devoted to assuring +it will not occur again," the public was told. They were +further assured that "The chances of a recurrence are small-- +a problem of this magnitude never occurred before." + +In the meantime, however, police and corporate +security maintained their own suspicions about +"the chances of recurrence" and the real reason why +a "problem of this magnitude" had appeared, seemingly +out of nowhere. Police and security knew for a fact +that hackers of unprecedented sophistication were illegally +entering, and reprogramming, certain digital switching stations. +Rumors of hidden "viruses" and secret "logic bombs" +in the switches ran rampant in the underground, +with much chortling over AT&T's predicament, +and idle speculation over what unsung hacker genius +was responsible for it. Some hackers, including police +informants, were trying hard to finger one another +as the true culprits of the Crash. + +Telco people found little comfort in objectivity when +they contemplated these possibilities. It was just too close +to the bone for them; it was embarrassing; it hurt so much, +it was hard even to talk about. + +There has always been thieving and misbehavior in the phone system. +There has always been trouble with the rival independents, +and in the local loops. But to have such trouble in the core +of the system, the long-distance switching stations, +is a horrifying affair. To telco people, this is +all the difference between finding roaches in your kitchen +and big horrid sewer-rats in your bedroom. + +From the outside, to the average citizen, the telcos +still seem gigantic and impersonal. The American public +seems to regard them as something akin to Soviet apparats. +Even when the telcos do their best corporate-citizen routine, +subsidizing magnet high-schools and sponsoring news-shows +on public television, they seem to win little except public suspicion. + +But from the inside, all this looks very different. +There's harsh competition. A legal and political system +that seems baffled and bored, when not actively hostile +to telco interests. There's a loss of morale, a deep sensation +of having somehow lost the upper hand. Technological change +has caused a loss of data and revenue to other, newer forms +of transmission. There's theft, and new forms of theft, +of growing scale and boldness and sophistication. +With all these factors, it was no surprise to see the telcos, +large and small, break out in a litany of bitter complaint. + +In late '88 and throughout 1989, telco representatives +grew shrill in their complaints to those few American law +enforcement officials who make it their business to try to +understand what telephone people are talking about. +Telco security officials had discovered the computer- +hacker underground, infiltrated it thoroughly, +and become deeply alarmed at its growing expertise. +Here they had found a target that was not only loathsome +on its face, but clearly ripe for counterattack. + +Those bitter rivals: AT&T, MCI and Sprint--and a crowd +of Baby Bells: PacBell, Bell South, Southwestern Bell, +NYNEX, USWest, as well as the Bell research consortium Bellcore, +and the independent long-distance carrier Mid-American-- +all were to have their role in the great hacker dragnet of 1990. +After years of being battered and pushed around, the telcos had, +at least in a small way, seized the initiative again. +After years of turmoil, telcos and government officials were +once again to work smoothly in concert in defense of the System. +Optimism blossomed; enthusiasm grew on all sides; +the prospective taste of vengeance was sweet. + +# + +From the beginning--even before the crackdown had a name-- +secrecy was a big problem. There were many good reasons +for secrecy in the hacker crackdown. Hackers and code-thieves +were wily prey, slinking back to their bedrooms and basements +and destroying vital incriminating evidence at the first hint of trouble. +Furthermore, the crimes themselves were heavily technical and difficult +to describe, even to police--much less to the general public. + +When such crimes HAD been described intelligibly to the public, +in the past, that very publicity had tended to INCREASE the crimes +enormously. Telco officials, while painfully aware of the vulnerabilities +of their systems, were anxious not to publicize those weaknesses. +Experience showed them that those weaknesses, once discovered, +would be pitilessly exploited by tens of thousands of people--not only +by professional grifters and by underground hackers and phone phreaks, +but by many otherwise more-or-less honest everyday folks, who regarded +stealing service from the faceless, soulless "Phone Company" as a kind of +harmless indoor sport. When it came to protecting their interests, +telcos had long since given up on general public sympathy for +"the Voice with a Smile." Nowadays the telco's "Voice" was +very likely to be a computer's; and the American public +showed much less of the proper respect and gratitude due +the fine public service bequeathed them by Dr. Bell and Mr. Vail. +The more efficient, high-tech, computerized, and impersonal +the telcos became, it seemed, the more they were met by +sullen public resentment and amoral greed. + +Telco officials wanted to punish the phone-phreak underground, in as +public and exemplary a manner as possible. They wanted to make dire +examples of the worst offenders, to seize the ringleaders and intimidate +the small fry, to discourage and frighten the wacky hobbyists, and send +the professional grifters to jail. To do all this, publicity was vital. + +Yet operational secrecy was even more so. If word got out that +a nationwide crackdown was coming, the hackers might simply vanish; +destroy the evidence, hide their computers, go to earth, +and wait for the campaign to blow over. Even the young +hackers were crafty and suspicious, and as for the professional grifters, +they tended to split for the nearest state-line at the first sign of trouble. +For the crackdown to work well, they would all have to be caught red-handed, +swept upon suddenly, out of the blue, from every corner of the compass. + +And there was another strong motive for secrecy. In the worst-case scenario, +a blown campaign might leave the telcos open to a devastating hacker +counter-attack. If there were indeed hackers loose in America who +had caused the January 15 Crash--if there were truly gifted hackers, +loose in the nation's long-distance switching systems, and enraged +or frightened by the crackdown--then they might react unpredictably +to an attempt to collar them. Even if caught, they might have talented +and vengeful friends still running around loose. Conceivably, +it could turn ugly. Very ugly. In fact, it was hard to imagine +just how ugly things might turn, given that possibility. + +Counter-attack from hackers was a genuine concern for the telcos. +In point of fact, they would never suffer any such counter-attack. +But in months to come, they would be at some pains to publicize +this notion and to utter grim warnings about it. + +Still, that risk seemed well worth running. Better to run the risk +of vengeful attacks, than to live at the mercy of potential crashers. +Any cop would tell you that a protection racket had no real future. + +And publicity was such a useful thing. Corporate security officers, +including telco security, generally work under conditions of great discretion. +And corporate security officials do not make money for their companies. +Their job is to PREVENT THE LOSS of money, which is much less glamorous +than actually winning profits. + +If you are a corporate security official, and you do your job brilliantly, +then nothing bad happens to your company at all. Because of this, you appear +completely superfluous. This is one of the many unattractive aspects +of security work. It's rare that these folks have the chance to draw +some healthy attention to their own efforts. + +Publicity also served the interest of their friends in law enforcement. +Public officials, including law enforcement officials, thrive by attracting +favorable public interest. A brilliant prosecution in a matter of vital +public interest can make the career of a prosecuting attorney. +And for a police officer, good publicity opens the purses of the legislature; +it may bring a citation, or a promotion, or at least a rise in status +and the respect of one's peers. + +But to have both publicity and secrecy is to have one's cake and eat it too. +In months to come, as we will show, this impossible act was to cause great +pain to the agents of the crackdown. But early on, it seemed possible +--maybe even likely--that the crackdown could successfully combine +the best of both worlds. The ARREST of hackers would be heavily publicized. +The actual DEEDS of the hackers, which were technically hard to explain +and also a security risk, would be left decently obscured. The THREAT +hackers posed would be heavily trumpeted; the likelihood of their actually +committing such fearsome crimes would be left to the public's imagination. +The spread of the computer underground, and its growing technical +sophistication, would be heavily promoted; the actual hackers themselves, +mostly bespectacled middle-class white suburban teenagers, +would be denied any personal publicity. + +It does not seem to have occurred to any telco official +that the hackers accused would demand a day in court; +that journalists would smile upon the hackers as +"good copy;" that wealthy high-tech entrepreneurs would +offer moral and financial support to crackdown victims; +that constitutional lawyers would show up with briefcases, +frowning mightily. This possibility does not seem to have +ever entered the game-plan. + +And even if it had, it probably would not have slowed +the ferocious pursuit of a stolen phone-company document, +mellifluously known as "Control Office Administration of +Enhanced 911 Services for Special Services and Major Account Centers." + +In the chapters to follow, we will explore the worlds +of police and the computer underground, and the large +shadowy area where they overlap. But first, we must +explore the battleground. Before we leave the world +of the telcos, we must understand what a switching system +actually is and how your telephone actually works. + +# + +To the average citizen, the idea of the telephone is represented by, +well, a TELEPHONE: a device that you talk into. To a telco +professional, however, the telephone itself is known, in lordly +fashion, as a "subset." The "subset" in your house is a mere adjunct, +a distant nerve ending, of the central switching stations, +which are ranked in levels of heirarchy, up to the long-distance electronic +switching stations, which are some of the largest computers on earth. + +Let us imagine that it is, say, 1925, before the +introduction of computers, when the phone system was +simpler and somewhat easier to grasp. Let's further +imagine that you are Miss Leticia Luthor, a fictional +operator for Ma Bell in New York City of the 20s. + +Basically, you, Miss Luthor, ARE the "switching system." +You are sitting in front of a large vertical switchboard, +known as a "cordboard," made of shiny wooden panels, +with ten thousand metal-rimmed holes punched in them, +known as jacks. The engineers would have put more +holes into your switchboard, but ten thousand is +as many as you can reach without actually having +to get up out of your chair. + +Each of these ten thousand holes has its own little electric lightbulb, +known as a "lamp," and its own neatly printed number code. + +With the ease of long habit, you are scanning your board for lit-up bulbs. +This is what you do most of the time, so you are used to it. + +A lamp lights up. This means that the phone +at the end of that line has been taken off the hook. +Whenever a handset is taken off the hook, that closes a circuit +inside the phone which then signals the local office, i.e. you, +automatically. There might be somebody calling, or then +again the phone might be simply off the hook, but this +does not matter to you yet. The first thing you do, +is record that number in your logbook, in your fine American +public-school handwriting. This comes first, naturally, +since it is done for billing purposes. + +You now take the plug of your answering cord, which goes +directly to your headset, and plug it into the lit-up hole. +"Operator," you announce. + +In operator's classes, before taking this job, you have +been issued a large pamphlet full of canned operator's +responses for all kinds of contingencies, which you had +to memorize. You have also been trained in a proper +non-regional, non-ethnic pronunciation and tone of voice. +You rarely have the occasion to make any spontaneous +remark to a customer, and in fact this is frowned upon +(except out on the rural lines where people have time +on their hands and get up to all kinds of mischief). + +A tough-sounding user's voice at the end of the line +gives you a number. Immediately, you write that number +down in your logbook, next to the caller's number, +which you just wrote earlier. You then look and see if +the number this guy wants is in fact on your switchboard, +which it generally is, since it's generally a local call. +Long distance costs so much that people use it sparingly. + +Only then do you pick up a calling-cord from a shelf +at the base of the switchboard. This is a long elastic cord +mounted on a kind of reel so that it will zip back in when +you unplug it. There are a lot of cords down there, +and when a bunch of them are out at once they look like +a nest of snakes. Some of the girls think there are bugs +living in those cable-holes. They're called "cable mites" +and are supposed to bite your hands and give you rashes. +You don't believe this, yourself. + +Gripping the head of your calling-cord, you slip the tip +of it deftly into the sleeve of the jack for the called person. +Not all the way in, though. You just touch it. If you hear +a clicking sound, that means the line is busy and you can't +put the call through. If the line is busy, you have to stick +the calling-cord into a "busy-tone jack," which will give +the guy a busy-tone. This way you don't have to talk to him +yourself and absorb his natural human frustration. + +But the line isn't busy. So you pop the cord all the way in. +Relay circuits in your board make the distant phone ring, +and if somebody picks it up off the hook, then a phone +conversation starts. You can hear this conversation +on your answering cord, until you unplug it. In fact +you could listen to the whole conversation if you wanted, +but this is sternly frowned upon by management, and frankly, +when you've overheard one, you've pretty much heard 'em all. + +You can tell how long the conversation lasts by the glow +of the calling-cord's lamp, down on the calling-cord's shelf. +When it's over, you unplug and the calling-cord zips back into place. + +Having done this stuff a few hundred thousand times, +you become quite good at it. In fact you're plugging, +and connecting, and disconnecting, ten, twenty, forty cords +at a time. It's a manual handicraft, really, quite satisfying +in a way, rather like weaving on an upright loom. + +Should a long-distance call come up, it would be different, +but not all that different. Instead of connecting the call +through your own local switchboard, you have to go up the hierarchy, +onto the long-distance lines, known as "trunklines." +Depending on how far the call goes, it may have to work +its way through a whole series of operators, which can +take quite a while. The caller doesn't wait on the line +while this complex process is negotiated across the country +by the gaggle of operators. Instead, the caller hangs up, +and you call him back yourself when the call has finally +worked its way through. + +After four or five years of this work, you get married, +and you have to quit your job, this being the natural order +of womanhood in the American 1920s. The phone company +has to train somebody else--maybe two people, since +the phone system has grown somewhat in the meantime. +And this costs money. + +In fact, to use any kind of human being as a switching +system is a very expensive proposition. Eight thousand +Leticia Luthors would be bad enough, but a quarter of a +million of them is a military-scale proposition and makes +drastic measures in automation financially worthwhile. + +Although the phone system continues to grow today, +the number of human beings employed by telcos has +been dropping steadily for years. Phone "operators" +now deal with nothing but unusual contingencies, +all routine operations having been shrugged off onto machines. +Consequently, telephone operators are considerably less +machine-like nowadays, and have been known to have accents +and actual character in their voices. When you reach +a human operator today, the operators are rather more +"human" than they were in Leticia's day--but on the other hand, +human beings in the phone system are much harder to reach +in the first place. + +Over the first half of the twentieth century, +"electromechanical" switching systems of growing +complexity were cautiously introduced into the phone system. +In certain backwaters, some of these hybrid systems are still +in use. But after 1965, the phone system began to go completely +electronic, and this is by far the dominant mode today. +Electromechanical systems have "crossbars," and "brushes," +and other large moving mechanical parts, which, while faster +and cheaper than Leticia, are still slow, and tend to wear out +fairly quickly. + +But fully electronic systems are inscribed on silicon chips, +and are lightning-fast, very cheap, and quite durable. +They are much cheaper to maintain than even the best +electromechanical systems, and they fit into half the space. +And with every year, the silicon chip grows smaller, faster, +and cheaper yet. Best of all, automated electronics work +around the clock and don't have salaries or health insurance. + +There are, however, quite serious drawbacks to the +use of computer-chips. When they do break down, it is +a daunting challenge to figure out what the heck has gone +wrong with them. A broken cordboard generally had +a problem in it big enough to see. A broken chip has +invisible, microscopic faults. And the faults in bad +software can be so subtle as to be practically theological. + +If you want a mechanical system to do something new, +then you must travel to where it is, and pull pieces out of it, +and wire in new pieces. This costs money. However, if you want +a chip to do something new, all you have to do is change its software, +which is easy, fast and dirt-cheap. You don't even have to see the chip +to change its program. Even if you did see the chip, it wouldn't look +like much. A chip with program X doesn't look one whit different from +a chip with program Y. + +With the proper codes and sequences, and access to specialized phone-lines, +you can change electronic switching systems all over America from anywhere +you please. + +And so can other people. If they know how, and if they want to, +they can sneak into a microchip via the special phonelines and diddle with it, +leaving no physical trace at all. If they broke into the operator's station +and held Leticia at gunpoint, that would be very obvious. If they broke into +a telco building and went after an electromechanical switch with a toolbelt, +that would at least leave many traces. But people can do all manner of amazing +things to computer switches just by typing on a keyboard, and keyboards are +everywhere today. The extent of this vulnerability is deep, dark, broad, +almost mind-boggling, and yet this is a basic, primal fact of life about +any computer on a network. + +Security experts over the past twenty years have insisted, +with growing urgency, that this basic vulnerability of computers +represents an entirely new level of risk, of unknown but obviously +dire potential to society. And they are right. + +An electronic switching station does pretty much +everything Letitia did, except in nanoseconds and +on a much larger scale. Compared to Miss Luthor's +ten thousand jacks, even a primitive 1ESS switching computer, +60s vintage, has a 128,000 lines. And the current AT&T +system of choice is the monstrous fifth-generation 5ESS. + +An Electronic Switching Station can scan every line on its "board" +in a tenth of a second, and it does this over and over, tirelessly, +around the clock. Instead of eyes, it uses "ferrod scanners" +to check the condition of local lines and trunks. Instead of hands, +it has "signal distributors," "central pulse distributors," +"magnetic latching relays," and "reed switches," which complete +and break the calls. Instead of a brain, it has a "central processor." +Instead of an instruction manual, it has a program. Instead of +a handwritten logbook for recording and billing calls, +it has magnetic tapes. And it never has to talk to anybody. +Everything a customer might say to it is done by punching +the direct-dial tone buttons on your subset. + +Although an Electronic Switching Station can't talk, +it does need an interface, some way to relate to its, er, +employers. This interface is known as the "master control +center." (This interface might be better known simply as +"the interface," since it doesn't actually "control" phone +calls directly. However, a term like "Master Control +Center" is just the kind of rhetoric that telco maintenance +engineers--and hackers--find particularly satisfying.) + +Using the master control center, a phone engineer can test +local and trunk lines for malfunctions. He (rarely she) +can check various alarm displays, measure traffic on the lines, +examine the records of telephone usage and the charges for those calls, +and change the programming. + +And, of course, anybody else who gets into the master control center +by remote control can also do these things, if he (rarely she) +has managed to figure them out, or, more likely, has somehow swiped +the knowledge from people who already know. + +In 1989 and 1990, one particular RBOC, BellSouth, +which felt particularly troubled, spent a purported $1.2 +million on computer security. Some think it spent as +much as two million, if you count all the associated costs. +Two million dollars is still very little compared to the +great cost-saving utility of telephonic computer systems. + +Unfortunately, computers are also stupid. +Unlike human beings, computers possess the truly +profound stupidity of the inanimate. + +In the 1960s, in the first shocks of spreading computerization, +there was much easy talk about the stupidity of computers-- +how they could "only follow the program" and were rigidly required +to do "only what they were told." There has been rather less talk +about the stupidity of computers since they began to achieve +grandmaster status in chess tournaments, and to manifest +many other impressive forms of apparent cleverness. + +Nevertheless, computers STILL are profoundly brittle and stupid; +they are simply vastly more subtle in their stupidity and brittleness. +The computers of the 1990s are much more reliable in their components +than earlier computer systems, but they are also called upon to do +far more complex things, under far more challenging conditions. + +On a basic mathematical level, every single line of +a software program offers a chance for some possible screwup. +Software does not sit still when it works; it "runs," +it interacts with itself and with its own inputs and outputs. +By analogy, it stretches like putty into millions of possible +shapes and conditions, so many shapes that they can never +all be successfully tested, not even in the lifespan of the universe. +Sometimes the putty snaps. + +The stuff we call "software" is not like anything that human society +is used to thinking about. Software is something like a machine, +and something like mathematics, and something like language, and +something like thought, and art, and information. . . . But software +is not in fact any of those other things. The protean quality +of software is one of the great sources of its fascination. +It also makes software very powerful, very subtle, +very unpredictable, and very risky. + +Some software is bad and buggy. Some is "robust," +even "bulletproof." The best software is that which has +been tested by thousands of users under thousands of +different conditions, over years. It is then known as +"stable." This does NOT mean that the software is +now flawless, free of bugs. It generally means that there +are plenty of bugs in it, but the bugs are well-identified +and fairly well understood. + +There is simply no way to assure that software is free +of flaws. Though software is mathematical in nature, +it cannot by "proven" like a mathematical theorem; +software is more like language, with inherent ambiguities, +with different definitions, different assumptions, +different levels of meaning that can conflict. + +Human beings can manage, more or less, with +human language because we can catch the gist of it. + +Computers, despite years of effort in "artificial intelligence," +have proven spectacularly bad in "catching the gist" of anything at all. +The tiniest bit of semantic grit may still bring the mightiest computer +tumbling down. One of the most hazardous things you can do to a +computer program is try to improve it--to try to make it safer. +Software "patches" represent new, untried un-"stable" software, +which is by definition riskier. + +The modern telephone system has come to depend, +utterly and irretrievably, upon software. And the +System Crash of January 15, 1990, was caused by an +IMPROVEMENT in software. Or rather, an ATTEMPTED +improvement. + +As it happened, the problem itself--the problem per se--took this form. +A piece of telco software had been written in C language, a standard +language of the telco field. Within the C software was a +long "do. . .while" construct. The "do. . .while" construct +contained a "switch" statement. The "switch" statement contained +an "if" clause. The "if" clause contained a "break." The "break" +was SUPPOSED to "break" the "if clause." Instead, the "break" +broke the "switch" statement. + +That was the problem, the actual reason why people picking up phones +on January 15, 1990, could not talk to one another. + +Or at least, that was the subtle, abstract, cyberspatial +seed of the problem. This is how the problem manifested itself +from the realm of programming into the realm of real life. + +The System 7 software for AT&T's 4ESS switching station, +the "Generic 44E14 Central Office Switch Software," +had been extensively tested, and was considered very stable. +By the end of 1989, eighty of AT&T's switching systems +nationwide had been programmed with the new software. Cautiously, +thirty-four stations were left to run the slower, less-capable +System 6, because AT&T suspected there might be shakedown problems +with the new and unprecedently sophisticated System 7 network. + +The stations with System 7 were programmed to switch over to a backup net +in case of any problems. In mid-December 1989, however, a new high-velocity, +high-security software patch was distributed to each of the 4ESS switches +that would enable them to switch over even more quickly, making the System 7 +network that much more secure. + +Unfortunately, every one of these 4ESS switches was now in possession +of a small but deadly flaw. + +In order to maintain the network, switches must monitor +the condition of other switches--whether they are up and running, +whether they have temporarily shut down, whether they are overloaded +and in need of assistance, and so forth. The new software helped +control this bookkeeping function by monitoring the status calls +from other switches. + +It only takes four to six seconds for a troubled 4ESS switch +to rid itself of all its calls, drop everything temporarily, +and re-boot its software from scratch. Starting over from scratch +will generally rid the switch of any software problems that may have +developed in the course of running the system. Bugs that arise will +be simply wiped out by this process. It is a clever idea. This process +of automatically re-booting from scratch is known as the "normal fault +recovery routine." Since AT&T's software is in fact exceptionally stable, +systems rarely have to go into "fault recovery" in the first place; +but AT&T has always boasted of its "real world" reliability, and this +tactic is a belt-and-suspenders routine. + +The 4ESS switch used its new software to monitor its fellow switches +as they recovered from faults. As other switches came back on line +after recovery, they would send their "OK" signals to the switch. +The switch would make a little note to that effect in its "status map," +recognizing that the fellow switch was back and ready to go, +and should be sent some calls and put back to regular work. + +Unfortunately, while it was busy bookkeeping with the status map, +the tiny flaw in the brand-new software came into play. +The flaw caused the 4ESS switch to interact, subtly but drastically, +with incoming telephone calls from human users. If--and only if-- +two incoming phone-calls happened to hit the switch within a hundredth +of a second, then a small patch of data would be garbled by the flaw. + +But the switch had been programmed to monitor itself +constantly for any possible damage to its data. +When the switch perceived that its data had been somehow garbled, +then it too would go down, for swift repairs to its software. +It would signal its fellow switches not to send any more work. +It would go into the fault-recovery mode for four to six seconds. +And then the switch would be fine again, and would send out its "OK, +ready for work" signal. + +However, the "OK, ready for work" signal was the VERY THING THAT +HAD CAUSED THE SWITCH TO GO DOWN IN THE FIRST PLACE. And ALL the +System 7 switches had the same flaw in their status-map software. +As soon as they stopped to make the bookkeeping note that their fellow +switch was "OK," then they too would become vulnerable to the slight +chance that two phone-calls would hit them within a hundredth of a second. + +At approximately 2:25 P.M. EST on Monday, January 15, +one of AT&T's 4ESS toll switching systems in New York City +had an actual, legitimate, minor problem. It went into fault +recovery routines, announced "I'm going down," then announced, +"I'm back, I'm OK." And this cheery message then blasted +throughout the network to many of its fellow 4ESS switches. + +Many of the switches, at first, completely escaped trouble. +These lucky switches were not hit by the coincidence of +two phone calls within a hundredth of a second. +Their software did not fail--at first. But three switches-- +in Atlanta, St. Louis, and Detroit--were unlucky, +and were caught with their hands full. And they went down. +And they came back up, almost immediately. And they too began +to broadcast the lethal message that they, too, were "OK" again, +activating the lurking software bug in yet other switches. + +As more and more switches did have that bit of bad luck +and collapsed, the call-traffic became more and more densely +packed in the remaining switches, which were groaning +to keep up with the load. And of course, as the calls +became more densely packed, the switches were MUCH MORE LIKELY +to be hit twice within a hundredth of a second. + +It only took four seconds for a switch to get well. +There was no PHYSICAL damage of any kind to the switches, +after all. Physically, they were working perfectly. +This situation was "only" a software problem. + +But the 4ESS switches were leaping up and down every +four to six seconds, in a virulent spreading wave all over America, +in utter, manic, mechanical stupidity. They kept KNOCKING +one another down with their contagious "OK" messages. + +It took about ten minutes for the chain reaction to cripple the network. +Even then, switches would periodically luck-out and manage to resume +their normal work. Many calls--millions of them--were managing +to get through. But millions weren't. + +The switching stations that used System 6 were not directly affected. +Thanks to these old-fashioned switches, AT&T's national system avoided +complete collapse. This fact also made it clear to engineers that +System 7 was at fault. + +Bell Labs engineers, working feverishly in New Jersey, Illinois, +and Ohio, first tried their entire repertoire of standard network +remedies on the malfunctioning System 7. None of the remedies worked, +of course, because nothing like this had ever happened to any +phone system before. + +By cutting out the backup safety network entirely, +they were able to reduce the frenzy of "OK" messages +by about half. The system then began to recover, as the +chain reaction slowed. By 11:30 P.M. on Monday January +15, sweating engineers on the midnight shift breathed a +sigh of relief as the last switch cleared-up. + +By Tuesday they were pulling all the brand-new 4ESS software +and replacing it with an earlier version of System 7. + +If these had been human operators, rather than +computers at work, someone would simply have +eventually stopped screaming. It would have been +OBVIOUS that the situation was not "OK," and common +sense would have kicked in. Humans possess common sense-- +at least to some extent. Computers simply don't. + +On the other hand, computers can handle hundreds +of calls per second. Humans simply can't. If every single +human being in America worked for the phone company, +we couldn't match the performance of digital switches: +direct-dialling, three-way calling, speed-calling, call- +waiting, Caller ID, all the rest of the cornucopia +of digital bounty. Replacing computers with operators +is simply not an option any more. + +And yet we still, anachronistically, expect humans to +be running our phone system. It is hard for us +to understand that we have sacrificed huge amounts +of initiative and control to senseless yet powerful machines. +When the phones fail, we want somebody to be responsible. +We want somebody to blame. + +When the Crash of January 15 happened, the American populace +was simply not prepared to understand that enormous landslides +in cyberspace, like the Crash itself, can happen, +and can be nobody's fault in particular. It was easier to believe, +maybe even in some odd way more reassuring to believe, +that some evil person, or evil group, had done this to us. +"Hackers" had done it. With a virus. A trojan horse. +A software bomb. A dirty plot of some kind. People believed this, +responsible people. In 1990, they were looking hard for evidence +to confirm their heartfelt suspicions. + +And they would look in a lot of places. + +Come 1991, however, the outlines of an apparent new reality +would begin to emerge from the fog. + +On July 1 and 2, 1991, computer-software collapses +in telephone switching stations disrupted service in +Washington DC, Pittsburgh, Los Angeles and San Francisco. +Once again, seemingly minor maintenance problems had +crippled the digital System 7. About twelve million +people were affected in the Crash of July 1, 1991. + +Said the New York Times Service: "Telephone company executives +and federal regulators said they were not ruling out the possibility +of sabotage by computer hackers, but most seemed to think the problems +stemmed from some unknown defect in the software running the networks." + +And sure enough, within the week, a red-faced software company, +DSC Communications Corporation of Plano, Texas, owned up +to "glitches" in the "signal transfer point" software that +DSC had designed for Bell Atlantic and Pacific Bell. +The immediate cause of the July 1 Crash was a single +mistyped character: one tiny typographical flaw +in one single line of the software. One mistyped letter, +in one single line, had deprived the nation's capital of phone service. +It was not particularly surprising that this tiny flaw had escaped attention: +a typical System 7 station requires TEN MILLION lines of code. + +On Tuesday, September 17, 1991, came the most spectacular outage yet. +This case had nothing to do with software failures--at least, not directly. +Instead, a group of AT&T's switching stations in New York City had simply +run out of electrical power and shut down cold. Their back-up batteries +had failed. Automatic warning systems were supposed to warn of the loss +of battery power, but those automatic systems had failed as well. + +This time, Kennedy, La Guardia, and Newark airports +all had their voice and data communications cut. +This horrifying event was particularly ironic, as attacks +on airport computers by hackers had long been a standard +nightmare scenario, much trumpeted by computer-security +experts who feared the computer underground. There had even +been a Hollywood thriller about sinister hackers ruining +airport computers--DIE HARD II. + +Now AT&T itself had crippled airports with computer malfunctions-- +not just one airport, but three at once, some of the busiest in the world. + +Air traffic came to a standstill throughout the Greater New York area, +causing more than 500 flights to be cancelled, in a spreading wave +all over America and even into Europe. Another 500 or so flights +were delayed, affecting, all in all, about 85,000 passengers. +(One of these passengers was the chairman of the Federal +Communications Commission.) + +Stranded passengers in New York and New Jersey were further +infuriated to discover that they could not even manage to +make a long distance phone call, to explain their delay +to loved ones or business associates. Thanks to the crash, +about four and a half million domestic calls, and half a million +international calls, failed to get through. + +The September 17 NYC Crash, unlike the previous ones, +involved not a whisper of "hacker" misdeeds. On the contrary, +by 1991, AT&T itself was suffering much of the vilification +that had formerly been directed at hackers. Congressmen were grumbling. +So were state and federal regulators. And so was the press. + +For their part, ancient rival MCI took out snide full-page +newspaper ads in New York, offering their own long-distance +services for the "next time that AT&T goes down." + +"You wouldn't find a classy company like AT&T using such advertising," +protested AT&T Chairman Robert Allen, unconvincingly. Once again, +out came the full-page AT&T apologies in newspapers, apologies for +"an inexcusable culmination of both human and mechanical failure." +(This time, however, AT&T offered no discount on later calls. +Unkind critics suggested that AT&T were worried about setting any precedent +for refunding the financial losses caused by telephone crashes.) + +Industry journals asked publicly if AT&T was "asleep at the switch." +The telephone network, America's purported marvel of high-tech reliability, +had gone down three times in 18 months. Fortune magazine listed the +Crash of September 17 among the "Biggest Business Goofs of 1991," +cruelly parodying AT&T's ad campaign in an article entitled +"AT&T Wants You Back (Safely On the Ground, God Willing)." + +Why had those New York switching systems simply run out of power? +Because no human being had attended to the alarm system. +Why did the alarm systems blare automatically, +without any human being noticing? Because the three +telco technicians who SHOULD have been listening +were absent from their stations in the power-room, +on another floor of the building--attending a training class. +A training class about the alarm systems for the power room! + +"Crashing the System" was no longer "unprecedented" by late 1991. +On the contrary, it no longer even seemed an oddity. By 1991, +it was clear that all the policemen in the world could no longer +"protect" the phone system from crashes. By far the worst crashes +the system had ever had, had been inflicted, by the system, +upon ITSELF. And this time nobody was making cocksure statements +that this was an anomaly, something that would never happen again. +By 1991 the System's defenders had met their nebulous Enemy, +and the Enemy was--the System. + + + +PART TWO: THE DIGITAL UNDERGROUND + + +The date was May 9, 1990. The Pope was touring Mexico City. +Hustlers from the Medellin Cartel were trying to buy +black-market Stinger missiles in Florida. On the comics page, +Doonesbury character Andy was dying of AIDS. And then. . .a highly +unusual item whose novelty and calculated rhetoric won it +headscratching attention in newspapers all over America. + +The US Attorney's office in Phoenix, Arizona, had issued +a press release announcing a nationwide law enforcement crackdown +against "illegal computer hacking activities." The sweep was +officially known as "Operation Sundevil." + +Eight paragraphs in the press release gave the bare facts: +twenty-seven search warrants carried out on May 8, with three arrests, +and a hundred and fifty agents on the prowl in "twelve" cities across America. +(Different counts in local press reports yielded "thirteen," "fourteen," and +"sixteen" cities.) Officials estimated that criminal losses of revenue +to telephone companies "may run into millions of dollars." Credit for +the Sundevil investigations was taken by the US Secret Service, +Assistant US Attorney Tim Holtzen of Phoenix, and the Assistant +Attorney General of Arizona, Gail Thackeray. + +The prepared remarks of Garry M. Jenkins, appearing in a U.S. Department +of Justice press release, were of particular interest. Mr. Jenkins was the +Assistant Director of the US Secret Service, and the highest-ranking federal +official to take any direct public role in the hacker crackdown of 1990. + +"Today, the Secret Service is sending a clear message to those computer hackers +who have decided to violate the laws of this nation in the mistaken belief +that they can successfully avoid detection by hiding behind the relative +anonymity of their computer terminals. (. . .) "Underground groups have been +formed for the purpose of exchanging information relevant to their criminal +activities. These groups often communicate with each other through message +systems between computers called `bulletin boards.' "Our experience shows +that many computer hacker suspects are no longer misguided teenagers, +mischievously playing games with their computers in their bedrooms. +Some are now high tech computer operators using computers to engage +in unlawful conduct." + +Who were these "underground groups" and "high-tech operators?" +Where had they come from? What did they want? Who WERE they? +Were they "mischievous?" Were they dangerous? How had "misguided teenagers" +managed to alarm the United States Secret Service? And just how widespread +was this sort of thing? + +Of all the major players in the Hacker Crackdown: the phone companies, +law enforcement, the civil libertarians, and the "hackers" themselves-- +the "hackers" are by far the most mysterious, by far the hardest to +understand, by far the WEIRDEST. + +Not only are "hackers" novel in their activities, but they come +in a variety of odd subcultures, with a variety of languages, +motives and values. + +The earliest proto-hackers were probably those unsung mischievous +telegraph boys who were summarily fired by the Bell Company in 1878. + +Legitimate "hackers," those computer enthusiasts who are independent-minded +but law-abiding, generally trace their spiritual ancestry to elite technical +universities, especially M.I.T. and Stanford, in the 1960s. + +But the genuine roots of the modern hacker UNDERGROUND can probably be traced +most successfully to a now much-obscured hippie anarchist movement known as +the Yippies. The Yippies, who took their name from the largely fictional +"Youth International Party," carried out a loud and lively policy of surrealistic +subversion and outrageous political mischief. Their basic tenets were flagrant +sexual promiscuity, open and copious drug use, the political overthrow of any +powermonger over thirty years of age, and an immediate end to the war +in Vietnam, by any means necessary, including the psychic levitation +of the Pentagon. + +The two most visible Yippies were Abbie Hoffman and Jerry Rubin. +Rubin eventually became a Wall Street broker. Hoffman, ardently sought +by federal authorities, went into hiding for seven years, +in Mexico, France, and the United States. While on the lam, +Hoffman continued to write and publish, with help from sympathizers +in the American anarcho-leftist underground. Mostly, Hoffman survived +through false ID and odd jobs. Eventually he underwent facial plastic +surgery and adopted an entirely new identity as one "Barry Freed." +After surrendering himself to authorities in 1980, Hoffman spent a year +in prison on a cocaine conviction. + +Hoffman's worldview grew much darker as the glory days of the 1960s faded. +In 1989, he purportedly committed suicide, under odd and, to some, rather +suspicious circumstances. + +Abbie Hoffman is said to have caused the Federal Bureau of Investigation +to amass the single largest investigation file ever opened on an individual +American citizen. (If this is true, it is still questionable whether the +FBI regarded Abbie Hoffman a serious public threat--quite possibly, +his file was enormous simply because Hoffman left colorful legendry +wherever he went). He was a gifted publicist, who regarded electronic +media as both playground and weapon. He actively enjoyed manipulating +network TV and other gullible, image-hungry media, with various weird lies, +mindboggling rumors, impersonation scams, and other sinister distortions, +all absolutely guaranteed to upset cops, Presidential candidates, +and federal judges. Hoffman's most famous work was a book self-reflexively +known as STEAL THIS BOOK, which publicized a number of methods by which young, +penniless hippie agitators might live off the fat of a system supported by +humorless drones. STEAL THIS BOOK, whose title urged readers to damage +the very means of distribution which had put it into their hands, +might be described as a spiritual ancestor of a computer virus. + +Hoffman, like many a later conspirator, made extensive use of +pay-phones for his agitation work--in his case, generally through +the use of cheap brass washers as coin-slugs. + +During the Vietnam War, there was a federal surtax imposed on telephone +service; Hoffman and his cohorts could, and did, argue that in systematically +stealing phone service they were engaging in civil disobedience: +virtuously denying tax funds to an illegal and immoral war. + +But this thin veil of decency was soon dropped entirely. +Ripping-off the System found its own justification in deep alienation +and a basic outlaw contempt for conventional bourgeois values. +Ingenious, vaguely politicized varieties of rip-off, +which might be described as "anarchy by convenience," +became very popular in Yippie circles, and because rip-off +was so useful, it was to survive the Yippie movement itself. + +In the early 1970s, it required fairly limited expertise +and ingenuity to cheat payphones, to divert "free" +electricity and gas service, or to rob vending machines +and parking meters for handy pocket change. It also required +a conspiracy to spread this knowledge, and the gall +and nerve actually to commit petty theft, but the Yippies +had these qualifications in plenty. In June 1971, Abbie +Hoffman and a telephone enthusiast sarcastically known +as "Al Bell" began publishing a newsletter called Youth +International Party Line. This newsletter was dedicated +to collating and spreading Yippie rip-off techniques, +especially of phones, to the joy of the freewheeling +underground and the insensate rage of all straight people. +As a political tactic, phone-service theft ensured +that Yippie advocates would always have ready access +to the long-distance telephone as a medium, despite +the Yippies' chronic lack of organization, discipline, +money, or even a steady home address. + +PARTY LINE was run out of Greenwich Village for a couple of years, +then "Al Bell" more or less defected from the faltering ranks of Yippiedom, +changing the newsletter's name to TAP or Technical Assistance Program. +After the Vietnam War ended, the steam began leaking rapidly out of American +radical dissent. But by this time, "Bell" and his dozen or so +core contributors had the bit between their teeth, +and had begun to derive tremendous gut-level satisfaction +from the sensation of pure TECHNICAL POWER. + +TAP articles, once highly politicized, became pitilessly jargonized +and technical, in homage or parody to the Bell System's own technical +documents, which TAP studied closely, gutted, and reproduced without +permission. The TAP elite revelled in gloating possession +of the specialized knowledge necessary to beat the system. + +"Al Bell" dropped out of the game by the late 70s, +and "Tom Edison" took over; TAP readers (some 1400 of +them, all told) now began to show more interest in telex +switches and the growing phenomenon of computer systems. + +In 1983, "Tom Edison" had his computer stolen and his house +set on fire by an arsonist. This was an eventually mortal blow +to TAP (though the legendary name was to be resurrected +in 1990 by a young Kentuckian computer-outlaw named "Predat0r.") + +# + +Ever since telephones began to make money, there have been +people willing to rob and defraud phone companies. +The legions of petty phone thieves vastly outnumber those +"phone phreaks" who "explore the system" for the sake +of the intellectual challenge. The New York metropolitan area +(long in the vanguard of American crime) claims over 150,000 +physical attacks on pay telephones every year! Studied carefully, +a modern payphone reveals itself as a little fortress, carefully +designed and redesigned over generations, to resist coin-slugs, +zaps of electricity, chunks of coin-shaped ice, prybars, magnets, +lockpicks, blasting caps. Public pay- phones must survive in a world +of unfriendly, greedy people, and a modern payphone is as exquisitely +evolved as a cactus. +Because the phone network pre-dates the computer network, +the scofflaws known as "phone phreaks" pre-date the scofflaws +known as "computer hackers." In practice, today, the line +between "phreaking" and "hacking" is very blurred, +just as the distinction between telephones and computers +has blurred. The phone system has been digitized, +and computers have learned to "talk" over phone-lines. +What's worse--and this was the point of the Mr. Jenkins +of the Secret Service--some hackers have learned to steal, +and some thieves have learned to hack. + +Despite the blurring, one can still draw a few useful +behavioral distinctions between "phreaks" and "hackers." +Hackers are intensely interested in the "system" per se, +and enjoy relating to machines. "Phreaks" are more +social, manipulating the system in a rough-and-ready +fashion in order to get through to other human beings, +fast, cheap and under the table. + +Phone phreaks love nothing so much as "bridges," +illegal conference calls of ten or twelve chatting +conspirators, seaboard to seaboard, lasting for many hours +--and running, of course, on somebody else's tab, +preferably a large corporation's. + +As phone-phreak conferences wear on, people drop out +(or simply leave the phone off the hook, while they +sashay off to work or school or babysitting), +and new people are phoned up and invited to join in, +from some other continent, if possible. Technical trivia, +boasts, brags, lies, head-trip deceptions, weird rumors, +and cruel gossip are all freely exchanged. + +The lowest rung of phone-phreaking is the theft of telephone access codes. +Charging a phone call to somebody else's stolen number is, of course, +a pig-easy way of stealing phone service, requiring practically no +technical expertise. This practice has been very widespread, +especially among lonely people without much money who are far from home. +Code theft has flourished especially in college dorms, military bases, +and, notoriously, among roadies for rock bands. Of late, code theft +has spread very rapidly among Third Worlders in the US, who pile up +enormous unpaid long-distance bills to the Caribbean, South America, +and Pakistan. + +The simplest way to steal phone-codes is simply to look over +a victim's shoulder as he punches-in his own code-number +on a public payphone. This technique is known as "shoulder-surfing," +and is especially common in airports, bus terminals, and train stations. +The code is then sold by the thief for a few dollars. The buyer abusing +the code has no computer expertise, but calls his Mom in New York, +Kingston or Caracas and runs up a huge bill with impunity. The losses +from this primitive phreaking activity are far, far greater than the +monetary losses caused by computer-intruding hackers. + +In the mid-to-late 1980s, until the introduction of sterner telco +security measures, COMPUTERIZED code theft worked like a charm, +and was virtually omnipresent throughout the digital underground, +among phreaks and hackers alike. This was accomplished through +programming one's computer to try random code numbers over the telephone +until one of them worked. Simple programs to do this were widely available +in the underground; a computer running all night was likely to come up with +a dozen or so useful hits. This could be repeated week after week until +one had a large library of stolen codes. + +Nowadays, the computerized dialling of hundreds of numbers +can be detected within hours and swiftly traced. +If a stolen code is repeatedly abused, this too can +be detected within a few hours. But for years in the 1980s, +the publication of stolen codes was a kind of elementary etiquette +for fledgling hackers. The simplest way to establish your bona-fides +as a raider was to steal a code through repeated random dialling +and offer it to the "community" for use. Codes could be both stolen, +and used, simply and easily from the safety of one's own bedroom, +with very little fear of detection or punishment. + +Before computers and their phone-line modems entered American homes +in gigantic numbers, phone phreaks had their own special telecommunications +hardware gadget, the famous "blue box." This fraud device (now rendered +increasingly useless by the digital evolution of the phone system) could +trick switching systems into granting free access to long-distance lines. +It did this by mimicking the system's own signal, a tone of 2600 hertz. + +Steven Jobs and Steve Wozniak, the founders of Apple Computer, Inc., +once dabbled in selling blue-boxes in college dorms in California. +For many, in the early days of phreaking, blue-boxing was scarcely +perceived as "theft," but rather as a fun (if sneaky) way to use +excess phone capacity harmlessly. After all, the long-distance +lines were JUST SITTING THERE. . . . Whom did it hurt, really? +If you're not DAMAGING the system, and you're not USING UP ANY +TANGIBLE RESOURCE, and if nobody FINDS OUT what you did, +then what real harm have you done? What exactly HAVE you "stolen," +anyway? If a tree falls in the forest and nobody hears it, +how much is the noise worth? Even now this remains a rather +dicey question. + +Blue-boxing was no joke to the phone companies, however. +Indeed, when Ramparts magazine, a radical publication in California, +printed the wiring schematics necessary to create a mute box in June 1972, +the magazine was seized by police and Pacific Bell phone-company officials. +The mute box, a blue-box variant, allowed its user to receive long-distance +calls free of charge to the caller. This device was closely described in a +Ramparts article wryly titled "Regulating the Phone Company In Your Home." +Publication of this article was held to be in violation of Californian +State Penal Code section 502.7, which outlaws ownership of wire-fraud +devices and the selling of "plans or instructions for any instrument, +apparatus, or device intended to avoid telephone toll charges." + +Issues of Ramparts were recalled or seized on the newsstands, +and the resultant loss of income helped put the magazine out of business. +This was an ominous precedent for free-expression issues, but the telco's +crushing of a radical-fringe magazine passed without serious challenge +at the time. Even in the freewheeling California 1970s, it was widely felt +that there was something sacrosanct about what the phone company knew; +that the telco had a legal and moral right to protect itself by shutting +off the flow of such illicit information. Most telco information was so +"specialized" that it would scarcely be understood by any honest member +of the public. If not published, it would not be missed. To print such +material did not seem part of the legitimate role of a free press. + +In 1990 there would be a similar telco-inspired attack +on the electronic phreak/hacking "magazine" Phrack. +The Phrack legal case became a central issue in the +Hacker Crackdown, and gave rise to great controversy. +Phrack would also be shut down, for a time, at least, +but this time both the telcos and their law-enforcement +allies would pay a much larger price for their actions. +The Phrack case will be examined in detail, later. + +Phone-phreaking as a social practice is still very +much alive at this moment. Today, phone-phreaking +is thriving much more vigorously than the better-known +and worse-feared practice of "computer hacking." +New forms of phreaking are spreading rapidly, following +new vulnerabilities in sophisticated phone services. + +Cellular phones are especially vulnerable; their chips +can be re-programmed to present a false caller ID +and avoid billing. Doing so also avoids police tapping, +making cellular-phone abuse a favorite among drug-dealers. +"Call-sell operations" using pirate cellular phones can, +and have, been run right out of the backs of cars, which move +from "cell" to "cell" in the local phone system, retailing +stolen long-distance service, like some kind of demented +electronic version of the neighborhood ice-cream truck. + +Private branch-exchange phone systems in large corporations +can be penetrated; phreaks dial-up a local company, enter its +internal phone-system, hack it, then use the company's own +PBX system to dial back out over the public network, +causing the company to be stuck with the resulting +long-distance bill. This technique is known as "diverting." +"Diverting" can be very costly, especially because phreaks +tend to travel in packs and never stop talking. +Perhaps the worst by-product of this "PBX fraud" +is that victim companies and telcos have sued one another +over the financial responsibility for the stolen calls, +thus enriching not only shabby phreaks but well-paid lawyers. + +"Voice-mail systems" can also be abused; phreaks +can seize their own sections of these sophisticated +electronic answering machines, and use them for trading +codes or knowledge of illegal techniques. Voice-mail +abuse does not hurt the company directly, but finding +supposedly empty slots in your company's answering +machine all crammed with phreaks eagerly chattering +and hey-duding one another in impenetrable jargon can +cause sensations of almost mystical repulsion and dread. + +Worse yet, phreaks have sometimes been known to react +truculently to attempts to "clean up" the voice-mail system. +Rather than humbly acquiescing to being thrown out of their playground, +they may very well call up the company officials at work (or at home) +and loudly demand free voice-mail addresses of their very own. +Such bullying is taken very seriously by spooked victims. + +Acts of phreak revenge against straight people are rare, +but voice-mail systems are especially tempting and vulnerable, +and an infestation of angry phreaks in one's voice-mail system is no joke. +They can erase legitimate messages; or spy on private messages; +or harass users with recorded taunts and obscenities. +They've even been known to seize control of voice-mail security, +and lock out legitimate users, or even shut down the system entirely. + +Cellular phone-calls, cordless phones, and ship-to-shore +telephony can all be monitored by various forms of radio; +this kind of "passive monitoring" is spreading explosively today. +Technically eavesdropping on other people's cordless and cellular +phone-calls is the fastest-growing area in phreaking today. +This practice strongly appeals to the lust for power and conveys +gratifying sensations of technical superiority over the eavesdropping +victim. Monitoring is rife with all manner of tempting evil mischief. +Simple prurient snooping is by far the most common activity. +But credit-card numbers unwarily spoken over the phone can be recorded, +stolen and used. And tapping people's phone-calls (whether through +active telephone taps or passive radio monitors) does lend itself +conveniently to activities like blackmail, industrial espionage, +and political dirty tricks. + +It should be repeated that telecommunications fraud, +the theft of phone service, causes vastly greater monetary +losses than the practice of entering into computers by stealth. +Hackers are mostly young suburban American white males, +and exist in their hundreds--but "phreaks" come from both sexes +and from many nationalities, ages and ethnic backgrounds, +and are flourishing in the thousands. + +# + +The term "hacker" has had an unfortunate history. +This book, The Hacker Crackdown, has little to say about +"hacking" in its finer, original sense. The term can signify +the free-wheeling intellectual exploration of the highest +and deepest potential of computer systems. Hacking can +describe the determination to make access to computers +and information as free and open as possible. Hacking +can involve the heartfelt conviction that beauty can +be found in computers, that the fine aesthetic in a perfect +program can liberate the mind and spirit. This is "hacking" +as it was defined in Steven Levy's much-praised history +of the pioneer computer milieu, Hackers, published in 1984. + +Hackers of all kinds are absolutely soaked through with heroic +anti-bureaucratic sentiment. Hackers long for recognition +as a praiseworthy cultural archetype, the postmodern electronic +equivalent of the cowboy and mountain man. Whether they deserve +such a reputation is something for history to decide. But many hackers-- +including those outlaw hackers who are computer intruders, and whose +activities are defined as criminal--actually attempt to LIVE UP TO +this techno-cowboy reputation. And given that electronics and +telecommunications are still largely unexplored territories, +there is simply NO TELLING what hackers might uncover. + +For some people, this freedom is the very breath of oxygen, +the inventive spontaneity that makes life worth living +and that flings open doors to marvellous possibility and +individual empowerment. But for many people +--and increasingly so--the hacker is an ominous figure, +a smart-aleck sociopath ready to burst out of his basement +wilderness and savage other people's lives for his own +anarchical convenience. + +Any form of power without responsibility, without direct +and formal checks and balances, is frightening to people-- +and reasonably so. It should be frankly admitted that +hackers ARE frightening, and that the basis of this fear +is not irrational. + +Fear of hackers goes well beyond the fear of merely criminal activity. + +Subversion and manipulation of the phone system +is an act with disturbing political overtones. +In America, computers and telephones are potent symbols +of organized authority and the technocratic business elite. + +But there is an element in American culture that +has always strongly rebelled against these symbols; +rebelled against all large industrial computers +and all phone companies. A certain anarchical tinge deep +in the American soul delights in causing confusion and pain +to all bureaucracies, including technological ones. + +There is sometimes malice and vandalism in this attitude, +but it is a deep and cherished part of the American national character. +The outlaw, the rebel, the rugged individual, the pioneer, +the sturdy Jeffersonian yeoman, the private citizen resisting +interference in his pursuit of happiness--these are figures that all +Americans recognize, and that many will strongly applaud and defend. + +Many scrupulously law-abiding citizens today do cutting-edge work +with electronics--work that has already had tremendous social influence +and will have much more in years to come. In all truth, these talented, +hardworking, law-abiding, mature, adult people are far more disturbing +to the peace and order of the current status quo than any scofflaw group +of romantic teenage punk kids. These law-abiding hackers have the power, +ability, and willingness to influence other people's lives quite unpredictably. +They have means, motive, and opportunity to meddle drastically with the +American social order. When corralled into governments, universities, +or large multinational companies, and forced to follow rulebooks +and wear suits and ties, they at least have some conventional halters +on their freedom of action. But when loosed alone, or in small groups, +and fired by imagination and the entrepreneurial spirit, they can move +mountains--causing landslides that will likely crash directly into your +office and living room. + +These people, as a class, instinctively recognize that a public, +politicized attack on hackers will eventually spread to them-- +that the term "hacker," once demonized, might be used to knock +their hands off the levers of power and choke them out of existence. +There are hackers today who fiercely and publicly resist any besmirching +of the noble title of hacker. Naturally and understandably, they deeply +resent the attack on their values implicit in using the word "hacker" +as a synonym for computer-criminal. + +This book, sadly but in my opinion unavoidably, rather adds +to the degradation of the term. It concerns itself mostly with "hacking" +in its commonest latter-day definition, i.e., intruding into computer +systems by stealth and without permission. The term "hacking" is used +routinely today by almost all law enforcement officials with any +professional interest in computer fraud and abuse. American police +describe almost any crime committed with, by, through, or against +a computer as hacking. + +Most importantly, "hacker" is what computer-intruders +choose to call THEMSELVES. Nobody who "hacks" into systems +willingly describes himself (rarely, herself) as a "computer intruder," +"computer trespasser," "cracker," "wormer," "darkside hacker" +or "high tech street gangster." Several other demeaning terms +have been invented in the hope that the press and public +will leave the original sense of the word alone. But few people +actually use these terms. (I exempt the term "cyberpunk," +which a few hackers and law enforcement people actually do use. +The term "cyberpunk" is drawn from literary criticism and has +some odd and unlikely resonances, but, like hacker, +cyberpunk too has become a criminal pejorative today.) + +In any case, breaking into computer systems was hardly alien +to the original hacker tradition. The first tottering systems +of the 1960s required fairly extensive internal surgery merely +to function day-by-day. Their users "invaded" the deepest, +most arcane recesses of their operating software almost +as a matter of routine. "Computer security" in these early, +primitive systems was at best an afterthought. What security +there was, was entirely physical, for it was assumed that +anyone allowed near this expensive, arcane hardware would be +a fully qualified professional expert. + +In a campus environment, though, this meant that grad students, +teaching assistants, undergraduates, and eventually, +all manner of dropouts and hangers-on ended up accessing +and often running the works. + +Universities, even modern universities, are not in +the business of maintaining security over information. +On the contrary, universities, as institutions, pre-date +the "information economy" by many centuries and are not- +for-profit cultural entities, whose reason for existence +(purportedly) is to discover truth, codify it through +techniques of scholarship, and then teach it. Universities +are meant to PASS THE TORCH OF CIVILIZATION, not just +download data into student skulls, and the values of the +academic community are strongly at odds with those of all +would-be information empires. Teachers at all levels, from +kindergarten up, have proven to be shameless and persistent +software and data pirates. Universities do not merely +"leak information" but vigorously broadcast free thought. + +This clash of values has been fraught with controversy. +Many hackers of the 1960s remember their professional +apprenticeship as a long guerilla war against the uptight +mainframe-computer "information priesthood." These computer-hungry +youngsters had to struggle hard for access to computing power, +and many of them were not above certain, er, shortcuts. +But, over the years, this practice freed computing +from the sterile reserve of lab-coated technocrats and +was largely responsible for the explosive growth of computing +in general society--especially PERSONAL computing. + +Access to technical power acted like catnip on certain +of these youngsters. Most of the basic techniques of +computer intrusion: password cracking, trapdoors, backdoors, +trojan horses--were invented in college environments in the 1960s, +in the early days of network computing. Some off-the-cuff +experience at computer intrusion was to be in the informal +resume of most "hackers" and many future industry giants. +Outside of the tiny cult of computer enthusiasts, few people +thought much about the implications of "breaking into" +computers. This sort of activity had not yet been publicized, +much less criminalized. + +In the 1960s, definitions of "property" and "privacy" +had not yet been extended to cyberspace. Computers +were not yet indispensable to society. There were no vast +databanks of vulnerable, proprietary information stored +in computers, which might be accessed, copied without +permission, erased, altered, or sabotaged. The stakes +were low in the early days--but they grew every year, +exponentially, as computers themselves grew. + +By the 1990s, commercial and political pressures +had become overwhelming, and they broke the social +boundaries of the hacking subculture. Hacking +had become too important to be left to the hackers. +Society was now forced to tackle the intangible nature +of cyberspace-as-property, cyberspace as privately-owned +unreal-estate. In the new, severe, responsible, high-stakes +context of the "Information Society" of the 1990s, +"hacking" was called into question. + +What did it mean to break into a computer without +permission and use its computational power, or look +around inside its files without hurting anything? +What were computer-intruding hackers, anyway--how should +society, and the law, best define their actions? +Were they just BROWSERS, harmless intellectual explorers? +Were they VOYEURS, snoops, invaders of privacy? Should +they be sternly treated as potential AGENTS OF ESPIONAGE, +or perhaps as INDUSTRIAL SPIES? Or were they best +defined as TRESPASSERS, a very common teenage +misdemeanor? Was hacking THEFT OF SERVICE? +(After all, intruders were getting someone else's +computer to carry out their orders, without permission +and without paying). Was hacking FRAUD? Maybe it was +best described as IMPERSONATION. The commonest mode +of computer intrusion was (and is) to swipe or snoop +somebody else's password, and then enter the computer +in the guise of another person--who is commonly stuck +with the blame and the bills. + +Perhaps a medical metaphor was better--hackers should +be defined as "sick," as COMPUTER ADDICTS unable +to control their irresponsible, compulsive behavior. + +But these weighty assessments meant little to the +people who were actually being judged. From inside +the underground world of hacking itself, all these +perceptions seem quaint, wrongheaded, stupid, or meaningless. +The most important self-perception of underground hackers-- +from the 1960s, right through to the present day--is that +they are an ELITE. The day-to-day struggle in the underground +is not over sociological definitions--who cares?--but for power, +knowledge, and status among one's peers. + +When you are a hacker, it is your own inner conviction +of your elite status that enables you to break, or let +us say "transcend," the rules. It is not that ALL rules +go by the board. The rules habitually broken by hackers +are UNIMPORTANT rules--the rules of dopey greedhead telco +bureaucrats and pig-ignorant government pests. + +Hackers have their OWN rules, which separate behavior +which is cool and elite, from behavior which is rodentlike, +stupid and losing. These "rules," however, are mostly unwritten +and enforced by peer pressure and tribal feeling. Like all rules +that depend on the unspoken conviction that everybody else +is a good old boy, these rules are ripe for abuse. The mechanisms +of hacker peer- pressure, "teletrials" and ostracism, are rarely used +and rarely work. Back-stabbing slander, threats, and electronic +harassment are also freely employed in down-and-dirty intrahacker feuds, +but this rarely forces a rival out of the scene entirely. The only real +solution for the problem of an utterly losing, treacherous and rodentlike +hacker is to TURN HIM IN TO THE POLICE. Unlike the Mafia or Medellin Cartel, +the hacker elite cannot simply execute the bigmouths, creeps and troublemakers +among their ranks, so they turn one another in with astonishing frequency. + +There is no tradition of silence or OMERTA in the hacker underworld. +Hackers can be shy, even reclusive, but when they do talk, hackers +tend to brag, boast and strut. Almost everything hackers do is INVISIBLE; +if they don't brag, boast, and strut about it, then NOBODY WILL EVER KNOW. +If you don't have something to brag, boast, and strut about, then nobody +in the underground will recognize you and favor you with vital cooperation +and respect. + +The way to win a solid reputation in the underground +is by telling other hackers things that could only +have been learned by exceptional cunning and stealth. +Forbidden knowledge, therefore, is the basic currency +of the digital underground, like seashells among +Trobriand Islanders. Hackers hoard this knowledge, +and dwell upon it obsessively, and refine it, +and bargain with it, and talk and talk about it. + +Many hackers even suffer from a strange obsession to TEACH-- +to spread the ethos and the knowledge of the digital underground. +They'll do this even when it gains them no particular advantage +and presents a grave personal risk. + +And when that risk catches up with them, they will go right on teaching +and preaching--to a new audience this time, their interrogators from law +enforcement. Almost every hacker arrested tells everything he knows-- +all about his friends, his mentors, his disciples--legends, threats, +horror stories, dire rumors, gossip, hallucinations. This is, of course, +convenient for law enforcement--except when law enforcement begins +to believe hacker legendry. + +Phone phreaks are unique among criminals in their willingness +to call up law enforcement officials--in the office, at their homes-- +and give them an extended piece of their mind. It is hard not to +interpret this as BEGGING FOR ARREST, and in fact it is an act +of incredible foolhardiness. Police are naturally nettled +by these acts of chutzpah and will go well out of their way +to bust these flaunting idiots. But it can also be interpreted +as a product of a world-view so elitist, so closed and hermetic, +that electronic police are simply not perceived as "police," +but rather as ENEMY PHONE PHREAKS who should be scolded +into behaving "decently." + +Hackers at their most grandiloquent perceive themselves +as the elite pioneers of a new electronic world. +Attempts to make them obey the democratically +established laws of contemporary American society are +seen as repression and persecution. After all, they argue, +if Alexander Graham Bell had gone along with the rules +of the Western Union telegraph company, there would have +been no telephones. If Jobs and Wozniak had believed +that IBM was the be-all and end-all, there would have +been no personal computers. If Benjamin Franklin and +Thomas Jefferson had tried to "work within the system" +there would have been no United States. + +Not only do hackers privately believe this as an article of faith, +but they have been known to write ardent manifestos about it. +Here are some revealing excerpts from an especially vivid hacker manifesto: +"The Techno-Revolution" by "Dr. Crash," which appeared in electronic +form in Phrack Volume 1, Issue 6, Phile 3. + + +"To fully explain the true motives behind hacking, +we must first take a quick look into the past. In the 1960s, +a group of MIT students built the first modern computer system. +This wild, rebellious group of young men were the first to bear +the name `hackers.' The systems that they developed were intended +to be used to solve world problems and to benefit all of mankind. +"As we can see, this has not been the case. The computer system +has been solely in the hands of big businesses and the government. +The wonderful device meant to enrich life has become a weapon which +dehumanizes people. To the government and large businesses, +people are no more than disk space, and the government doesn't +use computers to arrange aid for the poor, but to control nuclear +death weapons. The average American can only have access +to a small microcomputer which is worth only a fraction +of what they pay for it. The businesses keep the +true state-of-the-art equipment away from the people +behind a steel wall of incredibly high prices and bureaucracy. +It is because of this state of affairs that hacking was born. (. . .) +"Of course, the government doesn't want the monopoly of technology broken, +so they have outlawed hacking and arrest anyone who is caught. (. . .) +The phone company is another example of technology abused and kept +from people with high prices. (. . .) "Hackers often find that their +existing equipment, due to the monopoly tactics of computer companies, +is inefficient for their purposes. Due to the exorbitantly high prices, +it is impossible to legally purchase the necessary equipment. +This need has given still another segment of the fight: Credit Carding. +Carding is a way of obtaining the necessary goods without paying for them. +It is again due to the companies' stupidity that Carding is so easy, +and shows that the world's businesses are in the hands of those +with considerably less technical know-how than we, the hackers. (. . .) +"Hacking must continue. We must train newcomers to the art of hacking. +(. . . .) And whatever you do, continue the fight. Whether you know it +or not, if you are a hacker, you are a revolutionary. Don't worry, +you're on the right side." + +The defense of "carding" is rare. Most hackers regard credit-card +theft as "poison" to the underground, a sleazy and immoral effort that, +worse yet, is hard to get away with. Nevertheless, manifestos advocating +credit-card theft, the deliberate crashing of computer systems, +and even acts of violent physical destruction such as vandalism +and arson do exist in the underground. These boasts and threats +are taken quite seriously by the police. And not every hacker +is an abstract, Platonic computer-nerd. Some few are quite experienced +at picking locks, robbing phone-trucks, and breaking and entering buildings. + +Hackers vary in their degree of hatred for authority +and the violence of their rhetoric. But, at a bottom line, +they are scofflaws. They don't regard the current rules +of electronic behavior as respectable efforts to preserve +law and order and protect public safety. They regard these +laws as immoral efforts by soulless corporations to protect +their profit margins and to crush dissidents. "Stupid" people, +including police, businessmen, politicians, and journalists, +simply have no right to judge the actions of those possessed of genius, +techno-revolutionary intentions, and technical expertise. + +# + +Hackers are generally teenagers and college kids not +engaged in earning a living. They often come from fairly +well-to-do middle-class backgrounds, and are markedly +anti-materialistic (except, that is, when it comes to +computer equipment). Anyone motivated by greed for +mere money (as opposed to the greed for power, +knowledge and status) is swiftly written-off as a narrow- +minded breadhead whose interests can only be corrupt +and contemptible. Having grown up in the 1970s and +1980s, the young Bohemians of the digital underground +regard straight society as awash in plutocratic corruption, +where everyone from the President down is for sale and +whoever has the gold makes the rules. + +Interestingly, there's a funhouse-mirror image of this attitude +on the other side of the conflict. The police are also +one of the most markedly anti-materialistic groups +in American society, motivated not by mere money +but by ideals of service, justice, esprit-de-corps, +and, of course, their own brand of specialized knowledge +and power. Remarkably, the propaganda war between cops +and hackers has always involved angry allegations +that the other side is trying to make a sleazy buck. +Hackers consistently sneer that anti-phreak prosecutors +are angling for cushy jobs as telco lawyers and that +computer-crime police are aiming to cash in later +as well-paid computer-security consultants in the private sector. + +For their part, police publicly conflate all +hacking crimes with robbing payphones with crowbars. +Allegations of "monetary losses" from computer intrusion +are notoriously inflated. The act of illicitly copying +a document from a computer is morally equated with +directly robbing a company of, say, half a million dollars. +The teenage computer intruder in possession of this "proprietary" +document has certainly not sold it for such a sum, would likely +have little idea how to sell it at all, and quite probably +doesn't even understand what he has. He has not made a cent +in profit from his felony but is still morally equated with +a thief who has robbed the church poorbox and lit out for Brazil. + +Police want to believe that all hackers are thieves. +It is a tortuous and almost unbearable act for the American +justice system to put people in jail because they want +to learn things which are forbidden for them to know. +In an American context, almost any pretext for punishment +is better than jailing people to protect certain restricted +kinds of information. Nevertheless, POLICING INFORMATION +is part and parcel of the struggle against hackers. + +This dilemma is well exemplified by the remarkable +activities of "Emmanuel Goldstein," editor and publisher +of a print magazine known as 2600: The Hacker Quarterly. +Goldstein was an English major at Long Island's State University +of New York in the '70s, when he became involved with the local +college radio station. His growing interest in electronics +caused him to drift into Yippie TAP circles and thus into +the digital underground, where he became a self-described +techno-rat. His magazine publishes techniques of computer +intrusion and telephone "exploration" as well as gloating +exposes of telco misdeeds and governmental failings. + +Goldstein lives quietly and very privately in a large, +crumbling Victorian mansion in Setauket, New York. +The seaside house is decorated with telco decals, chunks of +driftwood, and the basic bric-a-brac of a hippie crash-pad. +He is unmarried, mildly unkempt, and survives mostly +on TV dinners and turkey-stuffing eaten straight out +of the bag. Goldstein is a man of considerable charm +and fluency, with a brief, disarming smile and the kind +of pitiless, stubborn, thoroughly recidivist integrity +that America's electronic police find genuinely alarming. + +Goldstein took his nom-de-plume, or "handle," from +a character in Orwell's 1984, which may be taken, +correctly, as a symptom of the gravity of his sociopolitical +worldview. He is not himself a practicing computer +intruder, though he vigorously abets these actions, +especially when they are pursued against large +corporations or governmental agencies. Nor is he a thief, +for he loudly scorns mere theft of phone service, in favor +of "exploring and manipulating the system." He is probably +best described and understood as a DISSIDENT. + +Weirdly, Goldstein is living in modern America +under conditions very similar to those of former +East European intellectual dissidents. In other words, +he flagrantly espouses a value-system that is deeply +and irrevocably opposed to the system of those in power +and the police. The values in 2600 are generally expressed +in terms that are ironic, sarcastic, paradoxical, or just +downright confused. But there's no mistaking their +radically anti-authoritarian tenor. 2600 holds that +technical power and specialized knowledge, of any kind +obtainable, belong by right in the hands of those individuals +brave and bold enough to discover them--by whatever means necessary. +Devices, laws, or systems that forbid access, and the free +spread of knowledge, are provocations that any free +and self-respecting hacker should relentlessly attack. +The "privacy" of governments, corporations and other soulless +technocratic organizations should never be protected +at the expense of the liberty and free initiative +of the individual techno-rat. + +However, in our contemporary workaday world, both governments +and corporations are very anxious indeed to police information +which is secret, proprietary, restricted, confidential, +copyrighted, patented, hazardous, illegal, unethical, +embarrassing, or otherwise sensitive. This makes Goldstein +persona non grata, and his philosophy a threat. + +Very little about the conditions of Goldstein's daily +life would astonish, say, Vaclav Havel. (We may note +in passing that President Havel once had his word-processor +confiscated by the Czechoslovak police.) Goldstein lives +by SAMIZDAT, acting semi-openly as a data-center +for the underground, while challenging the powers-that-be +to abide by their own stated rules: freedom of speech +and the First Amendment. + +Goldstein thoroughly looks and acts the part of techno-rat, +with shoulder-length ringlets and a piratical black +fisherman's-cap set at a rakish angle. He often shows up +like Banquo's ghost at meetings of computer professionals, +where he listens quietly, half-smiling and taking thorough notes. + +Computer professionals generally meet publicly, +and find it very difficult to rid themselves of Goldstein +and his ilk without extralegal and unconstitutional actions. +Sympathizers, many of them quite respectable people +with responsible jobs, admire Goldstein's attitude and +surreptitiously pass him information. An unknown but +presumably large proportion of Goldstein's 2,000-plus +readership are telco security personnel and police, +who are forced to subscribe to 2600 to stay abreast +of new developments in hacking. They thus find themselves +PAYING THIS GUY'S RENT while grinding their teeth in anguish, +a situation that would have delighted Abbie Hoffman +(one of Goldstein's few idols). + +Goldstein is probably the best-known public representative +of the hacker underground today, and certainly the best-hated. +Police regard him as a Fagin, a corrupter of youth, and speak +of him with untempered loathing. He is quite an accomplished gadfly. +After the Martin Luther King Day Crash of 1990, Goldstein, +for instance, adeptly rubbed salt into the wound in the pages of 2600. +"Yeah, it was fun for the phone phreaks as we watched the network crumble," +he admitted cheerfully. "But it was also an ominous sign of what's +to come. . . . Some AT&T people, aided by well-meaning but ignorant media, +were spreading the notion that many companies had the same software +and therefore could face the same problem someday. Wrong. This was +entirely an AT&T software deficiency. Of course, other companies could +face entirely DIFFERENT software problems. But then, so too could AT&T." + +After a technical discussion of the system's failings, +the Long Island techno-rat went on to offer thoughtful +criticism to the gigantic multinational's hundreds of +professionally qualified engineers. "What we don't know +is how a major force in communications like AT&T could +be so sloppy. What happened to backups? Sure, +computer systems go down all the time, but people +making phone calls are not the same as people logging +on to computers. We must make that distinction. It's not +acceptable for the phone system or any other essential +service to `go down.' If we continue to trust technology +without understanding it, we can look forward to many +variations on this theme. + +"AT&T owes it to its customers to be prepared to INSTANTLY +switch to another network if something strange and unpredictable +starts occurring. The news here isn't so much the failure +of a computer program, but the failure of AT&T's entire structure." + +The very idea of this. . . . this PERSON. . . . offering +"advice" about "AT&T's entire structure" is more than +some people can easily bear. How dare this near-criminal +dictate what is or isn't "acceptable" behavior from AT&T? +Especially when he's publishing, in the very same issue, +detailed schematic diagrams for creating various switching-network +signalling tones unavailable to the public. + +"See what happens when you drop a `silver box' tone or two +down your local exchange or through different long distance +service carriers," advises 2600 contributor "Mr. Upsetter" +in "How To Build a Signal Box." "If you experiment systematically +and keep good records, you will surely discover something interesting." + +This is, of course, the scientific method, generally regarded +as a praiseworthy activity and one of the flowers of modern civilization. +One can indeed learn a great deal with this sort of structured +intellectual activity. Telco employees regard this mode of "exploration" +as akin to flinging sticks of dynamite into their pond to see what lives +on the bottom. + +2600 has been published consistently since 1984. +It has also run a bulletin board computer system, +printed 2600 T-shirts, taken fax calls. . . . +The Spring 1991 issue has an interesting announcement on page 45: +"We just discovered an extra set of wires attached to our fax line +and heading up the pole. (They've since been clipped.) +Your faxes to us and to anyone else could be monitored." +In the worldview of 2600, the tiny band of techno-rat brothers +(rarely, sisters) are a beseiged vanguard of the truly free and honest. +The rest of the world is a maelstrom of corporate crime and high-level +governmental corruption, occasionally tempered with well-meaning +ignorance. To read a few issues in a row is to enter a nightmare +akin to Solzhenitsyn's, somewhat tempered by the fact that 2600 +is often extremely funny. + +Goldstein did not become a target of the Hacker Crackdown, +though he protested loudly, eloquently, and publicly about it, +and it added considerably to his fame. It was not that he is not +regarded as dangerous, because he is so regarded. Goldstein has had +brushes with the law in the past: in 1985, a 2600 bulletin board +computer was seized by the FBI, and some software on it was formally +declared "a burglary tool in the form of a computer program." +But Goldstein escaped direct repression in 1990, because his +magazine is printed on paper, and recognized as subject +to Constitutional freedom of the press protection. +As was seen in the Ramparts case, this is far from +an absolute guarantee. Still, as a practical matter, +shutting down 2600 by court-order would create so much +legal hassle that it is simply unfeasible, at least +for the present. Throughout 1990, both Goldstein +and his magazine were peevishly thriving. + +Instead, the Crackdown of 1990 would concern itself +with the computerized version of forbidden data. +The crackdown itself, first and foremost, was about +BULLETIN BOARD SYSTEMS. Bulletin Board Systems, most often +known by the ugly and un-pluralizable acronym "BBS," are +the life-blood of the digital underground. Boards were +also central to law enforcement's tactics and strategy +in the Hacker Crackdown. + +A "bulletin board system" can be formally defined as +a computer which serves as an information and message- +passing center for users dialing-up over the phone-lines +through the use of modems. A "modem," or modulator- +demodulator, is a device which translates the digital +impulses of computers into audible analog telephone +signals, and vice versa. Modems connect computers +to phones and thus to each other. + +Large-scale mainframe computers have been connected since the 1960s, +but PERSONAL computers, run by individuals out of their homes, +were first networked in the late 1970s. The "board" created +by Ward Christensen and Randy Suess in February 1978, +in Chicago, Illinois, is generally regarded as the first +personal-computer bulletin board system worthy of the name. + +Boards run on many different machines, employing many +different kinds of software. Early boards were crude and buggy, +and their managers, known as "system operators" or "sysops," +were hard-working technical experts who wrote their own software. +But like most everything else in the world of electronics, +boards became faster, cheaper, better-designed, and generally +far more sophisticated throughout the 1980s. They also moved +swiftly out of the hands of pioneers and into those of the +general public. By 1985 there were something in the +neighborhood of 4,000 boards in America. By 1990 it was +calculated, vaguely, that there were about 30,000 boards in +the US, with uncounted thousands overseas. + +Computer bulletin boards are unregulated enterprises. +Running a board is a rough-and-ready, catch-as-catch-can proposition. +Basically, anybody with a computer, modem, software and a phone-line +can start a board. With second-hand equipment and public-domain +free software, the price of a board might be quite small-- +less than it would take to publish a magazine or even a +decent pamphlet. Entrepreneurs eagerly sell bulletin-board +software, and will coach nontechnical amateur sysops in its use. + +Boards are not "presses." They are not magazines, +or libraries, or phones, or CB radios, or traditional cork +bulletin boards down at the local laundry, though they +have some passing resemblance to those earlier media. +Boards are a new medium--they may even be a LARGE NUMBER of new media. + +Consider these unique characteristics: boards are cheap, +yet they can have a national, even global reach. +Boards can be contacted from anywhere in the global +telephone network, at NO COST to the person running the board-- +the caller pays the phone bill, and if the caller is local, +the call is free. Boards do not involve an editorial elite +addressing a mass audience. The "sysop" of a board is not +an exclusive publisher or writer--he is managing an electronic salon, +where individuals can address the general public, play the part +of the general public, and also exchange private mail +with other individuals. And the "conversation" on boards, +though fluid, rapid, and highly interactive, is not spoken, +but written. It is also relatively anonymous, sometimes completely so. + +And because boards are cheap and ubiquitous, regulations +and licensing requirements would likely be practically unenforceable. +It would almost be easier to "regulate," "inspect," and "license" +the content of private mail--probably more so, since the mail system +is operated by the federal government. Boards are run by individuals, +independently, entirely at their own whim. + +For the sysop, the cost of operation is not the primary +limiting factor. Once the investment in a computer and +modem has been made, the only steady cost is the charge +for maintaining a phone line (or several phone lines). +The primary limits for sysops are time and energy. +Boards require upkeep. New users are generally "validated"-- +they must be issued individual passwords, and called at +home by voice-phone, so that their identity can be +verified. Obnoxious users, who exist in plenty, must be +chided or purged. Proliferating messages must be deleted +when they grow old, so that the capacity of the system +is not overwhelmed. And software programs (if such things +are kept on the board) must be examined for possible +computer viruses. If there is a financial charge to use +the board (increasingly common, especially in larger and +fancier systems) then accounts must be kept, and users +must be billed. And if the board crashes--a very common +occurrence--then repairs must be made. + +Boards can be distinguished by the amount of effort +spent in regulating them. First, we have the completely +open board, whose sysop is off chugging brews and +watching re-runs while his users generally degenerate +over time into peevish anarchy and eventual silence. +Second comes the supervised board, where the sysop +breaks in every once in a while to tidy up, calm brawls, +issue announcements, and rid the community of dolts +and troublemakers. Third is the heavily supervised +board, which sternly urges adult and responsible behavior +and swiftly edits any message considered offensive, +impertinent, illegal or irrelevant. And last comes +the completely edited "electronic publication," which +is presented to a silent audience which is not allowed +to respond directly in any way. + +Boards can also be grouped by their degree of anonymity. +There is the completely anonymous board, where everyone +uses pseudonyms--"handles"--and even the sysop is unaware +of the user's true identity. The sysop himself is likely +pseudonymous on a board of this type. Second, and rather +more common, is the board where the sysop knows (or thinks +he knows) the true names and addresses of all users, +but the users don't know one another's names and may not know his. +Third is the board where everyone has to use real names, +and roleplaying and pseudonymous posturing are forbidden. + +Boards can be grouped by their immediacy. "Chat-lines" +are boards linking several users together over several +different phone-lines simultaneously, so that people +exchange messages at the very moment that they type. +(Many large boards feature "chat" capabilities along +with other services.) Less immediate boards, +perhaps with a single phoneline, store messages serially, +one at a time. And some boards are only open for business +in daylight hours or on weekends, which greatly slows response. +A NETWORK of boards, such as "FidoNet," can carry electronic mail +from board to board, continent to continent, across huge distances-- +but at a relative snail's pace, so that a message can take several +days to reach its target audience and elicit a reply. + +Boards can be grouped by their degree of community. +Some boards emphasize the exchange of private, +person-to-person electronic mail. Others emphasize +public postings and may even purge people who "lurk," +merely reading posts but refusing to openly participate. +Some boards are intimate and neighborly. Others are frosty +and highly technical. Some are little more than storage +dumps for software, where users "download" and "upload" programs, +but interact among themselves little if at all. + +Boards can be grouped by their ease of access. Some boards +are entirely public. Others are private and restricted only +to personal friends of the sysop. Some boards divide users by status. +On these boards, some users, especially beginners, strangers or children, +will be restricted to general topics, and perhaps forbidden to post. +Favored users, though, are granted the ability to post as they please, +and to stay "on-line" as long as they like, even to the disadvantage +of other people trying to call in. High-status users can be given access +to hidden areas in the board, such as off-color topics, private discussions, +and/or valuable software. Favored users may even become "remote sysops" +with the power to take remote control of the board through their own +home computers. Quite often "remote sysops" end up doing all the work +and taking formal control of the enterprise, despite the fact that it's +physically located in someone else's house. Sometimes several "co-sysops" +share power. + +And boards can also be grouped by size. Massive, nationwide +commercial networks, such as CompuServe, Delphi, GEnie and Prodigy, +are run on mainframe computers and are generally not considered "boards," +though they share many of their characteristics, such as electronic mail, +discussion topics, libraries of software, and persistent and growing problems +with civil-liberties issues. Some private boards have as many as +thirty phone-lines and quite sophisticated hardware. And then +there are tiny boards. + +Boards vary in popularity. Some boards are huge and crowded, +where users must claw their way in against a constant busy-signal. +Others are huge and empty--there are few things sadder than a formerly +flourishing board where no one posts any longer, and the dead conversations +of vanished users lie about gathering digital dust. Some boards are tiny +and intimate, their telephone numbers intentionally kept confidential +so that only a small number can log on. + +And some boards are UNDERGROUND. + +Boards can be mysterious entities. The activities of +their users can be hard to differentiate from conspiracy. +Sometimes they ARE conspiracies. Boards have harbored, +or have been accused of harboring, all manner of fringe groups, +and have abetted, or been accused of abetting, every manner +of frowned-upon, sleazy, radical, and criminal activity. +There are Satanist boards. Nazi boards. Pornographic boards. +Pedophile boards. Drug- dealing boards. Anarchist boards. +Communist boards. Gay and Lesbian boards (these exist in great profusion, +many of them quite lively with well-established histories). +Religious cult boards. Evangelical boards. Witchcraft +boards, hippie boards, punk boards, skateboarder boards. +Boards for UFO believers. There may well be boards for +serial killers, airline terrorists and professional assassins. +There is simply no way to tell. Boards spring up, flourish, +and disappear in large numbers, in most every corner of +the developed world. Even apparently innocuous public +boards can, and sometimes do, harbor secret areas known +only to a few. And even on the vast, public, commercial services, +private mail is very private--and quite possibly criminal. + +Boards cover most every topic imaginable and some +that are hard to imagine. They cover a vast spectrum +of social activity. However, all board users do have +something in common: their possession of computers +and phones. Naturally, computers and phones are +primary topics of conversation on almost every board. + +And hackers and phone phreaks, those utter devotees +of computers and phones, live by boards. They swarm by boards. +They are bred by boards. By the late 1980s, phone-phreak groups +and hacker groups, united by boards, had proliferated fantastically. + + +As evidence, here is a list of hacker groups compiled +by the editors of Phrack on August 8, 1988. + + +The Administration. +Advanced Telecommunications, Inc. +ALIAS. +American Tone Travelers. +Anarchy Inc. +Apple Mafia. +The Association. +Atlantic Pirates Guild. + +Bad Ass Mother Fuckers. +Bellcore. +Bell Shock Force. +Black Bag. + +Camorra. +C&M Productions. +Catholics Anonymous. +Chaos Computer Club. +Chief Executive Officers. +Circle Of Death. +Circle Of Deneb. +Club X. +Coalition of Hi-Tech +Pirates. +Coast-To-Coast. +Corrupt Computing. +Cult Of The +Dead Cow. +Custom Retaliations. + +Damage Inc. +D&B Communications. +The Danger Gang. +Dec Hunters. +Digital Gang. +DPAK. + +Eastern Alliance. +The Elite Hackers Guild. +Elite Phreakers and Hackers Club. +The Elite Society Of America. +EPG. +Executives Of Crime. +Extasyy Elite. + +Fargo 4A. +Farmers Of Doom. +The Federation. +Feds R Us. +First Class. +Five O. +Five Star. +Force Hackers. +The 414s. + +Hack-A-Trip. +Hackers Of America. +High Mountain Hackers. +High Society. +The Hitchhikers. + +IBM Syndicate. +The Ice Pirates. +Imperial Warlords. +Inner Circle. +Inner Circle II. +Insanity Inc. +International Computer Underground Bandits. + +Justice League of America. + +Kaos Inc. +Knights Of Shadow. +Knights Of The Round Table. + +League Of Adepts. +Legion Of Doom. +Legion Of Hackers. +Lords Of Chaos. +Lunatic Labs, Unlimited. + +Master Hackers. +MAD! +The Marauders. +MD/PhD. + +Metal Communications, Inc. +MetalliBashers, Inc. +MBI. + +Metro Communications. +Midwest Pirates Guild. + +NASA Elite. +The NATO Association. +Neon Knights. + +Nihilist Order. +Order Of The Rose. +OSS. + +Pacific Pirates Guild. +Phantom Access Associates. + +PHido PHreaks. +The Phirm. +Phlash. +PhoneLine Phantoms. +Phone Phreakers Of America. +Phortune 500. + +Phreak Hack Delinquents. +Phreak Hack Destroyers. + +Phreakers, Hackers, And Laundromat Employees Gang (PHALSE Gang). +Phreaks Against Geeks. +Phreaks Against Phreaks Against Geeks. +Phreaks and Hackers of America. +Phreaks Anonymous World Wide. +Project Genesis. +The Punk Mafia. + +The Racketeers. +Red Dawn Text Files. +Roscoe Gang. + + +SABRE. +Secret Circle of Pirates. +Secret Service. +707 Club. +Shadow Brotherhood. +Sharp Inc. +65C02 Elite. + +Spectral Force. +Star League. +Stowaways. +Strata-Crackers. + + +Team Hackers '86. +Team Hackers '87. + +TeleComputist Newsletter Staff. +Tribunal Of Knowledge. + +Triple Entente. +Turn Over And Die Syndrome (TOADS). + +300 Club. +1200 Club. +2300 Club. +2600 Club. +2601 Club. + +2AF. + +The United Soft WareZ Force. +United Technical Underground. + +Ware Brigade. +The Warelords. +WASP. + +Contemplating this list is an impressive, almost humbling business. +As a cultural artifact, the thing approaches poetry. + +Underground groups--subcultures--can be distinguished +from independent cultures by their habit of referring +constantly to the parent society. Undergrounds by their +nature constantly must maintain a membrane of differentiation. +Funny/distinctive clothes and hair, specialized jargon, specialized +ghettoized areas in cities, different hours of rising, working, +sleeping. . . . The digital underground, which specializes in information, +relies very heavily on language to distinguish itself. As can be seen +from this list, they make heavy use of parody and mockery. +It's revealing to see who they choose to mock. + +First, large corporations. We have the Phortune 500, +The Chief Executive Officers, Bellcore, IBM Syndicate, +SABRE (a computerized reservation service maintained +by airlines). The common use of "Inc." is telling-- +none of these groups are actual corporations, +but take clear delight in mimicking them. + +Second, governments and police. NASA Elite, NATO Association. +"Feds R Us" and "Secret Service" are fine bits of fleering boldness. +OSS--the Office of Strategic Services was the forerunner of the CIA. + +Third, criminals. Using stigmatizing pejoratives as a perverse +badge of honor is a time-honored tactic for subcultures: +punks, gangs, delinquents, mafias, pirates, bandits, racketeers. + +Specialized orthography, especially the use of "ph" for "f" +and "z" for the plural "s," are instant recognition symbols. +So is the use of the numeral "0" for the letter "O" +--computer-software orthography generally features a +slash through the zero, making the distinction obvious. + +Some terms are poetically descriptive of computer intrusion: +the Stowaways, the Hitchhikers, the PhoneLine Phantoms, Coast-to-Coast. +Others are simple bravado and vainglorious puffery. +(Note the insistent use of the terms "elite" and "master.") +Some terms are blasphemous, some obscene, others merely cryptic-- +anything to puzzle, offend, confuse, and keep the straights at bay. + +Many hacker groups further re-encrypt their names +by the use of acronyms: United Technical Underground +becomes UTU, Farmers of Doom become FoD, the United SoftWareZ +Force becomes, at its own insistence, "TuSwF," and woe to the +ignorant rodent who capitalizes the wrong letters. + +It should be further recognized that the members of these groups +are themselves pseudonymous. If you did, in fact, run across +the "PhoneLine Phantoms," you would find them to consist of +"Carrier Culprit," "The Executioner," "Black Majik," +"Egyptian Lover," "Solid State," and "Mr Icom." +"Carrier Culprit" will likely be referred to by his friends +as "CC," as in, "I got these dialups from CC of PLP." + +It's quite possible that this entire list refers to as +few as a thousand people. It is not a complete list +of underground groups--there has never been such a list, +and there never will be. Groups rise, flourish, decline, +share membership, maintain a cloud of wannabes and +casual hangers-on. People pass in and out, are ostracized, +get bored, are busted by police, or are cornered by telco +security and presented with huge bills. Many "underground +groups" are software pirates, "warez d00dz," who might break +copy protection and pirate programs, but likely wouldn't dare +to intrude on a computer-system. + +It is hard to estimate the true population of the digital +underground. There is constant turnover. Most hackers +start young, come and go, then drop out at age 22-- +the age of college graduation. And a large majority +of "hackers" access pirate boards, adopt a handle, +swipe software and perhaps abuse a phone-code or two, +while never actually joining the elite. + +Some professional informants, who make it their business +to retail knowledge of the underground to paymasters in private +corporate security, have estimated the hacker population +at as high as fifty thousand. This is likely highly inflated, +unless one counts every single teenage software pirate +and petty phone-booth thief. My best guess is about 5,000 people. +Of these, I would guess that as few as a hundred are truly "elite" +--active computer intruders, skilled enough to penetrate +sophisticated systems and truly to worry corporate security +and law enforcement. + +Another interesting speculation is whether this group +is growing or not. Young teenage hackers are often +convinced that hackers exist in vast swarms and will soon +dominate the cybernetic universe. Older and wiser +veterans, perhaps as wizened as 24 or 25 years old, +are convinced that the glory days are long gone, that the cops +have the underground's number now, and that kids these days +are dirt-stupid and just want to play Nintendo. + +My own assessment is that computer intrusion, as a non-profit act +of intellectual exploration and mastery, is in slow decline, +at least in the United States; but that electronic fraud, +especially telecommunication crime, is growing by leaps and bounds. + +One might find a useful parallel to the digital underground +in the drug underground. There was a time, now much-obscured +by historical revisionism, when Bohemians freely shared joints +at concerts, and hip, small-scale marijuana dealers might +turn people on just for the sake of enjoying a long stoned conversation +about the Doors and Allen Ginsberg. Now drugs are increasingly verboten, +except in a high-stakes, highly-criminal world of highly addictive drugs. +Over years of disenchantment and police harassment, a vaguely ideological, +free-wheeling drug underground has relinquished the business of drug-dealing +to a far more savage criminal hard-core. This is not a pleasant prospect +to contemplate, but the analogy is fairly compelling. + +What does an underground board look like? What distinguishes +it from a standard board? It isn't necessarily the conversation-- +hackers often talk about common board topics, such as hardware, software, +sex, science fiction, current events, politics, movies, personal gossip. +Underground boards can best be distinguished by their files, or "philes," +pre-composed texts which teach the techniques and ethos of the underground. +These are prized reservoirs of forbidden knowledge. Some are anonymous, +but most proudly bear the handle of the "hacker" who has created them, +and his group affiliation, if he has one. + +Here is a partial table-of-contents of philes from an underground board, +somewhere in the heart of middle America, circa 1991. The descriptions +are mostly self-explanatory. + + +BANKAMER.ZIP 5406 06-11-91 Hacking Bank America +CHHACK.ZIP 4481 06-11-91 Chilton Hacking +CITIBANK.ZIP 4118 06-11-91 Hacking Citibank +CREDIMTC.ZIP 3241 06-11-91 Hacking Mtc Credit Company +DIGEST.ZIP 5159 06-11-91 Hackers Digest +HACK.ZIP 14031 06-11-91 How To Hack +HACKBAS.ZIP 5073 06-11-91 Basics Of Hacking +HACKDICT.ZIP 42774 06-11-91 Hackers Dictionary +HACKER.ZIP 57938 06-11-91 Hacker Info +HACKERME.ZIP 3148 06-11-91 Hackers Manual +HACKHAND.ZIP 4814 06-11-91 Hackers Handbook +HACKTHES.ZIP 48290 06-11-91 Hackers Thesis +HACKVMS.ZIP 4696 06-11-91 Hacking Vms Systems +MCDON.ZIP 3830 06-11-91 Hacking Macdonalds (Home Of The Archs) +P500UNIX.ZIP 15525 06-11-91 Phortune 500 Guide To Unix +RADHACK.ZIP 8411 06-11-91 Radio Hacking +TAOTRASH.DOC 4096 12-25-89 Suggestions For Trashing +TECHHACK.ZIP 5063 06-11-91 Technical Hacking + + +The files above are do-it-yourself manuals about computer intrusion. +The above is only a small section of a much larger library of hacking +and phreaking techniques and history. We now move into a different +and perhaps surprising area. + ++------------+ + |Anarchy| ++------------+ + +ANARC.ZIP 3641 06-11-91 Anarchy Files +ANARCHST.ZIP 63703 06-11-91 Anarchist Book +ANARCHY.ZIP 2076 06-11-91 Anarchy At Home +ANARCHY3.ZIP 6982 06-11-91 Anarchy No 3 +ANARCTOY.ZIP 2361 06-11-91 Anarchy Toys +ANTIMODM.ZIP 2877 06-11-91 Anti-modem Weapons +ATOM.ZIP 4494 06-11-91 How To Make An Atom Bomb +BARBITUA.ZIP 3982 06-11-91 Barbiturate Formula +BLCKPWDR.ZIP 2810 06-11-91 Black Powder Formulas +BOMB.ZIP 3765 06-11-91 How To Make Bombs +BOOM.ZIP 2036 06-11-91 Things That Go Boom +CHLORINE.ZIP 1926 06-11-91 Chlorine Bomb +COOKBOOK.ZIP 1500 06-11-91 Anarchy Cook Book +DESTROY.ZIP 3947 06-11-91 Destroy Stuff +DUSTBOMB.ZIP 2576 06-11-91 Dust Bomb +ELECTERR.ZIP 3230 06-11-91 Electronic Terror +EXPLOS1.ZIP 2598 06-11-91 Explosives 1 +EXPLOSIV.ZIP 18051 06-11-91 More Explosives +EZSTEAL.ZIP 4521 06-11-91 Ez-stealing +FLAME.ZIP 2240 06-11-91 Flame Thrower +FLASHLT.ZIP 2533 06-11-91 Flashlight Bomb +FMBUG.ZIP 2906 06-11-91 How To Make An Fm Bug +OMEEXPL.ZIP 2139 06-11-91 Home Explosives +HOW2BRK.ZIP 3332 06-11-91 How To Break In +LETTER.ZIP 2990 06-11-91 Letter Bomb +LOCK.ZIP 2199 06-11-91 How To Pick Locks +MRSHIN.ZIP 3991 06-11-91 Briefcase Locks +NAPALM.ZIP 3563 06-11-91 Napalm At Home +NITRO.ZIP 3158 06-11-91 Fun With Nitro +PARAMIL.ZIP 2962 06-11-91 Paramilitary Info +PICKING.ZIP 3398 06-11-91 Picking Locks +PIPEBOMB.ZIP 2137 06-11-91 Pipe Bomb +POTASS.ZIP 3987 06-11-91 Formulas With Potassium +PRANK.TXT 11074 08-03-90 More Pranks To Pull On Idiots! +REVENGE.ZIP 4447 06-11-91 Revenge Tactics +ROCKET.ZIP 2590 06-11-91 Rockets For Fun +SMUGGLE.ZIP 3385 06-11-91 How To Smuggle + +HOLY COW! The damned thing is full of stuff about bombs! + +What are we to make of this? + +First, it should be acknowledged that spreading +knowledge about demolitions to teenagers is a highly and +deliberately antisocial act. It is not, however, illegal. + +Second, it should be recognized that most of these +philes were in fact WRITTEN by teenagers. Most adult +American males who can remember their teenage years +will recognize that the notion of building a flamethrower +in your garage is an incredibly neat-o idea. ACTUALLY, +building a flamethrower in your garage, however, is +fraught with discouraging difficulty. Stuffing gunpowder +into a booby-trapped flashlight, so as to blow the arm off +your high-school vice-principal, can be a thing of dark +beauty to contemplate. Actually committing assault by +explosives will earn you the sustained attention of the +federal Bureau of Alcohol, Tobacco and Firearms. + +Some people, however, will actually try these plans. +A determinedly murderous American teenager can probably +buy or steal a handgun far more easily than he can brew +fake "napalm" in the kitchen sink. Nevertheless, +if temptation is spread before people, a certain number +will succumb, and a small minority will actually attempt +these stunts. A large minority of that small minority +will either fail or, quite likely, maim themselves, +since these "philes" have not been checked for accuracy, +are not the product of professional experience, +and are often highly fanciful. But the gloating menace +of these philes is not to be entirely dismissed. + +Hackers may not be "serious" about bombing; if they were, +we would hear far more about exploding flashlights, homemade bazookas, +and gym teachers poisoned by chlorine and potassium. +However, hackers are VERY serious about forbidden knowledge. +They are possessed not merely by curiosity, but by +a positive LUST TO KNOW. The desire to know what +others don't is scarcely new. But the INTENSITY +of this desire, as manifested by these young technophilic +denizens of the Information Age, may in fact BE new, +and may represent some basic shift in social values-- +a harbinger of what the world may come to, as society +lays more and more value on the possession, +assimilation and retailing of INFORMATION +as a basic commodity of daily life. + +There have always been young men with obsessive interests +in these topics. Never before, however, have they been able +to network so extensively and easily, and to propagandize +their interests with impunity to random passers-by. +High-school teachers will recognize that there's always +one in a crowd, but when the one in a crowd escapes control +by jumping into the phone-lines, and becomes a hundred such kids +all together on a board, then trouble is brewing visibly. +The urge of authority to DO SOMETHING, even something drastic, +is hard to resist. And in 1990, authority did something. +In fact authority did a great deal. + +# + +The process by which boards create hackers goes something +like this. A youngster becomes interested in computers-- +usually, computer games. He hears from friends that +"bulletin boards" exist where games can be obtained for free. +(Many computer games are "freeware," not copyrighted-- +invented simply for the love of it and given away to the public; +some of these games are quite good.) He bugs his parents for a modem, +or quite often, uses his parents' modem. + +The world of boards suddenly opens up. Computer games +can be quite expensive, real budget-breakers for a kid, +but pirated games, stripped of copy protection, are cheap or free. +They are also illegal, but it is very rare, almost unheard of, +for a small-scale software pirate to be prosecuted. +Once "cracked" of its copy protection, the program, +being digital data, becomes infinitely reproducible. +Even the instructions to the game, any manuals that accompany it, +can be reproduced as text files, or photocopied from legitimate sets. +Other users on boards can give many useful hints in game-playing tactics. +And a youngster with an infinite supply of free computer games can +certainly cut quite a swath among his modem-less friends. + +And boards are pseudonymous. No one need know that you're +fourteen years old--with a little practice at subterfuge, +you can talk to adults about adult things, and be accepted +and taken seriously! You can even pretend to be a girl, +or an old man, or anybody you can imagine. If you find this +kind of deception gratifying, there is ample opportunity +to hone your ability on boards. + +But local boards can grow stale. And almost every board maintains +a list of phone-numbers to other boards, some in distant, tempting, +exotic locales. Who knows what they're up to, in Oregon or Alaska +or Florida or California? It's very easy to find out--just order +the modem to call through its software--nothing to this, just typing +on a keyboard, the same thing you would do for most any computer game. +The machine reacts swiftly and in a few seconds you are talking to +a bunch of interesting people on another seaboard. + +And yet the BILLS for this trivial action can be staggering! +Just by going tippety-tap with your fingers, you may have +saddled your parents with four hundred bucks in long-distance charges, +and gotten chewed out but good. That hardly seems fair. + +How horrifying to have made friends in another state +and to be deprived of their company--and their software-- +just because telephone companies demand absurd amounts of money! +How painful, to be restricted to boards in one's own AREA CODE-- +what the heck is an "area code" anyway, and what makes it so special? +A few grumbles, complaints, and innocent questions of this sort +will often elicit a sympathetic reply from another board user-- +someone with some stolen codes to hand. You dither a while, +knowing this isn't quite right, then you make up your mind +to try them anyhow--AND THEY WORK! Suddenly you're doing something +even your parents can't do. Six months ago you were just some kid--now, +you're the Crimson Flash of Area Code 512! You're bad--you're nationwide! + +Maybe you'll stop at a few abused codes. Maybe you'll decide that +boards aren't all that interesting after all, that it's wrong, +not worth the risk --but maybe you won't. The next step +is to pick up your own repeat-dialling program-- +to learn to generate your own stolen codes. +(This was dead easy five years ago, much harder +to get away with nowadays, but not yet impossible.) +And these dialling programs are not complex or intimidating-- +some are as small as twenty lines of software. + +Now, you too can share codes. You can trade codes to learn +other techniques. If you're smart enough to catch on, +and obsessive enough to want to bother, and ruthless enough +to start seriously bending rules, then you'll get better, fast. +You start to develop a rep. You move up to a heavier class +of board--a board with a bad attitude, the kind of board +that naive dopes like your classmates and your former self +have never even heard of! You pick up the jargon of phreaking +and hacking from the board. You read a few of those anarchy philes-- +and man, you never realized you could be a real OUTLAW without +ever leaving your bedroom. + +You still play other computer games, but now you have a new +and bigger game. This one will bring you a different kind of status +than destroying even eight zillion lousy space invaders. + +Hacking is perceived by hackers as a "game." This is +not an entirely unreasonable or sociopathic perception. +You can win or lose at hacking, succeed or fail, +but it never feels "real." It's not simply that +imaginative youngsters sometimes have a hard time +telling "make-believe" from "real life." Cyberspace +is NOT REAL! "Real" things are physical objects +like trees and shoes and cars. Hacking takes place +on a screen. Words aren't physical, numbers +(even telephone numbers and credit card numbers) +aren't physical. Sticks and stones may break my bones, +but data will never hurt me. Computers SIMULATE reality, +like computer games that simulate tank battles or dogfights +or spaceships. Simulations are just make-believe, +and the stuff in computers is NOT REAL. + +Consider this: if "hacking" is supposed to be so serious and +real-life and dangerous, then how come NINE-YEAR-OLD KIDS have +computers and modems? You wouldn't give a nine year old his own car, +or his own rifle, or his own chainsaw--those things are "real." + +People underground are perfectly aware that the "game" +is frowned upon by the powers that be. Word gets around +about busts in the underground. Publicizing busts is one +of the primary functions of pirate boards, but they also +promulgate an attitude about them, and their own idiosyncratic +ideas of justice. The users of underground boards won't complain +if some guy is busted for crashing systems, spreading viruses, +or stealing money by wire-fraud. They may shake their heads +with a sneaky grin, but they won't openly defend these practices. +But when a kid is charged with some theoretical amount of theft: +$233,846.14, for instance, because he sneaked into a computer +and copied something, and kept it in his house on a floppy disk-- +this is regarded as a sign of near-insanity from prosecutors, +a sign that they've drastically mistaken the immaterial game +of computing for their real and boring everyday world +of fatcat corporate money. + +It's as if big companies and their suck-up lawyers +think that computing belongs to them, and they can +retail it with price stickers, as if it were boxes +of laundry soap! But pricing "information" is like +trying to price air or price dreams. Well, anybody +on a pirate board knows that computing can be, +and ought to be, FREE. Pirate boards are little +independent worlds in cyberspace, and they don't belong +to anybody but the underground. Underground boards +aren't "brought to you by Procter & Gamble." + +To log on to an underground board can mean to +experience liberation, to enter a world where, +for once, money isn't everything and adults +don't have all the answers. + +Let's sample another vivid hacker manifesto. Here are +some excerpts from "The Conscience of a Hacker," by "The Mentor," +from Phrack Volume One, Issue 7, Phile 3. + +"I made a discovery today. I found a computer. +Wait a second, this is cool. It does what I want it to. +If it makes a mistake, it's because I screwed it up. +Not because it doesn't like me. (. . .) +"And then it happened. . .a door opened to a world. . . +rushing through the phone line like heroin through an +addict's veins, an electronic pulse is sent out, +a refuge from day-to-day incompetencies is sought. . . +a board is found. `This is it. . .this is where I belong. . .' +"I know everyone here. . .even if I've never met them, +never talked to them, may never hear from them again. . . +I know you all. . . (. . .) + +"This is our world now. . .the world of the electron +and the switch, the beauty of the baud. We make use of a +service already existing without paying for what could be +dirt-cheap if it wasn't run by profiteering gluttons, and you +call us criminals. We explore. . .and you call us criminals. +We seek after knowledge. . .and you call us criminals. +We exist without skin color, without nationality, +without religious bias. . .and you call us criminals. +You build atomic bombs, you wage wars, you murder, +cheat and lie to us and try to make us believe that +it's for our own good, yet we're the criminals. + +"Yes, I am a criminal. My crime is that of curiosity. +My crime is that of judging people by what they say and think, +not what they look like. My crime is that of outsmarting you, +something that you will never forgive me for." + +# + +There have been underground boards almost as long +as there have been boards. One of the first was 8BBS, +which became a stronghold of the West Coast phone-phreak elite. +After going on-line in March 1980, 8BBS sponsored "Susan Thunder," +and "Tuc," and, most notoriously, "the Condor." "The Condor" +bore the singular distinction of becoming the most vilified +American phreak and hacker ever. Angry underground associates, +fed up with Condor's peevish behavior, turned him in to police, +along with a heaping double-helping of outrageous hacker legendry. +As a result, Condor was kept in solitary confinement for seven months, +for fear that he might start World War Three by triggering missile silos +from the prison payphone. (Having served his time, Condor is now +walking around loose; WWIII has thus far conspicuously failed to occur.) + +The sysop of 8BBS was an ardent free-speech enthusiast +who simply felt that ANY attempt to restrict the expression +of his users was unconstitutional and immoral. +Swarms of the technically curious entered 8BBS +and emerged as phreaks and hackers, until, in 1982, +a friendly 8BBS alumnus passed the sysop a new modem +which had been purchased by credit-card fraud. +Police took this opportunity to seize the entire board +and remove what they considered an attractive nuisance. + +Plovernet was a powerful East Coast pirate board +that operated in both New York and Florida. +Owned and operated by teenage hacker "Quasi Moto," +Plovernet attracted five hundred eager users in 1983. +"Emmanuel Goldstein" was one-time co-sysop of Plovernet, +along with "Lex Luthor," founder of the "Legion of Doom" group. +Plovernet bore the signal honor of being the original home +of the "Legion of Doom," about which the reader will be hearing +a great deal, soon. + +"Pirate-80," or "P-80," run by a sysop known as "Scan-Man," +got into the game very early in Charleston, and continued +steadily for years. P-80 flourished so flagrantly that +even its most hardened users became nervous, and some +slanderously speculated that "Scan Man" must have ties +to corporate security, a charge he vigorously denied. + +"414 Private" was the home board for the first GROUP +to attract conspicuous trouble, the teenage "414 Gang," +whose intrusions into Sloan-Kettering Cancer Center and +Los Alamos military computers were to be a nine-days-wonder in 1982. + +At about this time, the first software piracy boards +began to open up, trading cracked games for the Atari 800 +and the Commodore C64. Naturally these boards were +heavily frequented by teenagers. And with the 1983 +release of the hacker-thriller movie War Games, +the scene exploded. It seemed that every kid +in America had demanded and gotten a modem for Christmas. +Most of these dabbler wannabes put their modems in the attic +after a few weeks, and most of the remainder minded their +P's and Q's and stayed well out of hot water. But some +stubborn and talented diehards had this hacker kid in +War Games figured for a happening dude. They simply +could not rest until they had contacted the underground-- +or, failing that, created their own. + +In the mid-80s, underground boards sprang up like digital fungi. +ShadowSpawn Elite. Sherwood Forest I, II, and III. +Digital Logic Data Service in Florida, sysoped by no less +a man than "Digital Logic" himself; Lex Luthor of the +Legion of Doom was prominent on this board, since it +was in his area code. Lex's own board, "Legion of Doom," +started in 1984. The Neon Knights ran a network of Apple- +hacker boards: Neon Knights North, South, East and West. +Free World II was run by "Major Havoc." Lunatic Labs +is still in operation as of this writing. Dr. Ripco +in Chicago, an anything-goes anarchist board with an +extensive and raucous history, was seized by Secret Service +agents in 1990 on Sundevil day, but up again almost immediately, +with new machines and scarcely diminished vigor. + +The St. Louis scene was not to rank with major centers +of American hacking such as New York and L.A. But St. +Louis did rejoice in possession of "Knight Lightning" +and "Taran King," two of the foremost JOURNALISTS native +to the underground. Missouri boards like Metal Shop, +Metal Shop Private, Metal Shop Brewery, may not have +been the heaviest boards around in terms of illicit +expertise. But they became boards where hackers could +exchange social gossip and try to figure out what the +heck was going on nationally--and internationally. +Gossip from Metal Shop was put into the form of news files, +then assembled into a general electronic publication, +Phrack, a portmanteau title coined from "phreak" and "hack." +The Phrack editors were as obsessively curious about other +hackers as hackers were about machines. + +Phrack, being free of charge and lively reading, began +to circulate throughout the underground. As Taran King +and Knight Lightning left high school for college, +Phrack began to appear on mainframe machines linked to BITNET, +and, through BITNET to the "Internet," that loose but +extremely potent not-for-profit network where academic, +governmental and corporate machines trade data through +the UNIX TCP/IP protocol. (The "Internet Worm" of +November 2-3,1988, created by Cornell grad student Robert Morris, +was to be the largest and best-publicized computer-intrusion scandal +to date. Morris claimed that his ingenious "worm" program was meant +to harmlessly explore the Internet, but due to bad programming, +the Worm replicated out of control and crashed some six thousand +Internet computers. Smaller-scale and less ambitious Internet hacking +was a standard for the underground elite.) + +Most any underground board not hopelessly lame and out-of-it +would feature a complete run of Phrack--and, possibly, +the lesser-known standards of the underground: +the Legion of Doom Technical Journal, the obscene +and raucous Cult of the Dead Cow files, P/HUN magazine, +Pirate, the Syndicate Reports, and perhaps the highly +anarcho-political Activist Times Incorporated. + +Possession of Phrack on one's board was prima facie +evidence of a bad attitude. Phrack was seemingly everywhere, +aiding, abetting, and spreading the underground ethos. +And this did not escape the attention of corporate security +or the police. + +We now come to the touchy subject of police and boards. +Police, do, in fact, own boards. In 1989, there were +police-sponsored boards in California, Colorado, Florida, +Georgia, Idaho, Michigan, Missouri, Texas, and Virginia: +boards such as "Crime Bytes," "Crimestoppers," "All Points" +and "Bullet-N-Board." Police officers, as private computer +enthusiasts, ran their own boards in Arizona, California, +Colorado, Connecticut, Florida, Missouri, Maryland, +New Mexico, North Carolina, Ohio, Tennessee and Texas. +Police boards have often proved helpful in community relations. +Sometimes crimes are reported on police boards. + +Sometimes crimes are COMMITTED on police boards. +This has sometimes happened by accident, as naive hackers +blunder onto police boards and blithely begin offering telephone codes. +Far more often, however, it occurs through the now almost-traditional +use of "sting boards." The first police sting-boards were established +in 1985: "Underground Tunnel" in Austin, Texas, whose sysop +Sgt. Robert Ansley called himself "Pluto"--"The Phone Company" +in Phoenix, Arizona, run by Ken MacLeod of the Maricopa County +Sheriff's office--and Sgt. Dan Pasquale's board in Fremont, California. +Sysops posed as hackers, and swiftly garnered coteries of ardent users, +who posted codes and loaded pirate software with abandon, +and came to a sticky end. + +Sting boards, like other boards, are cheap to operate, +very cheap by the standards of undercover police operations. +Once accepted by the local underground, sysops will likely be +invited into other pirate boards, where they can compile more dossiers. +And when the sting is announced and the worst offenders arrested, +the publicity is generally gratifying. The resultant paranoia +in the underground--perhaps more justly described as a "deterrence effect"-- +tends to quell local lawbreaking for quite a while. + +Obviously police do not have to beat the underbrush for hackers. +On the contrary, they can go trolling for them. Those caught +can be grilled. Some become useful informants. They can lead +the way to pirate boards all across the country. + +And boards all across the country showed the sticky +fingerprints of Phrack, and of that loudest and most +flagrant of all underground groups, the "Legion of Doom." + +The term "Legion of Doom" came from comic books. The Legion of Doom, +a conspiracy of costumed super- villains headed by the chrome-domed +criminal ultra- mastermind Lex Luthor, gave Superman a lot of four-color +graphic trouble for a number of decades. Of course, Superman, +that exemplar of Truth, Justice, and the American Way, +always won in the long run. This didn't matter to the hacker Doomsters-- +"Legion of Doom" was not some thunderous and evil Satanic reference, +it was not meant to be taken seriously. "Legion of Doom" came +from funny-books and was supposed to be funny. + +"Legion of Doom" did have a good mouthfilling ring to it, though. +It sounded really cool. Other groups, such as the "Farmers of Doom," +closely allied to LoD, recognized this grandiloquent quality, +and made fun of it. There was even a hacker group called +"Justice League of America," named after Superman's club +of true-blue crimefighting superheros. + +But they didn't last; the Legion did. + +The original Legion of Doom, hanging out on Quasi Moto's Plovernet board, +were phone phreaks. They weren't much into computers. "Lex Luthor" himself +(who was under eighteen when he formed the Legion) was a COSMOS expert, +COSMOS being the "Central System for Mainframe Operations," +a telco internal computer network. Lex would eventually become +quite a dab hand at breaking into IBM mainframes, but although +everyone liked Lex and admired his attitude, he was not considered +a truly accomplished computer intruder. Nor was he the "mastermind" +of the Legion of Doom--LoD were never big on formal leadership. +As a regular on Plovernet and sysop of his "Legion of Doom BBS," +Lex was the Legion's cheerleader and recruiting officer. + +Legion of Doom began on the ruins of an earlier phreak group, +The Knights of Shadow. Later, LoD was to subsume the personnel +of the hacker group "Tribunal of Knowledge." People came and went +constantly in LoD; groups split up or formed offshoots. + +Early on, the LoD phreaks befriended a few computer-intrusion +enthusiasts, who became the associated "Legion of Hackers." +Then the two groups conflated into the "Legion of Doom/Hackers," +or LoD/H. When the original "hacker" wing, Messrs. "Compu-Phreak" +and "Phucked Agent 04," found other matters to occupy their time, +the extra "/H" slowly atrophied out of the name; but by this time +the phreak wing, Messrs. Lex Luthor, "Blue Archer," "Gary Seven," +"Kerrang Khan," "Master of Impact," "Silver Spy," "The Marauder," +and "The Videosmith," had picked up a plethora of intrusion +expertise and had become a force to be reckoned with. + +LoD members seemed to have an instinctive understanding +that the way to real power in the underground lay through +covert publicity. LoD were flagrant. Not only was it one +of the earliest groups, but the members took pains to widely +distribute their illicit knowledge. Some LoD members, +like "The Mentor," were close to evangelical about it. +Legion of Doom Technical Journal began to show up on boards +throughout the underground. + +LoD Technical Journal was named in cruel parody +of the ancient and honored AT&T Technical Journal. +The material in these two publications was quite similar-- +much of it, adopted from public journals and discussions +in the telco community. And yet, the predatory attitude +of LoD made even its most innocuous data seem deeply sinister; +an outrage; a clear and present danger. + +To see why this should be, let's consider the following +(invented) paragraphs, as a kind of thought experiment. + +(A) "W. Fred Brown, AT&T Vice President for +Advanced Technical Development, testified May 8 +at a Washington hearing of the National Telecommunications +and Information Administration (NTIA), regarding +Bellcore's GARDEN project. GARDEN (Generalized +Automatic Remote Distributed Electronic Network) is a +telephone-switch programming tool that makes it possible +to develop new telecom services, including hold-on-hold +and customized message transfers, from any keypad terminal, +within seconds. The GARDEN prototype combines centrex +lines with a minicomputer using UNIX operating system software." + +(B) "Crimson Flash 512 of the Centrex Mobsters reports: +D00dz, you wouldn't believe this GARDEN bullshit Bellcore's +just come up with! Now you don't even need a lousy Commodore +to reprogram a switch--just log on to GARDEN as a technician, +and you can reprogram switches right off the keypad in any +public phone booth! You can give yourself hold-on-hold +and customized message transfers, and best of all, +the thing is run off (notoriously insecure) centrex lines +using--get this--standard UNIX software! Ha ha ha ha!" + +Message (A), couched in typical techno-bureaucratese, +appears tedious and almost unreadable. (A) scarcely seems +threatening or menacing. Message (B), on the other hand, +is a dreadful thing, prima facie evidence of a dire conspiracy, +definitely not the kind of thing you want your teenager reading. + +The INFORMATION, however, is identical. It is PUBLIC +information, presented before the federal government in +an open hearing. It is not "secret." It is not "proprietary." +It is not even "confidential." On the contrary, the +development of advanced software systems is a matter +of great public pride to Bellcore. + +However, when Bellcore publicly announces a project of this kind, +it expects a certain attitude from the public--something along +the lines of GOSH WOW, YOU GUYS ARE GREAT, KEEP THAT UP, WHATEVER IT IS-- +certainly not cruel mimickry, one-upmanship and outrageous speculations +about possible security holes. + +Now put yourself in the place of a policeman confronted by +an outraged parent, or telco official, with a copy of Version (B). +This well-meaning citizen, to his horror, has discovered +a local bulletin-board carrying outrageous stuff like (B), +which his son is examining with a deep and unhealthy interest. +If (B) were printed in a book or magazine, you, as an American +law enforcement officer, would know that it would take +a hell of a lot of trouble to do anything about it; +but it doesn't take technical genius to recognize that +if there's a computer in your area harboring stuff like (B), +there's going to be trouble. + +In fact, if you ask around, any computer-literate cop +will tell you straight out that boards with stuff like (B) +are the SOURCE of trouble. And the WORST source of trouble +on boards are the ringleaders inventing and spreading stuff like (B). +If it weren't for these jokers, there wouldn't BE any trouble. + +And Legion of Doom were on boards like nobody else. +Plovernet. The Legion of Doom Board. The Farmers of Doom Board. +Metal Shop. OSUNY. Blottoland. Private Sector. Atlantis. +Digital Logic. Hell Phrozen Over. + +LoD members also ran their own boards. "Silver Spy" started +his own board, "Catch-22," considered one of the heaviest around. +So did "Mentor," with his "Phoenix Project." When they didn't run boards +themselves, they showed up on other people's boards, to brag, boast, +and strut. And where they themselves didn't go, their philes went, +carrying evil knowledge and an even more evil attitude. + +As early as 1986, the police were under the vague impression +that EVERYONE in the underground was Legion of Doom. +LoD was never that large--considerably smaller than either +"Metal Communications" or "The Administration," for instance-- +but LoD got tremendous press. Especially in Phrack, +which at times read like an LoD fan magazine; and Phrack +was everywhere, especially in the offices of telco security. +You couldn't GET busted as a phone phreak, a hacker, +or even a lousy codes kid or warez dood, without the cops +asking if you were LoD. + +This was a difficult charge to deny, as LoD never +distributed membership badges or laminated ID cards. +If they had, they would likely have died out quickly, +for turnover in their membership was considerable. +LoD was less a high-tech street-gang than an ongoing +state-of-mind. LoD was the Gang That Refused to Die. +By 1990, LoD had RULED for ten years, and it seemed WEIRD +to police that they were continually busting people who were +only sixteen years old. All these teenage small-timers +were pleading the tiresome hacker litany of "just curious, +no criminal intent." Somewhere at the center of this +conspiracy there had to be some serious adult masterminds, +not this seemingly endless supply of myopic suburban +white kids with high SATs and funny haircuts. + +There was no question that most any American hacker +arrested would "know" LoD. They knew the handles +of contributors to LoD Tech Journal, and were likely +to have learned their craft through LoD boards and LoD activism. +But they'd never met anyone from LoD. Even some of the +rotating cadre who were actually and formally "in LoD" +knew one another only by board-mail and pseudonyms. +This was a highly unconventional profile for a criminal conspiracy. +Computer networking, and the rapid evolution of the digital underground, +made the situation very diffuse and confusing. + +Furthermore, a big reputation in the digital underground +did not coincide with one's willingness to commit "crimes." +Instead, reputation was based on cleverness and technical mastery. +As a result, it often seemed that the HEAVIER the hackers were, +the LESS likely they were to have committed any kind of common, +easily prosecutable crime. There were some hackers who could really steal. +And there were hackers who could really hack. But the two groups didn't seem +to overlap much, if at all. For instance, most people in the underground +looked up to "Emmanuel Goldstein" of 2600 as a hacker demigod. +But Goldstein's publishing activities were entirely legal-- +Goldstein just printed dodgy stuff and talked about politics, +he didn't even hack. When you came right down to it, +Goldstein spent half his time complaining that computer security +WASN'T STRONG ENOUGH and ought to be drastically improved +across the board! + +Truly heavy-duty hackers, those with serious technical skills +who had earned the respect of the underground, never stole money +or abused credit cards. Sometimes they might abuse phone-codes-- +but often, they seemed to get all the free phone-time they wanted +without leaving a trace of any kind. + +The best hackers, the most powerful and technically accomplished, +were not professional fraudsters. They raided computers habitually, +but wouldn't alter anything, or damage anything. They didn't even steal +computer equipment--most had day-jobs messing with hardware, +and could get all the cheap secondhand equipment they wanted. +The hottest hackers, unlike the teenage wannabes, weren't snobs +about fancy or expensive hardware. Their machines tended to be +raw second-hand digital hot-rods full of custom add-ons that +they'd cobbled together out of chickenwire, memory chips and spit. +Some were adults, computer software writers and consultants by trade, +and making quite good livings at it. Some of them ACTUALLY WORKED +FOR THE PHONE COMPANY--and for those, the "hackers" actually found +under the skirts of Ma Bell, there would be little mercy in 1990. + +It has long been an article of faith in the +underground that the "best" hackers never get caught. +They're far too smart, supposedly. They never get caught +because they never boast, brag, or strut. These demigods +may read underground boards (with a condescending smile), +but they never say anything there. The "best" hackers, +according to legend, are adult computer professionals, +such as mainframe system administrators, who already know +the ins and outs of their particular brand of security. +Even the "best" hacker can't break in to just any computer at random: +the knowledge of security holes is too specialized, varying widely +with different software and hardware. But if people are employed to run, +say, a UNIX mainframe or a VAX/VMS machine, then they tend to learn +security from the inside out. Armed with this knowledge, +they can look into most anybody else's UNIX or VMS +without much trouble or risk, if they want to. +And, according to hacker legend, of course they want to, +so of course they do. They just don't make a big deal +of what they've done. So nobody ever finds out. + +It is also an article of faith in the underground that +professional telco people "phreak" like crazed weasels. +OF COURSE they spy on Madonna's phone calls--I mean, +WOULDN'T YOU? Of course they give themselves free long- +distance--why the hell should THEY pay, they're running +the whole shebang! + +It has, as a third matter, long been an article of faith +that any hacker caught can escape serious punishment if +he confesses HOW HE DID IT. Hackers seem to believe +that governmental agencies and large corporations are +blundering about in cyberspace like eyeless jellyfish +or cave salamanders. They feel that these large +but pathetically stupid organizations will proffer up +genuine gratitude, and perhaps even a security post +and a big salary, to the hot-shot intruder who will deign +to reveal to them the supreme genius of his modus operandi. + +In the case of longtime LoD member "Control-C," +this actually happened, more or less. Control-C had led +Michigan Bell a merry chase, and when captured in 1987, +he turned out to be a bright and apparently physically +harmless young fanatic, fascinated by phones. There was +no chance in hell that Control-C would actually repay the +enormous and largely theoretical sums in long-distance +service that he had accumulated from Michigan Bell. +He could always be indicted for fraud or computer-intrusion, +but there seemed little real point in this--he hadn't +physically damaged any computer. He'd just plead guilty, +and he'd likely get the usual slap-on-the-wrist, +and in the meantime it would be a big hassle for Michigan Bell +just to bring up the case. But if kept on the payroll, +he might at least keep his fellow hackers at bay. + +There were uses for him. For instance, a contrite +Control-C was featured on Michigan Bell internal posters, +sternly warning employees to shred their trash. +He'd always gotten most of his best inside info from +"trashing"--raiding telco dumpsters, for useful data +indiscreetly thrown away. He signed these posters, too. +Control-C had become something like a Michigan Bell mascot. +And in fact, Control-C DID keep other hackers at bay. +Little hackers were quite scared of Control-C and his +heavy-duty Legion of Doom friends. And big hackers WERE +his friends and didn't want to screw up his cushy situation. + +No matter what one might say of LoD, they did stick together. +When "Wasp," an apparently genuinely malicious New York hacker, +began crashing Bellcore machines, Control-C received swift volunteer +help from "the Mentor" and the Georgia LoD wing made up of +"The Prophet," "Urvile," and "Leftist." Using Mentor's Phoenix +Project board to coordinate, the Doomsters helped telco security +to trap Wasp, by luring him into a machine with a tap +and line-trace installed. Wasp lost. LoD won! And my, did they brag. + +Urvile, Prophet and Leftist were well-qualified for this activity, +probably more so even than the quite accomplished Control-C. +The Georgia boys knew all about phone switching-stations. +Though relative johnny-come-latelies in the Legion of Doom, +they were considered some of LoD's heaviest guys, +into the hairiest systems around. They had the good fortune +to live in or near Atlanta, home of the sleepy and apparently +tolerant BellSouth RBOC. + +As RBOC security went, BellSouth were "cake." US West (of Arizona, +the Rockies and the Pacific Northwest) were tough and aggressive, +probably the heaviest RBOC around. Pacific Bell, California's PacBell, +were sleek, high-tech, and longtime veterans of the LA phone-phreak wars. +NYNEX had the misfortune to run the New York City area, and were warily +prepared for most anything. Even Michigan Bell, a division of the +Ameritech RBOC, at least had the elementary sense to hire their own hacker +as a useful scarecrow. But BellSouth, even though their corporate P.R. +proclaimed them to have "Everything You Expect From a Leader," were pathetic. + +When rumor about LoD's mastery of Georgia's switching network got around +to BellSouth through Bellcore and telco security scuttlebutt, +they at first refused to believe it. If you paid serious attention +to every rumor out and about these hacker kids, you would hear all kinds +of wacko saucer-nut nonsense: that the National Security Agency +monitored all American phone calls, that the CIA and DEA tracked +traffic on bulletin-boards with word-analysis programs, +that the Condor could start World War III from a payphone. + +If there were hackers into BellSouth switching-stations, then how come +nothing had happened? Nothing had been hurt. BellSouth's machines +weren't crashing. BellSouth wasn't suffering especially badly from fraud. +BellSouth's customers weren't complaining. BellSouth was headquartered +in Atlanta, ambitious metropolis of the new high-tech Sunbelt; +and BellSouth was upgrading its network by leaps and bounds, +digitizing the works left right and center. They could hardly be +considered sluggish or naive. BellSouth's technical expertise +was second to none, thank you kindly. But then came the Florida business. + +On June 13, 1989, callers to the Palm Beach County Probation Department, +in Delray Beach, Florida, found themselves involved in a remarkable +discussion with a phone-sex worker named "Tina" in New York State. +Somehow, ANY call to this probation office near Miami was instantly +and magically transported across state lines, at no extra charge to the user, +to a pornographic phone-sex hotline hundreds of miles away! + +This practical joke may seem utterly hilarious at first hearing, +and indeed there was a good deal of chuckling about it in +phone phreak circles, including the Autumn 1989 issue of 2600. +But for Southern Bell (the division of the BellSouth RBOC +supplying local service for Florida, Georgia, North Carolina +and South Carolina), this was a smoking gun. For the first time ever, +a computer intruder had broken into a BellSouth central office +switching station and re-programmed it! + +Or so BellSouth thought in June 1989. Actually, LoD members had been +frolicking harmlessly in BellSouth switches since September 1987. +The stunt of June 13--call-forwarding a number through manipulation +of a switching station--was child's play for hackers as accomplished +as the Georgia wing of LoD. Switching calls interstate sounded like +a big deal, but it took only four lines of code to accomplish this. +An easy, yet more discreet, stunt, would be to call-forward another +number to your own house. If you were careful and considerate, +and changed the software back later, then not a soul would know. +Except you. And whoever you had bragged to about it. + +As for BellSouth, what they didn't know wouldn't hurt them. + +Except now somebody had blown the whole thing wide open, and BellSouth knew. + +A now alerted and considerably paranoid BellSouth began searching switches +right and left for signs of impropriety, in that hot summer of 1989. +No fewer than forty-two BellSouth employees were put on 12-hour shifts, +twenty-four hours a day, for two solid months, poring over records +and monitoring computers for any sign of phony access. These forty-two +overworked experts were known as BellSouth's "Intrusion Task Force." + +What the investigators found astounded them. Proprietary telco databases +had been manipulated: phone numbers had been created out of thin air, +with no users' names and no addresses. And perhaps worst of all, +no charges and no records of use. The new digital ReMOB (Remote Observation) +diagnostic feature had been extensively tampered with--hackers had learned to +reprogram ReMOB software, so that they could listen in on any switch-routed +call at their leisure! They were using telco property to SPY! + +The electrifying news went out throughout law enforcement in 1989. +It had never really occurred to anyone at BellSouth that their prized +and brand-new digital switching-stations could be RE-PROGRAMMED. +People seemed utterly amazed that anyone could have the nerve. +Of course these switching stations were "computers," and everybody +knew hackers liked to "break into computers:" but telephone people's +computers were DIFFERENT from normal people's computers. + +The exact reason WHY these computers were "different" was +rather ill-defined. It certainly wasn't the extent of their security. +The security on these BellSouth computers was lousy; the AIMSX computers, +for instance, didn't even have passwords. But there was no question that +BellSouth strongly FELT that their computers were very different indeed. +And if there were some criminals out there who had not gotten that message, +BellSouth was determined to see that message taught. + +After all, a 5ESS switching station was no mere bookkeeping system for +some local chain of florists. Public service depended on these stations. +Public SAFETY depended on these stations. + +And hackers, lurking in there call-forwarding or ReMobbing, could spy +on anybody in the local area! They could spy on telco officials! +They could spy on police stations! They could spy on local offices +of the Secret Service. . . . + +In 1989, electronic cops and hacker-trackers began using scrambler-phones +and secured lines. It only made sense. There was no telling who was into +those systems. Whoever they were, they sounded scary. This was some +new level of antisocial daring. Could be West German hackers, in the pay +of the KGB. That too had seemed a weird and farfetched notion, +until Clifford Stoll had poked and prodded a sluggish Washington +law-enforcement bureaucracy into investigating a computer intrusion +that turned out to be exactly that--HACKERS, IN THE PAY OF THE KGB! +Stoll, the systems manager for an Internet lab in Berkeley California, +had ended up on the front page of the New Nork Times, proclaimed a national +hero in the first true story of international computer espionage. +Stoll's counterspy efforts, which he related in a bestselling book, +The Cuckoo's Egg, in 1989, had established the credibility of `hacking' +as a possible threat to national security. The United States Secret Service +doesn't mess around when it suspects a possible action by a foreign +intelligence apparat. + +The Secret Service scrambler-phones and secured lines put +a tremendous kink in law enforcement's ability to operate freely; +to get the word out, cooperate, prevent misunderstandings. +Nevertheless, 1989 scarcely seemed the time for half-measures. +If the police and Secret Service themselves were not operationally secure, +then how could they reasonably demand measures of security from +private enterprise? At least, the inconvenience made people aware +of the seriousness of the threat. + +If there was a final spur needed to get the police off the dime, +it came in the realization that the emergency 911 system was vulnerable. +The 911 system has its own specialized software, but it is run on the same +digital switching systems as the rest of the telephone network. +911 is not physically different from normal telephony. But it is +certainly culturally different, because this is the area of +telephonic cyberspace reserved for the police and emergency services. + +Your average policeman may not know much about hackers or phone-phreaks. +Computer people are weird; even computer COPS are rather weird; +the stuff they do is hard to figure out. But a threat to the 911 system +is anything but an abstract threat. If the 911 system goes, people can die. + +Imagine being in a car-wreck, staggering to a phone-booth, +punching 911 and hearing "Tina" pick up the phone-sex line +somewhere in New York! The situation's no longer comical, somehow. + +And was it possible? No question. Hackers had attacked 911 +systems before. Phreaks can max-out 911 systems just by siccing +a bunch of computer-modems on them in tandem, dialling them over +and over until they clog. That's very crude and low-tech, +but it's still a serious business. + +The time had come for action. It was time to take stern measures +with the underground. It was time to start picking up the dropped threads, +the loose edges, the bits of braggadocio here and there; it was time to get +on the stick and start putting serious casework together. Hackers weren't +"invisible." They THOUGHT they were invisible; but the truth was, +they had just been tolerated too long. + +Under sustained police attention in the summer of '89, the digital +underground began to unravel as never before. + +The first big break in the case came very early on: July 1989, +the following month. The perpetrator of the "Tina" switch was caught, +and confessed. His name was "Fry Guy," a 16-year-old in Indiana. +Fry Guy had been a very wicked young man. + +Fry Guy had earned his handle from a stunt involving French fries. +Fry Guy had filched the log-in of a local MacDonald's manager +and had logged-on to the MacDonald's mainframe on the Sprint +Telenet system. Posing as the manager, Fry Guy had altered +MacDonald's records, and given some teenage hamburger-flipping +friends of his, generous raises. He had not been caught. + +Emboldened by success, Fry Guy moved on to credit-card abuse. +Fry Guy was quite an accomplished talker; with a gift for +"social engineering." If you can do "social engineering" +--fast-talk, fake-outs, impersonation, conning, scamming-- +then card abuse comes easy. (Getting away with it in +the long run is another question). + +Fry Guy had run across "Urvile" of the Legion of Doom +on the ALTOS Chat board in Bonn, Germany. ALTOS Chat +was a sophisticated board, accessible through globe-spanning +computer networks like BITnet, Tymnet, and Telenet. +ALTOS was much frequented by members of Germany's +Chaos Computer Club. Two Chaos hackers who hung out on ALTOS, +"Jaeger" and "Pengo," had been the central villains of +Clifford Stoll's Cuckoo's Egg case: consorting in East Berlin +with a spymaster from the KGB, and breaking into American +computers for hire, through the Internet. + +When LoD members learned the story of Jaeger's depredations +from Stoll's book, they were rather less than impressed, +technically speaking. On LoD's own favorite board of the moment, +"Black Ice," LoD members bragged that they themselves could have done +all the Chaos break-ins in a week flat! Nevertheless, LoD were grudgingly +impressed by the Chaos rep, the sheer hairy-eyed daring of hash-smoking +anarchist hackers who had rubbed shoulders with the fearsome big-boys +of international Communist espionage. LoD members sometimes traded +bits of knowledge with friendly German hackers on ALTOS--phone numbers +for vulnerable VAX/VMS computers in Georgia, for instance. +Dutch and British phone phreaks, and the Australian clique of +"Phoenix," "Nom," and "Electron," were ALTOS regulars, too. +In underground circles, to hang out on ALTOS was considered +the sign of an elite dude, a sophisticated hacker of the +international digital jet-set. + +Fry Guy quickly learned how to raid information from credit-card +consumer-reporting agencies. He had over a hundred stolen credit-card +numbers in his notebooks, and upwards of a thousand swiped long-distance +access codes. He knew how to get onto Altos, and how to talk the talk of +the underground convincingly. He now wheedled knowledge of switching-station +tricks from Urvile on the ALTOS system. + +Combining these two forms of knowledge enabled Fry Guy to bootstrap +his way up to a new form of wire-fraud. First, he'd snitched credit card +numbers from credit-company computers. The data he copied included names, +addresses and phone numbers of the random card-holders. + +Then Fry Guy, impersonating a card-holder, called up Western Union +and asked for a cash advance on "his" credit card. Western Union, +as a security guarantee, would call the customer back, at home, +to verify the transaction. + +But, just as he had switched the Florida probation office to "Tina" +in New York, Fry Guy switched the card-holder's number to a local pay-phone. +There he would lurk in wait, muddying his trail by routing and re-routing +the call, through switches as far away as Canada. When the call came through, +he would boldly "social-engineer," or con, the Western Union people, pretending +to be the legitimate card-holder. Since he'd answered the proper phone number, +the deception was not very hard. Western Union's money was then shipped to +a confederate of Fry Guy's in his home town in Indiana. + +Fry Guy and his cohort, using LoD techniques, stole six thousand dollars +from Western Union between December 1988 and July 1989. They also dabbled +in ordering delivery of stolen goods through card-fraud. Fry Guy +was intoxicated with success. The sixteen-year-old fantasized wildly +to hacker rivals, boasting that he'd used rip-off money to hire himself +a big limousine, and had driven out-of-state with a groupie from +his favorite heavy-metal band, Motley Crue. + +Armed with knowledge, power, and a gratifying stream of free money, +Fry Guy now took it upon himself to call local representatives +of Indiana Bell security, to brag, boast, strut, and utter +tormenting warnings that his powerful friends in the notorious +Legion of Doom could crash the national telephone network. +Fry Guy even named a date for the scheme: the Fourth of July, +a national holiday. + +This egregious example of the begging-for-arrest syndrome was shortly +followed by Fry Guy's arrest. After the Indiana telephone company figured +out who he was, the Secret Service had DNRs--Dialed Number Recorders-- +installed on his home phone lines. These devices are not taps, and can't +record the substance of phone calls, but they do record the phone numbers +of all calls going in and out. Tracing these numbers showed Fry Guy's +long-distance code fraud, his extensive ties to pirate bulletin boards, +and numerous personal calls to his LoD friends in Atlanta. By July 11, +1989, Prophet, Urvile and Leftist also had Secret Service DNR +"pen registers" installed on their own lines. + +The Secret Service showed up in force at Fry Guy's house on July 22, 1989, +to the horror of his unsuspecting parents. The raiders were led by +a special agent from the Secret Service's Indianapolis office. +However, the raiders were accompanied and advised by Timothy M. Foley +of the Secret Service's Chicago office (a gentleman about whom +we will soon be hearing a great deal). + +Following federal computer-crime techniques that had been standard +since the early 1980s, the Secret Service searched the house thoroughly, +and seized all of Fry Guy's electronic equipment and notebooks. +All Fry Guy's equipment went out the door in the custody of the +Secret Service, which put a swift end to his depredations. + +The USSS interrogated Fry Guy at length. His case was put in the charge +of Deborah Daniels, the federal US Attorney for the Southern District +of Indiana. Fry Guy was charged with eleven counts of computer fraud, +unauthorized computer access, and wire fraud. The evidence was thorough +and irrefutable. For his part, Fry Guy blamed his corruption on the +Legion of Doom and offered to testify against them. + +Fry Guy insisted that the Legion intended to crash the phone system +on a national holiday. And when AT&T crashed on Martin Luther King Day, +1990, this lent a credence to his claim that genuinely alarmed telco +security and the Secret Service. + +Fry Guy eventually pled guilty on May 31, 1990. On September 14, +he was sentenced to forty-four months' probation and four hundred hours' +community service. He could have had it much worse; but it made sense +to prosecutors to take it easy on this teenage minor, while zeroing +in on the notorious kingpins of the Legion of Doom. + +But the case against LoD had nagging flaws. Despite the best effort +of investigators, it was impossible to prove that the Legion had crashed +the phone system on January 15, because they, in fact, hadn't done so. +The investigations of 1989 did show that certain members of +the Legion of Doom had achieved unprecedented power over the telco +switching stations, and that they were in active conspiracy +to obtain more power yet. Investigators were privately convinced +that the Legion of Doom intended to do awful things with this knowledge, +but mere evil intent was not enough to put them in jail. + +And although the Atlanta Three--Prophet, Leftist, and especially Urvile-- +had taught Fry Guy plenty, they were not themselves credit-card fraudsters. +The only thing they'd "stolen" was long-distance service--and since they'd +done much of that through phone-switch manipulation, there was no easy way +to judge how much they'd "stolen," or whether this practice was even "theft" +of any easily recognizable kind. + +Fry Guy's theft of long-distance codes had cost the phone companies plenty. +The theft of long-distance service may be a fairly theoretical "loss," +but it costs genuine money and genuine time to delete all those stolen codes, +and to re-issue new codes to the innocent owners of those corrupted codes. +The owners of the codes themselves are victimized, and lose time and money +and peace of mind in the hassle. And then there were the credit-card victims +to deal with, too, and Western Union. When it came to rip-off, Fry Guy was +far more of a thief than LoD. It was only when it came to actual computer +expertise that Fry Guy was small potatoes. + +The Atlanta Legion thought most "rules" of cyberspace were for rodents +and losers, but they DID have rules. THEY NEVER CRASHED ANYTHING, +AND THEY NEVER TOOK MONEY. These were rough rules-of-thumb, and +rather dubious principles when it comes to the ethical subtleties +of cyberspace, but they enabled the Atlanta Three to operate with +a relatively clear conscience (though never with peace of mind). + +If you didn't hack for money, if you weren't robbing people of actual funds +--money in the bank, that is-- then nobody REALLY got hurt, in LoD's opinion. +"Theft of service" was a bogus issue, and "intellectual property" was +a bad joke. But LoD had only elitist contempt for rip-off artists, +"leechers," thieves. They considered themselves clean. In their opinion, +if you didn't smash-up or crash any systems --(well, not on purpose, anyhow-- +accidents can happen, just ask Robert Morris) then it was very unfair +to call you a "vandal" or a "cracker." When you were hanging out on-line +with your "pals" in telco security, you could face them down from the higher +plane of hacker morality. And you could mock the police from the supercilious +heights of your hacker's quest for pure knowledge. + +But from the point of view of law enforcement and telco security, however, +Fry Guy was not really dangerous. The Atlanta Three WERE dangerous. +It wasn't the crimes they were committing, but the DANGER, +the potential hazard, the sheer TECHNICAL POWER LoD had accumulated, +that had made the situation untenable. Fry Guy was not LoD. +He'd never laid eyes on anyone in LoD; his only contacts with them +had been electronic. Core members of the Legion of Doom tended to meet +physically for conventions every year or so, to get drunk, give each other +the hacker high-sign, send out for pizza and ravage hotel suites. +Fry Guy had never done any of this. Deborah Daniels assessed Fry Guy +accurately as "an LoD wannabe." + +Nevertheless Fry Guy's crimes would be directly attributed to LoD +in much future police propaganda. LoD would be described as +"a closely knit group" involved in "numerous illegal activities" +including "stealing and modifying individual credit histories," +and "fraudulently obtaining money and property." Fry Guy did this, +but the Atlanta Three didn't; they simply weren't into theft, +but rather intrusion. This caused a strange kink in +the prosecution's strategy. LoD were accused of +"disseminating information about attacking computers +to other computer hackers in an effort to shift the focus +of law enforcement to those other hackers and away from the Legion of Doom." + +This last accusation (taken directly from a press release by the Chicago +Computer Fraud and Abuse Task Force) sounds particularly far-fetched. +One might conclude at this point that investigators would have been +well-advised to go ahead and "shift their focus" from the "Legion of Doom." +Maybe they SHOULD concentrate on "those other hackers"--the ones who were +actually stealing money and physical objects. + +But the Hacker Crackdown of 1990 was not a simple policing action. +It wasn't meant just to walk the beat in cyberspace--it was a CRACKDOWN, +a deliberate attempt to nail the core of the operation, to send a dire +and potent message that would settle the hash of the digital underground +for good. + +By this reasoning, Fry Guy wasn't much more than the electronic equivalent +of a cheap streetcorner dope dealer. As long as the masterminds of LoD were +still flagrantly operating, pushing their mountains of illicit knowledge +right and left, and whipping up enthusiasm for blatant lawbreaking, +then there would be an INFINITE SUPPLY of Fry Guys. + +Because LoD were flagrant, they had left trails everywhere, +to be picked up by law enforcement in New York, Indiana, +Florida, Texas, Arizona, Missouri, even Australia. +But 1990's war on the Legion of Doom was led out of Illinois, +by the Chicago Computer Fraud and Abuse Task Force. + +# + +The Computer Fraud and Abuse Task Force, led by federal prosecutor +William J. Cook, had started in 1987 and had swiftly become one +of the most aggressive local "dedicated computer-crime units." +Chicago was a natural home for such a group. The world's first +computer bulletin-board system had been invented in Illinois. +The state of Illinois had some of the nation's first and sternest +computer crime laws. Illinois State Police were markedly alert +to the possibilities of white-collar crime and electronic fraud. + +And William J. Cook in particular was a rising star in +electronic crime-busting. He and his fellow federal prosecutors +at the U.S. Attorney's office in Chicago had a tight relation +with the Secret Service, especially go-getting Chicago-based agent +Timothy Foley. While Cook and his Department of Justice colleagues +plotted strategy, Foley was their man on the street. + +Throughout the 1980s, the federal government had given prosecutors +an armory of new, untried legal tools against computer crime. +Cook and his colleagues were pioneers in the use of these new statutes +in the real-life cut-and-thrust of the federal courtroom. + +On October 2, 1986, the US Senate had passed the +"Computer Fraud and Abuse Act" unanimously, but there +were pitifully few convictions under this statute. +Cook's group took their name from this statute, +since they were determined to transform this powerful but +rather theoretical Act of Congress into a real-life engine +of legal destruction against computer fraudsters and scofflaws. + +It was not a question of merely discovering crimes, +investigating them, and then trying and punishing their +perpetrators. The Chicago unit, like most everyone else +in the business, already KNEW who the bad guys were: +the Legion of Doom and the writers and editors of Phrack. +The task at hand was to find some legal means of putting +these characters away. + +This approach might seem a bit dubious, to someone not +acquainted with the gritty realities of prosecutorial work. +But prosecutors don't put people in jail for crimes +they have committed; they put people in jail for crimes +they have committed THAT CAN BE PROVED IN COURT. +Chicago federal police put Al Capone in prison +for income-tax fraud. Chicago is a big town, +with a rough-and-ready bare-knuckle tradition +on both sides of the law. + +Fry Guy had broken the case wide open and alerted telco security +to the scope of the problem. But Fry Guy's crimes would not +put the Atlanta Three behind bars--much less the wacko underground +journalists of Phrack. So on July 22, 1989, the same day that +Fry Guy was raided in Indiana, the Secret Service descended upon +the Atlanta Three. + +This was likely inevitable. By the summer of 1989, law enforcement +were closing in on the Atlanta Three from at least six directions at once. +First, there were the leads from Fry Guy, which had led to the DNR registers +being installed on the lines of the Atlanta Three. The DNR evidence alone +would have finished them off, sooner or later. + +But second, the Atlanta lads were already well-known to Control-C +and his telco security sponsors. LoD's contacts with telco security +had made them overconfident and even more boastful than usual; +they felt that they had powerful friends in high places, +and that they were being openly tolerated by telco security. +But BellSouth's Intrusion Task Force were hot on the trail of LoD +and sparing no effort or expense. + +The Atlanta Three had also been identified by name and listed +on the extensive anti-hacker files maintained, and retailed for pay, +by private security operative John Maxfield of Detroit. +Maxfield, who had extensive ties to telco security +and many informants in the underground, was a bete noire +of the Phrack crowd, and the dislike was mutual. + + +The Atlanta Three themselves had written articles for Phrack. +This boastful act could not possibly escape telco and law enforcement +attention. + +"Knightmare," a high-school age hacker from Arizona, +was a close friend and disciple of Atlanta LoD, +but he had been nabbed by the formidable Arizona +Organized Crime and Racketeering Unit. Knightmare was +on some of LoD's favorite boards--"Black Ice" in particular-- +and was privy to their secrets. And to have Gail Thackeray, +the Assistant Attorney General of Arizona, on one's trail +was a dreadful peril for any hacker. + +And perhaps worst of all, Prophet had committed a major blunder +by passing an illicitly copied BellSouth computer-file to Knight Lightning, +who had published it in Phrack. This, as we will see, was an act of dire +consequence for almost everyone concerned. + +On July 22, 1989, the Secret Service showed up at the Leftist's house, +where he lived with his parents. A massive squad of some twenty officers +surrounded the building: Secret Service, federal marshals, local police, +possibly BellSouth telco security; it was hard to tell in the crush. +Leftist's dad, at work in his basement office, first noticed +a muscular stranger in plain clothes crashing through the +back yard with a drawn pistol. As more strangers poured +into the house, Leftist's dad naturally assumed there was +an armed robbery in progress. + +Like most hacker parents, Leftist's mom and dad had only the vaguest +notions of what their son had been up to all this time. Leftist had +a day-job repairing computer hardware. His obsession with computers +seemed a bit odd, but harmless enough, and likely to produce a well- +paying career. The sudden, overwhelming raid left Leftist's +parents traumatized. + +The Leftist himself had been out after work with his co-workers, +surrounding a couple of pitchers of margaritas. As he came trucking +on tequila-numbed feet up the pavement, toting a bag full of floppy-disks, +he noticed a large number of unmarked cars parked in his driveway. +All the cars sported tiny microwave antennas. + +The Secret Service had knocked the front door off its hinges, +almost flattening his mom. + +Inside, Leftist was greeted by Special Agent James Cool +of the US Secret Service, Atlanta office. Leftist was flabbergasted. +He'd never met a Secret Service agent before. He could not imagine +that he'd ever done anything worthy of federal attention. +He'd always figured that if his activities became intolerable, +one of his contacts in telco security would give him a private +phone-call and tell him to knock it off. + +But now Leftist was pat-searched for weapons by grim professionals, +and his bag of floppies was quickly seized. He and his parents were +all shepherded into separate rooms and grilled at length as a score +of officers scoured their home for anything electronic. + +Leftist was horrified as his treasured IBM AT personal computer +with its forty-meg hard disk, and his recently purchased 80386 IBM-clone +with a whopping hundred-meg hard disk, both went swiftly out the door +in Secret Service custody. They also seized all his disks, all his notebooks, +and a tremendous booty in dogeared telco documents that Leftist had snitched +out of trash dumpsters. + +Leftist figured the whole thing for a big misunderstanding. +He'd never been into MILITARY computers. He wasn't a SPY or a COMMUNIST. +He was just a good ol' Georgia hacker, and now he just wanted all these +people out of the house. But it seemed they wouldn't go until he made +some kind of statement. + +And so, he levelled with them. + +And that, Leftist said later from his federal prison camp in Talladega, +Alabama, was a big mistake. The Atlanta area was unique, +in that it had three members of the Legion of Doom who actually +occupied more or less the same physical locality. Unlike the rest +of LoD, who tended to associate by phone and computer, +Atlanta LoD actually WERE "tightly knit." It was no real +surprise that the Secret Service agents apprehending Urvile +at the computer-labs at Georgia Tech, would discover Prophet +with him as well. + +Urvile, a 21-year-old Georgia Tech student in polymer chemistry, +posed quite a puzzling case for law enforcement. Urvile--also known +as "Necron 99," as well as other handles, for he tended to change his +cover-alias about once a month--was both an accomplished hacker +and a fanatic simulation-gamer. + +Simulation games are an unusual hobby; but then hackers are unusual people, +and their favorite pastimes tend to be somewhat out of the ordinary. +The best-known American simulation game is probably "Dungeons & Dragons," +a multi-player parlor entertainment played with paper, maps, pencils, +statistical tables and a variety of oddly-shaped dice. Players pretend +to be heroic characters exploring a wholly-invented fantasy world. +The fantasy worlds of simulation gaming are commonly pseudo-medieval, +involving swords and sorcery--spell-casting wizards, knights in armor, +unicorns and dragons, demons and goblins. + +Urvile and his fellow gamers preferred their fantasies highly technological. +They made use of a game known as "G.U.R.P.S.," the "Generic Universal Role +Playing System," published by a company called Steve Jackson Games (SJG). + +"G.U.R.P.S." served as a framework for creating a wide variety of artificial +fantasy worlds. Steve Jackson Games published a smorgasboard of books, +full of detailed information and gaming hints, which were used to flesh-out +many different fantastic backgrounds for the basic GURPS framework. +Urvile made extensive use of two SJG books called GURPS High-Tech +and GURPS Special Ops. + +In the artificial fantasy-world of GURPS Special Ops, +players entered a modern fantasy of intrigue and international espionage. +On beginning the game, players started small and powerless, +perhaps as minor-league CIA agents or penny-ante arms dealers. +But as players persisted through a series of game sessions +(game sessions generally lasted for hours, over long, +elaborate campaigns that might be pursued for months on end) +then they would achieve new skills, new knowledge, new power. +They would acquire and hone new abilities, such as marksmanship, +karate, wiretapping, or Watergate burglary. They could also win +various kinds of imaginary booty, like Berettas, or martini shakers, +or fast cars with ejection seats and machine-guns under the headlights. + +As might be imagined from the complexity of these games, +Urvile's gaming notes were very detailed and extensive. +Urvile was a "dungeon-master," inventing scenarios +for his fellow gamers, giant simulated adventure-puzzles +for his friends to unravel. Urvile's game notes covered +dozens of pages with all sorts of exotic lunacy, all about +ninja raids on Libya and break-ins on encrypted Red Chinese supercomputers. +His notes were written on scrap-paper and kept in loose-leaf binders. + +The handiest scrap paper around Urvile's college digs were the many pounds of +BellSouth printouts and documents that he had snitched out of telco dumpsters. +His notes were written on the back of misappropriated telco property. +Worse yet, the gaming notes were chaotically interspersed with Urvile's +hand-scrawled records involving ACTUAL COMPUTER INTRUSIONS that he +had committed. + +Not only was it next to impossible to tell Urvile's fantasy game-notes +from cyberspace "reality," but Urvile himself barely made this distinction. +It's no exaggeration to say that to Urvile it was ALL a game. Urvile was +very bright, highly imaginative, and quite careless of other people's notions +of propriety. His connection to "reality" was not something to which he paid +a great deal of attention. + +Hacking was a game for Urvile. It was an amusement he was carrying out, +it was something he was doing for fun. And Urvile was an obsessive young man. +He could no more stop hacking than he could stop in the middle of +a jigsaw puzzle, or stop in the middle of reading a Stephen Donaldson +fantasy trilogy. (The name "Urvile" came from a best-selling Donaldson novel.) + +Urvile's airy, bulletproof attitude seriously annoyed his interrogators. +First of all, he didn't consider that he'd done anything wrong. +There was scarcely a shred of honest remorse in him. On the contrary, +he seemed privately convinced that his police interrogators were operating +in a demented fantasy-world all their own. Urvile was too polite +and well-behaved to say this straight-out, but his reactions were askew +and disquieting. + +For instance, there was the business about LoD's ability +to monitor phone-calls to the police and Secret Service. +Urvile agreed that this was quite possible, and posed +no big problem for LoD. In fact, he and his friends +had kicked the idea around on the "Black Ice" board, +much as they had discussed many other nifty notions, +such as building personal flame-throwers and jury-rigging +fistfulls of blasting-caps. They had hundreds of dial-up numbers +for government agencies that they'd gotten through scanning Atlanta phones, +or had pulled from raided VAX/VMS mainframe computers. + +Basically, they'd never gotten around to listening in on the cops +because the idea wasn't interesting enough to bother with. +Besides, if they'd been monitoring Secret Service phone calls, +obviously they'd never have been caught in the first place. Right? + +The Secret Service was less than satisfied with this rapier-like hacker logic. + +Then there was the issue of crashing the phone system. No problem, +Urvile admitted sunnily. Atlanta LoD could have shut down phone service +all over Atlanta any time they liked. EVEN THE 911 SERVICE? +Nothing special about that, Urvile explained patiently. +Bring the switch to its knees, with say the UNIX "makedir" bug, +and 911 goes down too as a matter of course. The 911 system +wasn't very interesting, frankly. It might be tremendously +interesting to cops (for odd reasons of their own), but as +technical challenges went, the 911 service was yawnsville. + +So of course the Atlanta Three could crash service. +They probably could have crashed service all over +BellSouth territory, if they'd worked at it for a while. +But Atlanta LoD weren't crashers. Only losers and rodents +were crashers. LoD were ELITE. + +Urvile was privately convinced that sheer technical +expertise could win him free of any kind of problem. +As far as he was concerned, elite status in the digital +underground had placed him permanently beyond the intellectual +grasp of cops and straights. Urvile had a lot to learn. + +Of the three LoD stalwarts, Prophet was in the most direct trouble. +Prophet was a UNIX programming expert who burrowed in and out +of the Internet as a matter of course. He'd started his hacking +career at around age 14, meddling with a UNIX mainframe system +at the University of North Carolina. + +Prophet himself had written the handy Legion of Doom +file "UNIX Use and Security From the Ground Up." +UNIX (pronounced "you-nicks") is a powerful, +flexible computer operating-system, for multi-user, +multi-tasking computers. In 1969, when UNIX was created +in Bell Labs, such computers were exclusive to large +corporations and universities, but today UNIX is run +on thousands of powerful home machines. UNIX was +particularly well-suited to telecommunications programming, +and had become a standard in the field. Naturally, UNIX +also became a standard for the elite hacker and phone phreak. +Lately, Prophet had not been so active as Leftist and Urvile, +but Prophet was a recidivist. In 1986, when he was eighteen, +Prophet had been convicted of "unauthorized access +to a computer network" in North Carolina. He'd been +discovered breaking into the Southern Bell Data Network, +a UNIX-based internal telco network supposedly closed to the public. +He'd gotten a typical hacker sentence: six months suspended, +120 hours community service, and three years' probation. + +After that humiliating bust, Prophet had gotten rid of most of his +tonnage of illicit phreak and hacker data, and had tried to go straight. +He was, after all, still on probation. But by the autumn of 1988, +the temptations of cyberspace had proved too much for young Prophet, +and he was shoulder-to-shoulder with Urvile and Leftist into some +of the hairiest systems around. + +In early September 1988, he'd broken into BellSouth's centralized +automation system, AIMSX or "Advanced Information Management System." +AIMSX was an internal business network for BellSouth, where telco +employees stored electronic mail, databases, memos, and calendars, +and did text processing. Since AIMSX did not have public dial-ups, +it was considered utterly invisible to the public, and was not well-secured +--it didn't even require passwords. Prophet abused an account known +as "waa1," the personal account of an unsuspecting telco employee. +Disguised as the owner of waa1, Prophet made about ten visits to AIMSX. + +Prophet did not damage or delete anything in the system. +His presence in AIMSX was harmless and almost invisible. +But he could not rest content with that. + +One particular piece of processed text on AIMSX was a telco document +known as "Bell South Standard Practice 660-225-104SV Control Office +Administration of Enhanced 911 Services for Special Services +and Major Account Centers dated March 1988." + +Prophet had not been looking for this document. It was merely one +among hundreds of similar documents with impenetrable titles. +However, having blundered over it in the course of his illicit +wanderings through AIMSX, he decided to take it with him as a trophy. +It might prove very useful in some future boasting, bragging, +and strutting session. So, some time in September 1988, +Prophet ordered the AIMSX mainframe computer to copy this document +(henceforth called simply called "the E911 Document") and to transfer +this copy to his home computer. + +No one noticed that Prophet had done this. He had "stolen" +the E911 Document in some sense, but notions of property +in cyberspace can be tricky. BellSouth noticed nothing wrong, +because BellSouth still had their original copy. They had not +been "robbed" of the document itself. Many people were supposed +to copy this document--specifically, people who worked for the +nineteen BellSouth "special services and major account centers," +scattered throughout the Southeastern United States. That was +what it was for, why it was present on a computer network +in the first place: so that it could be copied and read-- +by telco employees. But now the data had been copied +by someone who wasn't supposed to look at it. + +Prophet now had his trophy. But he further decided to store +yet another copy of the E911 Document on another person's computer. +This unwitting person was a computer enthusiast named Richard Andrews +who lived near Joliet, Illinois. Richard Andrews was a UNIX programmer +by trade, and ran a powerful UNIX board called "Jolnet," in the basement +of his house. + +Prophet, using the handle "Robert Johnson," had obtained an account +on Richard Andrews' computer. And there he stashed the E911 Document, +by storing it in his own private section of Andrews' computer. + +Why did Prophet do this? If Prophet had eliminated the E911 Document +from his own computer, and kept it hundreds of miles away, on another machine, under an +alias, then he might have been fairly safe from discovery and prosecution-- +although his sneaky action had certainly put the unsuspecting Richard Andrews +at risk. + +But, like most hackers, Prophet was a pack-rat for illicit data. +When it came to the crunch, he could not bear to part from his trophy. +When Prophet's place in Decatur, Georgia was raided in July 1989, +there was the E911 Document, a smoking gun. And there was Prophet +in the hands of the Secret Service, doing his best to "explain." + +Our story now takes us away from the Atlanta Three and their raids +of the Summer of 1989. We must leave Atlanta Three "cooperating fully" +with their numerous investigators. And all three of them did cooperate, +as their Sentencing Memorandum from the US District Court of the +Northern Division of Georgia explained--just before all three of them +were sentenced to various federal prisons in November 1990. + +We must now catch up on the other aspects of the war on the Legion of Doom. +The war on the Legion was a war on a network--in fact, a network of three +networks, which intertwined and interrelated in a complex fashion. +The Legion itself, with Atlanta LoD, and their hanger-on Fry Guy, +were the first network. The second network was Phrack magazine, +with its editors and contributors. + +The third network involved the electronic circle around a hacker +known as "Terminus." + +The war against these hacker networks was carried out by +a law enforcement network. Atlanta LoD and Fry Guy +were pursued by USSS agents and federal prosecutors in Atlanta, +Indiana, and Chicago. "Terminus" found himself pursued by USSS +and federal prosecutors from Baltimore and Chicago. And the war +against Phrack was almost entirely a Chicago operation. + +The investigation of Terminus involved a great deal of energy, +mostly from the Chicago Task Force, but it was to be the least-known +and least-publicized of the Crackdown operations. Terminus, who lived +in Maryland, was a UNIX programmer and consultant, fairly well-known +(under his given name) in the UNIX community, as an acknowledged expert +on AT&T minicomputers. Terminus idolized AT&T, especially Bellcore, +and longed for public recognition as a UNIX expert; his highest ambition +was to work for Bell Labs. + +But Terminus had odd friends and a spotted history. +Terminus had once been the subject of an admiring interview +in Phrack (Volume II, Issue 14, Phile 2--dated May 1987). +In this article, Phrack co-editor Taran King described +"Terminus" as an electronics engineer, 5'9", brown-haired, +born in 1959--at 28 years old, quite mature for a hacker. + +Terminus had once been sysop of a phreak/hack underground board +called "MetroNet," which ran on an Apple II. Later he'd replaced +"MetroNet" with an underground board called "MegaNet," +specializing in IBMs. In his younger days, Terminus had written +one of the very first and most elegant code-scanning programs +for the IBM-PC. This program had been widely distributed +in the underground. Uncounted legions of PC-owning phreaks and +hackers had used Terminus's scanner program to rip-off telco codes. +This feat had not escaped the attention of telco security; +it hardly could, since Terminus's earlier handle, "Terminal Technician," +was proudly written right on the program. + +When he became a full-time computer professional +(specializing in telecommunications programming), +he adopted the handle Terminus, meant to indicate that he +had "reached the final point of being a proficient hacker." +He'd moved up to the UNIX-based "Netsys" board on an AT&T computer, +with four phone lines and an impressive 240 megs of storage. +"Netsys" carried complete issues of Phrack, and Terminus was +quite friendly with its publishers, Taran King and Knight Lightning. + +In the early 1980s, Terminus had been a regular on Plovernet, +Pirate-80, Sherwood Forest and Shadowland, all well-known pirate boards, +all heavily frequented by the Legion of Doom. As it happened, Terminus +was never officially "in LoD," because he'd never been given the official +LoD high-sign and back-slap by Legion maven Lex Luthor. Terminus had +never physically met anyone from LoD. But that scarcely mattered much-- +the Atlanta Three themselves had never been officially vetted by Lex, either. + +As far as law enforcement was concerned, the issues were clear. +Terminus was a full-time, adult computer professional +with particular skills at AT&T software and hardware-- +but Terminus reeked of the Legion of Doom and the underground. + +On February 1, 1990--half a month after the Martin Luther King Day Crash-- +USSS agents Tim Foley from Chicago, and Jack Lewis from the Baltimore office, +accompanied by AT&T security officer Jerry Dalton, travelled to Middle Town, +Maryland. There they grilled Terminus in his home (to the stark terror of +his wife and small children), and, in their customary fashion, hauled his +computers out the door. + +The Netsys machine proved to contain a plethora of arcane UNIX software-- +proprietary source code formally owned by AT&T. Software such as: +UNIX System Five Release 3.2; UNIX SV Release 3.1; UUCP communications +software; KORN SHELL; RFS; IWB; WWB; DWB; the C++ programming language; +PMON; TOOL CHEST; QUEST; DACT, and S FIND. + +In the long-established piratical tradition of the underground, +Terminus had been trading this illicitly-copied software with +a small circle of fellow UNIX programmers. Very unwisely, +he had stored seven years of his electronic mail on his Netsys machine, +which documented all the friendly arrangements he had made with +his various colleagues. + +Terminus had not crashed the AT&T phone system on January 15. +He was, however, blithely running a not-for-profit AT&T +software-piracy ring. This was not an activity AT&T found amusing. +AT&T security officer Jerry Dalton valued this "stolen" property +at over three hundred thousand dollars. + +AT&T's entry into the tussle of free enterprise had been complicated +by the new, vague groundrules of the information economy. +Until the break-up of Ma Bell, AT&T was forbidden to sell +computer hardware or software. Ma Bell was the phone company; +Ma Bell was not allowed to use the enormous revenue from +telephone utilities, in order to finance any entry into +the computer market. + +AT&T nevertheless invented the UNIX operating system. +And somehow AT&T managed to make UNIX a minor source of income. +Weirdly, UNIX was not sold as computer software, +but actually retailed under an obscure regulatory +exemption allowing sales of surplus equipment and scrap. +Any bolder attempt to promote or retail UNIX would have +aroused angry legal opposition from computer companies. +Instead, UNIX was licensed to universities, at modest rates, +where the acids of academic freedom ate away steadily at AT&T's +proprietary rights. + +Come the breakup, AT&T recognized that UNIX was a potential gold-mine. +By now, large chunks of UNIX code had been created that were not AT&T's, +and were being sold by others. An entire rival UNIX-based operating system +had arisen in Berkeley, California (one of the world's great founts of +ideological hackerdom). Today, "hackers" commonly consider "Berkeley UNIX" +to be technically superior to AT&T's "System V UNIX," but AT&T has not +allowed mere technical elegance to intrude on the real-world business +of marketing proprietary software. AT&T has made its own code deliberately +incompatible with other folks' UNIX, and has written code that it can prove +is copyrightable, even if that code happens to be somewhat awkward--"kludgey." +AT&T UNIX user licenses are serious business agreements, replete with very +clear copyright statements and non-disclosure clauses. + +AT&T has not exactly kept the UNIX cat in the bag, +but it kept a grip on its scruff with some success. +By the rampant, explosive standards of software piracy, +AT&T UNIX source code is heavily copyrighted, well-guarded, +well-licensed. UNIX was traditionally run only on +mainframe machines, owned by large groups of suit-and-tie +professionals, rather than on bedroom machines where +people can get up to easy mischief. + +And AT&T UNIX source code is serious high-level programming. +The number of skilled UNIX programmers with any actual motive +to swipe UNIX source code is small. It's tiny, compared to +the tens of thousands prepared to rip-off, say, entertaining +PC games like "Leisure Suit Larry." + +But by 1989, the warez-d00d underground, in the persons of Terminus +and his friends, was gnawing at AT&T UNIX. And the property in question +was not sold for twenty bucks over the counter at the local branch of +Babbage's or Egghead's; this was massive, sophisticated, multi-line, +multi-author corporate code worth tens of thousands of dollars. + +It must be recognized at this point that Terminus's purported ring of UNIX +software pirates had not actually made any money from their suspected crimes. +The $300,000 dollar figure bandied about for the contents of Terminus's +computer did not mean that Terminus was in actual illicit possession +of three hundred thousand of AT&T's dollars. Terminus was shipping +software back and forth, privately, person to person, for free. +He was not making a commercial business of piracy. He hadn't +asked for money; he didn't take money. He lived quite modestly. + +AT&T employees--as well as freelance UNIX consultants, like Terminus-- +commonly worked with "proprietary" AT&T software, both in the office +and at home on their private machines. AT&T rarely sent security officers +out to comb the hard disks of its consultants. Cheap freelance UNIX +contractors were quite useful to AT&T; they didn't have health insurance +or retirement programs, much less union membership in the Communication +Workers of America. They were humble digital drudges, wandering with mop +and bucket through the Great Technological Temple of AT&T; but when the +Secret Service arrived at their homes, it seemed they were eating with +company silverware and sleeping on company sheets! Outrageously, they +behaved as if the things they worked with every day belonged to them! + +And these were no mere hacker teenagers with their hands full +of trash-paper and their noses pressed to the corporate windowpane. +These guys were UNIX wizards, not only carrying AT&T data in their +machines and their heads, but eagerly networking about it, +over machines that were far more powerful than anything previously +imagined in private hands. How do you keep people disposable, +yet assure their awestruck respect for your property? It was a dilemma. + +Much UNIX code was public-domain, available for free. Much "proprietary" +UNIX code had been extensively re-written, perhaps altered so much that it +became an entirely new product--or perhaps not. Intellectual property rights +for software developers were, and are, extraordinarily complex and confused. +And software "piracy," like the private copying of videos, is one of the most +widely practiced "crimes" in the world today. + +The USSS were not experts in UNIX or familiar with the customs of its use. +The United States Secret Service, considered as a body, did not have one single +person in it who could program in a UNIX environment--no, not even one. +The Secret Service WERE making extensive use of expert help, but the "experts" +they had chosen were AT&T and Bellcore security officials, the very victims of +the purported crimes under investigation, the very people whose interest in +AT&T's "proprietary" software was most pronounced. + +On February 6, 1990, Terminus was arrested by Agent Lewis. +Eventually, Terminus would be sent to prison for his illicit +use of a piece of AT&T software. + +The issue of pirated AT&T software would bubble along in the background +during the war on the Legion of Doom. Some half-dozen of Terminus's on-line +acquaintances, including people in Illinois, Texas and California, +were grilled by the Secret Service in connection with the illicit +copying of software. Except for Terminus, however, none were charged +with a crime. None of them shared his peculiar prominence in the +hacker underground. + +But that did not mean that these people would, or could, +stay out of trouble. The transferral of illicit data in +cyberspace is hazy and ill-defined business, with paradoxical +dangers for everyone concerned: hackers, signal carriers, +board owners, cops, prosecutors, even random passers-by. +Sometimes, well-meant attempts to avert trouble +or punish wrongdoing bring more trouble than +would simple ignorance, indifference or impropriety. + +Terminus's "Netsys" board was not a common-or-garden +bulletin board system, though it had most of the usual +functions of a board. Netsys was not a stand-alone machine, +but part of the globe-spanning "UUCP" cooperative network. +The UUCP network uses a set of Unix software programs called +"Unix-to-Unix Copy," which allows Unix systems to throw data to +one another at high speed through the public telephone network. +UUCP is a radically decentralized, not-for-profit network of UNIX computers. +There are tens of thousands of these UNIX machines. Some are small, +but many are powerful and also link to other networks. UUCP has +certain arcane links to major networks such as JANET, EasyNet, BITNET, +JUNET, VNET, DASnet, PeaceNet and FidoNet, as well as the gigantic Internet. +(The so-called "Internet" is not actually a network itself, but rather an +"internetwork" connections standard that allows several globe-spanning +computer networks to communicate with one another. Readers fascinated +by the weird and intricate tangles of modern computer networks may enjoy +John S. Quarterman's authoritative 719-page explication, The Matrix, +Digital Press, 1990.) + +A skilled user of Terminus' UNIX machine could send and receive +electronic mail from almost any major computer network in the world. +Netsys was not called a "board" per se, but rather a "node." +"Nodes" were larger, faster, and more sophisticated than mere "boards," +and for hackers, to hang out on internationally-connected "nodes" +was quite the step up from merely hanging out on local "boards." + +Terminus's Netsys node in Maryland had a number of direct +links to other, similar UUCP nodes, run by people who shared his +interests and at least something of his free-wheeling attitude. +One of these nodes was Jolnet, owned by Richard Andrews, who, +like Terminus, was an independent UNIX consultant. +Jolnet also ran UNIX, and could be contacted at high speed +by mainframe machines from all over the world. Jolnet was +quite a sophisticated piece of work, technically speaking, +but it was still run by an individual, as a private, +not-for-profit hobby. Jolnet was mostly used by other +UNIX programmers--for mail, storage, and access to networks. +Jolnet supplied access network access to about two hundred people, +as well as a local junior college. + +Among its various features and services, Jolnet also carried +Phrack magazine. + +For reasons of his own, Richard Andrews had become suspicious +of a new user called "Robert Johnson." Richard Andrews +took it upon himself to have a look at what "Robert Johnson" +was storing in Jolnet. And Andrews found the E911 Document. + +"Robert Johnson" was the Prophet from the Legion of Doom, +and the E911 Document was illicitly copied data from Prophet's +raid on the BellSouth computers. + +The E911 Document, a particularly illicit piece of digital property, +was about to resume its long, complex, and disastrous career. + +It struck Andrews as fishy that someone not a telephone employee +should have a document referring to the "Enhanced 911 System." +Besides, the document itself bore an obvious warning. + +"WARNING: NOT FOR USE OR DISCLOSURE OUTSIDE BELLSOUTH +OR ANY OF ITS SUBSIDIARIES EXCEPT UNDER WRITTEN AGREEMENT." + +These standard nondisclosure tags are often appended to all sorts +of corporate material. Telcos as a species are particularly notorious +for stamping most everything in sight as "not for use or disclosure." +Still, this particular piece of data was about the 911 System. +That sounded bad to Rich Andrews. + +Andrews was not prepared to ignore this sort of trouble. +He thought it would be wise to pass the document along +to a friend and acquaintance on the UNIX network, for consultation. +So, around September 1988, Andrews sent yet another copy of the +E911 Document electronically to an AT&T employee, one Charles Boykin, +who ran a UNIX-based node called "attctc" in Dallas, Texas. + +"Attctc" was the property of AT&T, and was run from AT&T's +Customer Technology Center in Dallas, hence the name "attctc." +"Attctc" was better-known as "Killer," the name of the machine +that the system was running on. "Killer" was a hefty, powerful, +AT&T 3B2 500 model, a multi-user, multi-tasking UNIX platform +with 32 meg of memory and a mind-boggling 3.2 Gigabytes of storage. +When Killer had first arrived in Texas, in 1985, the 3B2 had been +one of AT&T's great white hopes for going head-to-head with IBM +for the corporate computer-hardware market. "Killer" had been shipped +to the Customer Technology Center in the Dallas Infomart, essentially +a high-technology mall, and there it sat, a demonstration model. + +Charles Boykin, a veteran AT&T hardware and digital communications expert, +was a local technical backup man for the AT&T 3B2 system. As a display model +in the Infomart mall, "Killer" had little to do, and it seemed a shame +to waste the system's capacity. So Boykin ingeniously wrote some UNIX +bulletin-board software for "Killer," and plugged the machine in to the +local phone network. "Killer's" debut in late 1985 made it the first +publicly available UNIX site in the state of Texas. Anyone who wanted to +play was welcome. + +The machine immediately attracted an electronic community. +It joined the UUCP network, and offered network links +to over eighty other computer sites, all of which became dependent +on Killer for their links to the greater world of cyberspace. +And it wasn't just for the big guys; personal computer users +also stored freeware programs for the Amiga, the Apple, +the IBM and the Macintosh on Killer's vast 3,200 meg archives. +At one time, Killer had the largest library of public-domain +Macintosh software in Texas. + +Eventually, Killer attracted about 1,500 users, +all busily communicating, uploading and downloading, +getting mail, gossipping, and linking to arcane +and distant networks. + +Boykin received no pay for running Killer. He considered +it good publicity for the AT&T 3B2 system (whose sales were +somewhat less than stellar), but he also simply enjoyed +the vibrant community his skill had created. He gave away +the bulletin-board UNIX software he had written, free of charge. + +In the UNIX programming community, Charlie Boykin had the +reputation of a warm, open-hearted, level-headed kind of guy. +In 1989, a group of Texan UNIX professionals voted Boykin +"System Administrator of the Year." He was considered +a fellow you could trust for good advice. + +In September 1988, without warning, the E911 Document +came plunging into Boykin's life, forwarded by Richard Andrews. +Boykin immediately recognized that the Document was hot property. +He was not a voice-communications man, and knew little about +the ins and outs of the Baby Bells, but he certainly knew what +the 911 System was, and he was angry to see confidential data +about it in the hands of a nogoodnik. This was clearly a +matter for telco security. So, on September 21, 1988, Boykin +made yet ANOTHER copy of the E911 Document and passed this +one along to a professional acquaintance of his, one Jerome Dalton, +from AT&T Corporate Information Security. Jerry Dalton was the +very fellow who would later raid Terminus's house. + +From AT&T's security division, the E911 Document went to Bellcore. + +Bellcore (or BELL COmmunications REsearch) had once been the central +laboratory of the Bell System. Bell Labs employees had invented +the UNIX operating system. Now Bellcore was a quasi-independent, +jointly owned company that acted as the research arm for all seven +of the Baby Bell RBOCs. Bellcore was in a good position to co-ordinate +security technology and consultation for the RBOCs, and the gentleman in +charge of this effort was Henry M. Kluepfel, a veteran of the Bell System +who had worked there for twenty-four years. + +On October 13, 1988, Dalton passed the E911 Document to Henry Kluepfel. +Kluepfel, a veteran expert witness in telecommunications fraud and +computer-fraud cases, had certainly seen worse trouble than this. +He recognized the document for what it was: a trophy from a hacker break-in. + +However, whatever harm had been done in the intrusion was presumably old news. +At this point there seemed little to be done. Kluepfel made a careful note +of the circumstances and shelved the problem for the time being. + +Whole months passed. + +February 1989 arrived. The Atlanta Three were living it up +in Bell South's switches, and had not yet met their comeuppance. +The Legion was thriving. So was Phrack magazine. +A good six months had passed since Prophet's AIMSX break-in. +Prophet, as hackers will, grew weary of sitting on his laurels. +"Knight Lightning" and "Taran King," the editors of Phrack, +were always begging Prophet for material they could publish. +Prophet decided that the heat must be off by this time, +and that he could safely brag, boast, and strut. + +So he sent a copy of the E911 Document--yet another one-- +from Rich Andrews' Jolnet machine to Knight Lightning's +BITnet account at the University of Missouri. +Let's review the fate of the document so far. + +0. The original E911 Document. This in the AIMSX system +on a mainframe computer in Atlanta, available to hundreds of people, +but all of them, presumably, BellSouth employees. An unknown number +of them may have their own copies of this document, but they are all +professionals and all trusted by the phone company. + +1. Prophet's illicit copy, at home on his own computer in Decatur, Georgia. + +2. Prophet's back-up copy, stored on Rich Andrew's Jolnet machine + in the basement of Rich Andrews' house near Joliet Illinois. + +3. Charles Boykin's copy on "Killer" in Dallas, Texas, + sent by Rich Andrews from Joliet. + +4. Jerry Dalton's copy at AT&T Corporate Information Security in New Jersey, + sent from Charles Boykin in Dallas. + +5. Henry Kluepfel's copy at Bellcore security headquarters in New Jersey, + sent by Dalton. +6. Knight Lightning's copy, sent by Prophet from Rich Andrews' machine, + and now in Columbia, Missouri. + +We can see that the "security" situation of this proprietary document, +once dug out of AIMSX, swiftly became bizarre. Without any money +changing hands, without any particular special effort, this data +had been reproduced at least six times and had spread itself all over +the continent. By far the worst, however, was yet to come. + +In February 1989, Prophet and Knight Lightning bargained electronically +over the fate of this trophy. Prophet wanted to boast, but, at the same time, +scarcely wanted to be caught. + +For his part, Knight Lightning was eager to publish as much of the document +as he could manage. Knight Lightning was a fledgling political-science major +with a particular interest in freedom-of-information issues. He would gladly +publish most anything that would reflect glory on the prowess of the +underground and embarrass the telcos. However, Knight Lightning himself +had contacts in telco security, and sometimes consulted them on material +he'd received that might be too dicey for publication. + +Prophet and Knight Lightning decided to edit the E911 Document +so as to delete most of its identifying traits. First of all, +its large "NOT FOR USE OR DISCLOSURE" warning had to go. +Then there were other matters. For instance, it listed +the office telephone numbers of several BellSouth 911 +specialists in Florida. If these phone numbers were +published in Phrack, the BellSouth employees involved +would very likely be hassled by phone phreaks, +which would anger BellSouth no end, and pose a +definite operational hazard for both Prophet and Phrack. + +So Knight Lightning cut the Document almost in half, +removing the phone numbers and some of the touchier +and more specific information. He passed it back +electronically to Prophet; Prophet was still nervous, +so Knight Lightning cut a bit more. They finally agreed +that it was ready to go, and that it would be published +in Phrack under the pseudonym, "The Eavesdropper." + +And this was done on February 25, 1989. + +The twenty-fourth issue of Phrack featured a chatty interview +with co-ed phone-phreak "Chanda Leir," three articles on BITNET +and its links to other computer networks, an article on 800 and 900 +numbers by "Unknown User," "VaxCat's" article on telco basics +(slyly entitled "Lifting Ma Bell's Veil of Secrecy,)" and +the usual "Phrack World News." + +The News section, with painful irony, featured an extended account +of the sentencing of "Shadowhawk," an eighteen-year-old Chicago hacker +who had just been put in federal prison by William J. Cook himself. + +And then there were the two articles by "The Eavesdropper." +The first was the edited E911 Document, now titled +"Control Office Administration Of Enhanced 911 Services +for Special Services and Major Account Centers." +Eavesdropper's second article was a glossary of terms +explaining the blizzard of telco acronyms and buzzwords +in the E911 Document. + +The hapless document was now distributed, in the usual Phrack routine, +to a good one hundred and fifty sites. Not a hundred and fifty PEOPLE, +mind you--a hundred and fifty SITES, some of these sites linked to UNIX +nodes or bulletin board systems, which themselves had readerships of tens, +dozens, even hundreds of people. + +This was February 1989. Nothing happened immediately. +Summer came, and the Atlanta crew were raided by the Secret Service. +Fry Guy was apprehended. Still nothing whatever happened to Phrack. +Six more issues of Phrack came out, 30 in all, more or less on +a monthly schedule. Knight Lightning and co-editor Taran King +went untouched. + +Phrack tended to duck and cover whenever the heat came down. +During the summer busts of 1987--(hacker busts tended to cluster in summer, +perhaps because hackers were easier to find at home than in college)-- +Phrack had ceased publication for several months, and laid low. +Several LoD hangers-on had been arrested, but nothing had happened +to the Phrack crew, the premiere gossips of the underground. +In 1988, Phrack had been taken over by a new editor, +"Crimson Death," a raucous youngster with a taste for anarchy files. +1989, however, looked like a bounty year for the underground. +Knight Lightning and his co-editor Taran King took up the reins again, +and Phrack flourished throughout 1989. Atlanta LoD went down hard in +the summer of 1989, but Phrack rolled merrily on. Prophet's E911 Document +seemed unlikely to cause Phrack any trouble. By January 1990, +it had been available in Phrack for almost a year. Kluepfel and Dalton, +officers of Bellcore and AT&T security, had possessed the document +for sixteen months--in fact, they'd had it even before Knight Lightning +himself, and had done nothing in particular to stop its distribution. +They hadn't even told Rich Andrews or Charles Boykin to erase the copies +from their UNIX nodes, Jolnet and Killer. + +But then came the monster Martin Luther King Day Crash of January 15, 1990. + +A flat three days later, on January 18, four agents showed up +at Knight Lightning's fraternity house. One was Timothy Foley, +the second Barbara Golden, both of them Secret Service agents +from the Chicago office. Also along was a University of Missouri +security officer, and Reed Newlin, a security man from Southwestern Bell, +the RBOC having jurisdiction over Missouri. + +Foley accused Knight Lightning of causing the nationwide crash +of the phone system. + +Knight Lightning was aghast at this allegation. On the face of it, +the suspicion was not entirely implausible--though Knight Lightning +knew that he himself hadn't done it. Plenty of hot-dog hackers +had bragged that they could crash the phone system, however. +"Shadowhawk," for instance, the Chicago hacker whom William Cook +had recently put in jail, had several times boasted on boards +that he could "shut down AT&T's public switched network." + +And now this event, or something that looked just like it, +had actually taken place. The Crash had lit a fire under +the Chicago Task Force. And the former fence-sitters at +Bellcore and AT&T were now ready to roll. The consensus +among telco security--already horrified by the skill of +the BellSouth intruders --was that the digital underground +was out of hand. LoD and Phrack must go. And in publishing +Prophet's E911 Document, Phrack had provided law enforcement +with what appeared to be a powerful legal weapon. + +Foley confronted Knight Lightning about the E911 Document. + +Knight Lightning was cowed. He immediately began "cooperating fully" +in the usual tradition of the digital underground. + +He gave Foley a complete run of Phrack, printed out in a set +of three-ring binders. He handed over his electronic mailing list +of Phrack subscribers. Knight Lightning was grilled for four hours +by Foley and his cohorts. Knight Lightning admitted that Prophet +had passed him the E911 Document, and he admitted that he had known +it was stolen booty from a hacker raid on a telephone company. +Knight Lightning signed a statement to this effect, and agreed, +in writing, to cooperate with investigators. + +Next day--January 19, 1990, a Friday --the Secret Service returned +with a search warrant, and thoroughly searched Knight Lightning's +upstairs room in the fraternity house. They took all his floppy disks, +though, interestingly, they left Knight Lightning in possession +of both his computer and his modem. (The computer had no hard disk, +and in Foley's judgement was not a store of evidence.) But this was a +very minor bright spot among Knight Lightning's rapidly multiplying troubles. +By this time, Knight Lightning was in plenty of hot water, not only with +federal police, prosecutors, telco investigators, and university security, +but with the elders of his own campus fraternity, who were outraged +to think that they had been unwittingly harboring a federal computer-criminal. + +On Monday, Knight Lightning was summoned to Chicago, where he was +further grilled by Foley and USSS veteran agent Barbara Golden, this time +with an attorney present. And on Tuesday, he was formally indicted +by a federal grand jury. + +The trial of Knight Lightning, which occurred on July 24-27, 1990, +was the crucial show-trial of the Hacker Crackdown. We will examine +the trial at some length in Part Four of this book. + +In the meantime, we must continue our dogged pursuit of the E911 Document. + +It must have been clear by January 1990 that the E911 Document, +in the form Phrack had published it back in February 1989, +had gone off at the speed of light in at least a hundred +and fifty different directions. To attempt to put this +electronic genie back in the bottle was flatly impossible. + +And yet, the E911 Document was STILL stolen property, +formally and legally speaking. Any electronic transference +of this document, by anyone unauthorized to have it, +could be interpreted as an act of wire fraud. Interstate +transfer of stolen property, including electronic property, +was a federal crime. + +The Chicago Computer Fraud and Abuse Task Force had been assured +that the E911 Document was worth a hefty sum of money. In fact, +they had a precise estimate of its worth from BellSouth security personnel: +$79,449. A sum of this scale seemed to warrant vigorous prosecution. +Even if the damage could not be undone, at least this large sum +offered a good legal pretext for stern punishment of the thieves. +It seemed likely to impress judges and juries. And it could be used +in court to mop up the Legion of Doom. + +The Atlanta crowd was already in the bag, by the time +the Chicago Task Force had gotten around to Phrack. +But the Legion was a hydra-headed thing. In late 89, +a brand-new Legion of Doom board, "Phoenix Project," +had gone up in Austin, Texas. Phoenix Project was sysoped +by no less a man than the Mentor himself, ably assisted by +University of Texas student and hardened Doomster "Erik Bloodaxe." + +As we have seen from his Phrack manifesto, the Mentor was a hacker +zealot who regarded computer intrusion as something close to a moral duty. +Phoenix Project was an ambitious effort, intended to revive the digital +underground to what Mentor considered the full flower of the early 80s. +The Phoenix board would also boldly bring elite hackers face-to-face +with the telco "opposition." On "Phoenix," America's cleverest hackers +would supposedly shame the telco squareheads out of their stick-in-the-mud +attitudes, and perhaps convince them that the Legion of Doom elite were really +an all-right crew. The premiere of "Phoenix Project" was heavily trumpeted +by Phrack,and "Phoenix Project" carried a complete run of Phrack issues, +including the E911 Document as Phrack had published it. + +Phoenix Project was only one of many--possibly hundreds--of nodes and boards +all over America that were in guilty possession of the E911 Document. +But Phoenix was an outright, unashamed Legion of Doom board. +Under Mentor's guidance, it was flaunting itself in the face +of telco security personnel. Worse yet, it was actively trying +to WIN THEM OVER as sympathizers for the digital underground elite. +"Phoenix" had no cards or codes on it. Its hacker elite considered +Phoenix at least technically legal. But Phoenix was a corrupting influence, +where hacker anarchy was eating away like digital acid at the underbelly +of corporate propriety. + +The Chicago Computer Fraud and Abuse Task Force now prepared +to descend upon Austin, Texas. + +Oddly, not one but TWO trails of the Task Force's investigation led +toward Austin. The city of Austin, like Atlanta, had made itself +a bulwark of the Sunbelt's Information Age, with a strong university +research presence, and a number of cutting-edge electronics companies, +including Motorola, Dell, CompuAdd, IBM, Sematech and MCC. + +Where computing machinery went, hackers generally followed. +Austin boasted not only "Phoenix Project," currently LoD's +most flagrant underground board, but a number of UNIX nodes. + +One of these nodes was "Elephant," run by a UNIX consultant +named Robert Izenberg. Izenberg, in search of a relaxed Southern +lifestyle and a lowered cost-of-living, had recently migrated +to Austin from New Jersey. In New Jersey, Izenberg had worked +for an independent contracting company, programming UNIX code for +AT&T itself. "Terminus" had been a frequent user on Izenberg's +privately owned Elephant node. + +Having interviewed Terminus and examined the records on Netsys, +the Chicago Task Force were now convinced that they had discovered +an underground gang of UNIX software pirates, who were demonstrably +guilty of interstate trafficking in illicitly copied AT&T source code. +Izenberg was swept into the dragnet around Terminus, the self-proclaimed +ultimate UNIX hacker. + +Izenberg, in Austin, had settled down into a UNIX job +with a Texan branch of IBM. Izenberg was no longer +working as a contractor for AT&T, but he had friends +in New Jersey, and he still logged on to AT&T UNIX +computers back in New Jersey, more or less whenever +it pleased him. Izenberg's activities appeared highly +suspicious to the Task Force. Izenberg might well be +breaking into AT&T computers, swiping AT&T software, +and passing it to Terminus and other possible confederates, +through the UNIX node network. And this data was worth, +not merely $79,499, but hundreds of thousands of dollars! + +On February 21, 1990, Robert Izenberg arrived home +from work at IBM to find that all the computers +had mysteriously vanished from his Austin apartment. +Naturally he assumed that he had been robbed. +His "Elephant" node, his other machines, his notebooks, +his disks, his tapes, all gone! However, nothing much +else seemed disturbed--the place had not been ransacked. +The puzzle becaming much stranger some five minutes later. +Austin U. S. Secret Service Agent Al Soliz, accompanied by +University of Texas campus-security officer Larry Coutorie +and the ubiquitous Tim Foley, made their appearance at Izenberg's door. +They were in plain clothes: slacks, polo shirts. They came in, +and Tim Foley accused Izenberg of belonging to the Legion of Doom. + +Izenberg told them that he had never heard of the "Legion of Doom." +And what about a certain stolen E911 Document, that posed a direct +threat to the police emergency lines? Izenberg claimed that he'd +never heard of that, either. + +His interrogators found this difficult to believe. +Didn't he know Terminus? + +Who? + +They gave him Terminus's real name. Oh yes, said Izenberg. +He knew THAT guy all right--he was leading discussions +on the Internet about AT&T computers, especially the AT&T 3B2. + +AT&T had thrust this machine into the marketplace, +but, like many of AT&T's ambitious attempts to enter +the computing arena, the 3B2 project had something less +than a glittering success. Izenberg himself had been +a contractor for the division of AT&T that supported the 3B2. +The entire division had been shut down. + +Nowadays, the cheapest and quickest way to get help with this +fractious piece of machinery was to join one of Terminus's +discussion groups on the Internet, where friendly and knowledgeable +hackers would help you for free. Naturally the remarks within this +group were less than flattering about the Death Star. . .was +THAT the problem? + +Foley told Izenberg that Terminus had been acquiring hot software +through his, Izenberg's, machine. + +Izenberg shrugged this off. A good eight megabytes of data flowed +through his UUCP site every day. UUCP nodes spewed data like fire hoses. +Elephant had been directly linked to Netsys--not surprising, since Terminus +was a 3B2 expert and Izenberg had been a 3B2 contractor. +Izenberg was also linked to "attctc" and the University of Texas. +Terminus was a well-known UNIX expert, and might have been up to +all manner of hijinks on Elephant. Nothing Izenberg could do about that. +That was physically impossible. Needle in a haystack. + +In a four-hour grilling, Foley urged Izenberg to come clean +and admit that he was in conspiracy with Terminus, +and a member of the Legion of Doom. + +Izenberg denied this. He was no weirdo teenage hacker-- +he was thirty-two years old, and didn't even have a "handle." +Izenberg was a former TV technician and electronics specialist +who had drifted into UNIX consulting as a full-grown adult. +Izenberg had never met Terminus, physically. He'd once bought +a cheap high-speed modem from him, though. + +Foley told him that this modem (a Telenet T2500 which ran at 19.2 kilobaud, +and which had just gone out Izenberg's door in Secret Service custody) +was likely hot property. Izenberg was taken aback to hear this; but then +again, most of Izenberg's equipment, like that of most freelance professionals +in the industry, was discounted, passed hand-to-hand through various kinds +of barter and gray-market. There was no proof that the modem was stolen, +and even if it were, Izenberg hardly saw how that gave them the right +to take every electronic item in his house. + +Still, if the United States Secret Service figured they needed +his computer for national security reasons--or whatever-- +then Izenberg would not kick. He figured he would somehow +make the sacrifice of his twenty thousand dollars' worth +of professional equipment, in the spirit of full cooperation +and good citizenship. + +Robert Izenberg was not arrested. Izenberg was not charged with any crime. +His UUCP node--full of some 140 megabytes of the files, mail, and data +of himself and his dozen or so entirely innocent users--went out the door +as "evidence." Along with the disks and tapes, Izenberg had lost about +800 megabytes of data. + +Six months would pass before Izenberg decided to phone the Secret Service +and ask how the case was going. That was the first time that Robert Izenberg +would ever hear the name of William Cook. As of January 1992, a full +two years after the seizure, Izenberg, still not charged with any crime, +would be struggling through the morass of the courts, in hope of recovering +his thousands of dollars' worth of seized equipment. + +In the meantime, the Izenberg case received absolutely no press coverage. +The Secret Service had walked into an Austin home, removed a UNIX bulletin- +board system, and met with no operational difficulties whatsoever. + +Except that word of a crackdown had percolated through the Legion of Doom. +"The Mentor" voluntarily shut down "The Phoenix Project." It seemed a pity, +especially as telco security employees had, in fact, shown up on Phoenix, +just as he had hoped--along with the usual motley crowd of LoD heavies, +hangers-on, phreaks, hackers and wannabes. There was "Sandy" Sandquist from +US SPRINT security, and some guy named Henry Kluepfel, from Bellcore itself! +Kluepfel had been trading friendly banter with hackers on Phoenix since +January 30th (two weeks after the Martin Luther King Day Crash). +The presence of such a stellar telco official seemed quite the coup +for Phoenix Project. + +Still, Mentor could judge the climate. Atlanta in ruins, +Phrack in deep trouble, something weird going on with UNIX nodes-- +discretion was advisable. Phoenix Project went off-line. + +Kluepfel, of course, had been monitoring this LoD bulletin +board for his own purposes--and those of the Chicago unit. +As far back as June 1987, Kluepfel had logged on to a Texas +underground board called "Phreak Klass 2600." There he'd +discovered an Chicago youngster named "Shadowhawk," +strutting and boasting about rifling AT&T computer files, +and bragging of his ambitions to riddle AT&T's Bellcore +computers with trojan horse programs. Kluepfel had passed +the news to Cook in Chicago, Shadowhawk's computers +had gone out the door in Secret Service custody, +and Shadowhawk himself had gone to jail. + +Now it was Phoenix Project's turn. Phoenix Project postured +about "legality" and "merely intellectual interest," but it reeked +of the underground. It had Phrack on it. It had the E911 Document. +It had a lot of dicey talk about breaking into systems, including some +bold and reckless stuff about a supposed "decryption service" that Mentor +and friends were planning to run, to help crack encrypted passwords off +of hacked systems. + +Mentor was an adult. There was a bulletin board at his place of work, +as well. Kleupfel logged onto this board, too, and discovered it to be +called "Illuminati." It was run by some company called Steve Jackson Games. + +On March 1, 1990, the Austin crackdown went into high gear. + +On the morning of March 1--a Thursday--21-year-old University of Texas +student "Erik Bloodaxe," co-sysop of Phoenix Project and an avowed member +of the Legion of Doom, was wakened by a police revolver levelled at his head. + +Bloodaxe watched, jittery, as Secret Service agents +appropriated his 300 baud terminal and, rifling his files, +discovered his treasured source-code for Robert Morris's +notorious Internet Worm. But Bloodaxe, a wily operator, +had suspected that something of the like might be coming. +All his best equipment had been hidden away elsewhere. +The raiders took everything electronic, however, +including his telephone. They were stymied by his +hefty arcade-style Pac-Man game, and left it in place, +as it was simply too heavy to move. + +Bloodaxe was not arrested. He was not charged with any crime. +A good two years later, the police still had what they had +taken from him, however. + +The Mentor was less wary. The dawn raid rousted him and his wife +from bed in their underwear, and six Secret Service agents, +accompanied by an Austin policeman and Henry Kluepfel himself, +made a rich haul. Off went the works, into the agents' white +Chevrolet minivan: an IBM PC-AT clone with 4 meg of RAM and +a 120-meg hard disk; a Hewlett-Packard LaserJet II printer; +a completely legitimate and highly expensive SCO-Xenix 286 +operating system; Pagemaker disks and documentation; +and the Microsoft Word word-processing program. Mentor's wife +had her incomplete academic thesis stored on the hard-disk; +that went, too, and so did the couple's telephone. As of two years later, +all this property remained in police custody. + +Mentor remained under guard in his apartment as agents prepared +to raid Steve Jackson Games. The fact that this was a business +headquarters and not a private residence did not deter the agents. +It was still very early; no one was at work yet. The agents prepared +to break down the door, but Mentor, eavesdropping on the Secret Service +walkie-talkie traffic, begged them not to do it, and offered his key +to the building. + +The exact details of the next events are unclear. The agents +would not let anyone else into the building. Their search warrant, +when produced, was unsigned. Apparently they breakfasted from the local +"Whataburger," as the litter from hamburgers was later found inside. +They also extensively sampled a bag of jellybeans kept by an SJG employee. +Someone tore a "Dukakis for President" sticker from the wall. + +SJG employees, diligently showing up for the day's work, were met +at the door and briefly questioned by U.S. Secret Service agents. +The employees watched in astonishment as agents wielding crowbars +and screwdrivers emerged with captive machines. They attacked +outdoor storage units with boltcutters. The agents wore +blue nylon windbreakers with "SECRET SERVICE" stencilled +across the back, with running-shoes and jeans. + +Jackson's company lost three computers, several hard-disks, +hundred of floppy disks, two monitors, three modems, +a laser printer, various powercords, cables, and adapters +(and, oddly, a small bag of screws, bolts and nuts). +The seizure of Illuminati BBS deprived SJG of all the programs, +text files, and private e-mail on the board. The loss of two other +SJG computers was a severe blow as well, since it caused the loss +of electronically stored contracts, financial projections, +address directories, mailing lists, personnel files, +business correspondence, and, not least, the drafts +of forthcoming games and gaming books. + +No one at Steve Jackson Games was arrested. No one was accused +of any crime. No charges were filed. Everything appropriated +was officially kept as "evidence" of crimes never specified. + +After the Phrack show-trial, the Steve Jackson Games scandal +was the most bizarre and aggravating incident of the Hacker +Crackdown of 1990. This raid by the Chicago Task Force +on a science-fiction gaming publisher was to rouse a +swarming host of civil liberties issues, and gave rise +to an enduring controversy that was still re-complicating itself, +and growing in the scope of its implications, a full two years later. + +The pursuit of the E911 Document stopped with the Steve Jackson Games raid. +As we have seen, there were hundreds, perhaps thousands of computer users +in America with the E911 Document in their possession. Theoretically, +Chicago had a perfect legal right to raid any of these people, +and could have legally seized the machines of anybody who subscribed to Phrack. +However, there was no copy of the E911 Document on Jackson's Illuminati board. +And there the Chicago raiders stopped dead; they have not raided anyone since. + +It might be assumed that Rich Andrews and Charlie Boykin, who had brought +the E911 Document to the attention of telco security, might be spared +any official suspicion. But as we have seen, the willingness to +"cooperate fully" offers little, if any, assurance against federal +anti-hacker prosecution. + +Richard Andrews found himself in deep trouble, thanks to the E911 Document. +Andrews lived in Illinois, the native stomping grounds of the Chicago +Task Force. On February 3 and 6, both his home and his place of work +were raided by USSS. His machines went out the door, too, and he was +grilled at length (though not arrested). Andrews proved to be in +purportedly guilty possession of: UNIX SVR 3.2; UNIX SVR 3.1; UUCP; +PMON; WWB; IWB; DWB; NROFF; KORN SHELL '88; C++; and QUEST, +among other items. Andrews had received this proprietary code-- +which AT&T officially valued at well over $250,000--through the +UNIX network, much of it supplied to him as a personal favor by Terminus. +Perhaps worse yet, Andrews admitted to returning the favor, by passing +Terminus a copy of AT&T proprietary STARLAN source code. + +Even Charles Boykin, himself an AT&T employee, entered some very hot water. +By 1990, he'd almost forgotten about the E911 problem he'd reported in +September 88; in fact, since that date, he'd passed two more security alerts +to Jerry Dalton, concerning matters that Boykin considered far worse than +the E911 Document. + +But by 1990, year of the crackdown, AT&T Corporate Information Security +was fed up with "Killer." This machine offered no direct income to AT&T, +and was providing aid and comfort to a cloud of suspicious yokels +from outside the company, some of them actively malicious toward AT&T, +its property, and its corporate interests. Whatever goodwill and publicity +had been won among Killer's 1,500 devoted users was considered no longer +worth the security risk. On February 20, 1990, Jerry Dalton arrived in +Dallas and simply unplugged the phone jacks, to the puzzled alarm +of Killer's many Texan users. Killer went permanently off-line, +with the loss of vast archives of programs and huge quantities +of electronic mail; it was never restored to service. AT&T showed +no particular regard for the "property" of these 1,500 people. +Whatever "property" the users had been storing on AT&T's computer +simply vanished completely. + +Boykin, who had himself reported the E911 problem, +now found himself under a cloud of suspicion. In a weird +private-security replay of the Secret Service seizures, +Boykin's own home was visited by AT&T Security and his +own machines were carried out the door. + +However, there were marked special features in the Boykin case. +Boykin's disks and his personal computers were swiftly examined +by his corporate employers and returned politely in just two days-- +(unlike Secret Service seizures, which commonly take months or years). +Boykin was not charged with any crime or wrongdoing, and he kept his job +with AT&T (though he did retire from AT&T in September 1991, +at the age of 52). + +It's interesting to note that the US Secret Service somehow failed +to seize Boykin's "Killer" node and carry AT&T's own computer out the door. +Nor did they raid Boykin's home. They seemed perfectly willing to take the +word of AT&T Security that AT&T's employee, and AT&T's "Killer" node, +were free of hacker contraband and on the up-and-up. + +It's digital water-under-the-bridge at this point, as Killer's +3,200 megabytes of Texan electronic community were erased in 1990, +and "Killer" itself was shipped out of the state. + +But the experiences of Andrews and Boykin, and the users of their systems, +remained side issues. They did not begin to assume the social, political, +and legal importance that gathered, slowly but inexorably, around the issue +of the raid on Steve Jackson Games. + +# + +We must now turn our attention to Steve Jackson Games itself, +and explain what SJG was, what it really did, and how it had +managed to attract this particularly odd and virulent kind of trouble. +The reader may recall that this is not the first but the second time +that the company has appeared in this narrative; a Steve Jackson game +called GURPS was a favorite pastime of Atlanta hacker Urvile, +and Urvile's science-fictional gaming notes had been mixed up +promiscuously with notes about his actual computer intrusions. + +First, Steve Jackson Games, Inc., was NOT a publisher of "computer games." +SJG published "simulation games," parlor games that were played on paper, +with pencils, and dice, and printed guidebooks full of rules and +statistics tables. There were no computers involved in the games themselves. +When you bought a Steve Jackson Game, you did not receive any software disks. +What you got was a plastic bag with some cardboard game tokens, +maybe a few maps or a deck of cards. Most of their products were books. + +However, computers WERE deeply involved in the Steve Jackson Games business. +Like almost all modern publishers, Steve Jackson and his fifteen employees +used computers to write text, to keep accounts, and to run the business +generally. They also used a computer to run their official bulletin board +system for Steve Jackson Games, a board called Illuminati. On Illuminati, +simulation gamers who happened to own computers and modems could associate, +trade mail, debate the theory and practice of gaming, and keep up with the +company's news and its product announcements. + +Illuminati was a modestly popular board, run on a small computer +with limited storage, only one phone-line, and no ties to large-scale +computer networks. It did, however, have hundreds of users, +many of them dedicated gamers willing to call from out-of-state. + +Illuminati was NOT an "underground" board. It did not feature hints +on computer intrusion, or "anarchy files," or illicitly posted +credit card numbers, or long-distance access codes. +Some of Illuminati's users, however, were members of the Legion of Doom. +And so was one of Steve Jackson's senior employees--the Mentor. +The Mentor wrote for Phrack, and also ran an underground board, +Phoenix Project--but the Mentor was not a computer professional. +The Mentor was the managing editor of Steve Jackson Games and +a professional game designer by trade. These LoD members did not +use Illuminati to help their HACKING activities. They used it to +help their GAME-PLAYING activities--and they were even more dedicated +to simulation gaming than they were to hacking. + +"Illuminati" got its name from a card-game that Steve Jackson himself, +the company's founder and sole owner, had invented. This multi-player +card-game was one of Mr Jackson's best-known, most successful, +most technically innovative products. "Illuminati" was a game +of paranoiac conspiracy in which various antisocial cults warred +covertly to dominate the world. "Illuminati" was hilarious, +and great fun to play, involving flying saucers, the CIA, the KGB, +the phone companies, the Ku Klux Klan, the South American Nazis, +the cocaine cartels, the Boy Scouts, and dozens of other splinter groups +from the twisted depths of Mr. Jackson's professionally fervid imagination. +For the uninitiated, any public discussion of the "Illuminati" card-game +sounded, by turns, utterly menacing or completely insane. + +And then there was SJG's "Car Wars," in which souped-up armored hot-rods +with rocket-launchers and heavy machine-guns did battle on the American +highways of the future. The lively Car Wars discussion on the Illuminati +board featured many meticulous, painstaking discussions of the effects +of grenades, land-mines, flamethrowers and napalm. It sounded like +hacker anarchy files run amuck. + +Mr Jackson and his co-workers earned their daily bread by supplying people +with make-believe adventures and weird ideas. The more far-out, the better. + +Simulation gaming is an unusual pastime, but gamers have not +generally had to beg the permission of the Secret Service to exist. +Wargames and role-playing adventures are an old and honored pastime, +much favored by professional military strategists. Once little-known, +these games are now played by hundreds of thousands of enthusiasts +throughout North America, Europe and Japan. Gaming-books, once restricted +to hobby outlets, now commonly appear in chain-stores like B. Dalton's +and Waldenbooks, and sell vigorously. + +Steve Jackson Games, Inc., of Austin, Texas, was a games company +of the middle rank. In 1989, SJG grossed about a million dollars. +Jackson himself had a good reputation in his industry as a talented +and innovative designer of rather unconventional games, but his company +was something less than a titan of the field--certainly not like the +multimillion-dollar TSR Inc., or Britain's gigantic "Games Workshop." +SJG's Austin headquarters was a modest two-story brick office-suite, +cluttered with phones, photocopiers, fax machines and computers. +It bustled with semi-organized activity and was littered with +glossy promotional brochures and dog-eared science-fiction novels. +Attached to the offices was a large tin-roofed warehouse piled twenty feet +high with cardboard boxes of games and books. Despite the weird imaginings +that went on within it, the SJG headquarters was quite a quotidian, +everyday sort of place. It looked like what it was: a publishers' digs. + +Both "Car Wars" and "Illuminati" were well-known, popular games. +But the mainstay of the Jackson organization was their Generic Universal +Role-Playing System, "G.U.R.P.S." The GURPS system was considered solid +and well-designed, an asset for players. But perhaps the most popular +feature of the GURPS system was that it allowed gaming-masters to design +scenarios that closely resembled well-known books, movies, and other works +of fantasy. Jackson had licensed and adapted works from many science fiction +and fantasy authors. There was GURPS Conan, GURPS Riverworld, +GURPS Horseclans, GURPS Witch World, names eminently familiar +to science-fiction readers. And there was GURPS Special Ops, +from the world of espionage fantasy and unconventional warfare. + +And then there was GURPS Cyberpunk. + +"Cyberpunk" was a term given to certain science fiction writers +who had entered the genre in the 1980s. "Cyberpunk," as the label implies, +had two general distinguishing features. First, its writers had a compelling +interest in information technology, an interest closely akin +to science fiction's earlier fascination with space travel. +And second, these writers were "punks," with all the +distinguishing features that that implies: Bohemian artiness, +youth run wild, an air of deliberate rebellion, funny clothes and hair, +odd politics, a fondness for abrasive rock and roll; in a word, trouble. + +The "cyberpunk" SF writers were a small group of mostly college-educated +white middle-class litterateurs, scattered through the US and Canada. +Only one, Rudy Rucker, a professor of computer science in Silicon Valley, +could rank with even the humblest computer hacker. But, except for +Professor Rucker, the "cyberpunk" authors were not programmers +or hardware experts; they considered themselves artists +(as, indeed, did Professor Rucker). However, these writers +all owned computers, and took an intense and public interest +in the social ramifications of the information industry. + +The cyberpunks had a strong following among the global generation +that had grown up in a world of computers, multinational networks, +and cable television. Their outlook was considered somewhat morbid, +cynical, and dark, but then again, so was the outlook of their +generational peers. As that generation matured and increased +in strength and influence, so did the cyberpunks. +As science-fiction writers went, they were doing +fairly well for themselves. By the late 1980s, +their work had attracted attention from gaming companies, +including Steve Jackson Games, which was planning a cyberpunk +simulation for the flourishing GURPS gaming-system. + +The time seemed ripe for such a product, which had already been proven +in the marketplace. The first games- company out of the gate, +with a product boldly called "Cyberpunk" in defiance of possible +infringement-of-copyright suits, had been an upstart group called +R. Talsorian. Talsorian's Cyberpunk was a fairly decent game, +but the mechanics of the simulation system left a lot to be desired. +Commercially, however, the game did very well. + +The next cyberpunk game had been the even more successful Shadowrun +by FASA Corporation. The mechanics of this game were fine, but the +scenario was rendered moronic by sappy fantasy elements like elves, +trolls, wizards, and dragons--all highly ideologically-incorrect, +according to the hard-edged, high-tech standards of cyberpunk science fiction. + +Other game designers were champing at the bit. Prominent among them +was the Mentor, a gentleman who, like most of his friends in the +Legion of Doom, was quite the cyberpunk devotee. Mentor reasoned +that the time had come for a REAL cyberpunk gaming-book--one that the +princes of computer-mischief in the Legion of Doom could play without +laughing themselves sick. This book, GURPS Cyberpunk, would reek +of culturally on-line authenticity. + +Mentor was particularly well-qualified for this task. +Naturally, he knew far more about computer-intrusion +and digital skullduggery than any previously published +cyberpunk author. Not only that, but he was good at his work. +A vivid imagination, combined with an instinctive feeling +for the working of systems and, especially, the loopholes +within them, are excellent qualities for a professional game designer. + +By March 1st, GURPS Cyberpunk was almost complete, ready to print and ship. +Steve Jackson expected vigorous sales for this item, which, he hoped, +would keep the company financially afloat for several months. +GURPS Cyberpunk, like the other GURPS "modules," was not a "game" +like a Monopoly set, but a BOOK: a bound paperback book the size +of a glossy magazine, with a slick color cover, and pages full of text, +illustrations, tables and footnotes. It was advertised as a game, +and was used as an aid to game-playing, but it was a book, +with an ISBN number, published in Texas, copyrighted, +and sold in bookstores. + +And now, that book, stored on a computer, had gone out the door +in the custody of the Secret Service. + +The day after the raid, Steve Jackson visited the local Secret Service +headquarters with a lawyer in tow. There he confronted Tim Foley +(still in Austin at that time) and demanded his book back. But there +was trouble. GURPS Cyberpunk, alleged a Secret Service agent to astonished +businessman Steve Jackson, was "a manual for computer crime." + +"It's science fiction," Jackson said. + +"No, this is real." + +This statement was repeated several times, by several agents. +Jackson's ominously accurate game had passed from pure, +obscure, small-scale fantasy into the impure, highly publicized, +large-scale fantasy of the Hacker Crackdown. + +No mention was made of the real reason for the search. +According to their search warrant, the raiders had expected +to find the E911 Document stored on Jackson's bulletin board system. +But that warrant was sealed; a procedure that most law enforcement agencies +will use only when lives are demonstrably in danger. The raiders' +true motives were not discovered until the Jackson search-warrant +was unsealed by his lawyers, many months later. The Secret Service, +and the Chicago Computer Fraud and Abuse Task Force, +said absolutely nothing to Steve Jackson about any threat +to the police 911 System. They said nothing about the Atlanta Three, +nothing about Phrack or Knight Lightning, nothing about Terminus. + +Jackson was left to believe that his computers had been seized because +he intended to publish a science fiction book that law enforcement +considered too dangerous to see print. + +This misconception was repeated again and again, for months, +to an ever-widening public audience. It was not the truth of the case; +but as months passed, and this misconception was publicly printed again +and again, it became one of the few publicly known "facts" about +the mysterious Hacker Crackdown. The Secret Service had seized a computer +to stop the publication of a cyberpunk science fiction book. + +The second section of this book, "The Digital Underground," +is almost finished now. We have become acquainted with all +the major figures of this case who actually belong to the +underground milieu of computer intrusion. We have some idea +of their history, their motives, their general modus operandi. +We now know, I hope, who they are, where they came from, +and more or less what they want. In the next section of this book, +"Law and Order," we will leave this milieu and directly enter the +world of America's computer-crime police. + +At this point, however, I have another figure to introduce: myself. + +My name is Bruce Sterling. I live in Austin, Texas, where I am +a science fiction writer by trade: specifically, a CYBERPUNK +science fiction writer. + +Like my "cyberpunk" colleagues in the U.S. and Canada, +I've never been entirely happy with this literary label-- +especially after it became a synonym for computer criminal. +But I did once edit a book of stories by my colleagues, +called Mirrorshades: the Cyberpunk Anthology, and I've +long been a writer of literary-critical cyberpunk manifestos. +I am not a "hacker" of any description, though I do have readers +in the digital underground. + +When the Steve Jackson Games seizure occurred, I naturally took +an intense interest. If "cyberpunk" books were being banned +by federal police in my own home town, I reasonably wondered +whether I myself might be next. Would my computer be seized +by the Secret Service? At the time, I was in possession +of an aging Apple IIe without so much as a hard disk. +If I were to be raided as an author of computer-crime manuals, +the loss of my feeble word-processor would likely provoke more +snickers than sympathy. + +I'd known Steve Jackson for many years. We knew +one another as colleagues, for we frequented +the same local science-fiction conventions. +I'd played Jackson games, and recognized his cleverness; +but he certainly had never struck me as a potential mastermind +of computer crime. + +I also knew a little about computer bulletin-board systems. +In the mid-1980s I had taken an active role in an Austin board +called "SMOF-BBS," one of the first boards dedicated to science fiction. +I had a modem, and on occasion I'd logged on to Illuminati, +which always looked entertainly wacky, but certainly harmless enough. + +At the time of the Jackson seizure, I had no experience +whatsoever with underground boards. But I knew that no one +on Illuminati talked about breaking into systems illegally, +or about robbing phone companies. Illuminati didn't even +offer pirated computer games. Steve Jackson, like many creative artists, +was markedly touchy about theft of intellectual property. + +It seemed to me that Jackson was either seriously suspected +of some crime--in which case, he would be charged soon, +and would have his day in court--or else he was innocent, +in which case the Secret Service would quickly return his equipment, +and everyone would have a good laugh. I rather expected the good laugh. +The situation was not without its comic side. The raid, known +as the "Cyberpunk Bust" in the science fiction community, +was winning a great deal of free national publicity both +for Jackson himself and the "cyberpunk" science fiction +writers generally. + +Besides, science fiction people are used to being misinterpreted. +Science fiction is a colorful, disreputable, slipshod occupation, +full of unlikely oddballs, which, of course, is why we like it. +Weirdness can be an occupational hazard in our field. People who +wear Halloween costumes are sometimes mistaken for monsters. + +Once upon a time--back in 1939, in New York City-- +science fiction and the U.S. Secret Service collided in +a comic case of mistaken identity. This weird incident +involved a literary group quite famous in science fiction, +known as "the Futurians," whose membership included +such future genre greats as Isaac Asimov, Frederik Pohl, +and Damon Knight. The Futurians were every bit as +offbeat and wacky as any of their spiritual descendants, +including the cyberpunks, and were given to communal living, +spontaneous group renditions of light opera, and midnight fencing +exhibitions on the lawn. The Futurians didn't have bulletin +board systems, but they did have the technological equivalent +in 1939--mimeographs and a private printing press. These were +in steady use, producing a stream of science-fiction fan magazines, +literary manifestos, and weird articles, which were picked up +in ink-sticky bundles by a succession of strange, gangly, +spotty young men in fedoras and overcoats. + +The neighbors grew alarmed at the antics of the Futurians +and reported them to the Secret Service as suspected counterfeiters. +In the winter of 1939, a squad of USSS agents with drawn guns burst into +"Futurian House," prepared to confiscate the forged currency and illicit +printing presses. There they discovered a slumbering science fiction fan +named George Hahn, a guest of the Futurian commune who had just arrived +in New York. George Hahn managed to explain himself and his group, +and the Secret Service agents left the Futurians in peace henceforth. +(Alas, Hahn died in 1991, just before I had discovered this astonishing +historical parallel, and just before I could interview him for this book.) + +But the Jackson case did not come to a swift and comic end. +No quick answers came his way, or mine; no swift reassurances +that all was right in the digital world, that matters were well +in hand after all. Quite the opposite. In my alternate role +as a sometime pop-science journalist, I interviewed Jackson +and his staff for an article in a British magazine. +The strange details of the raid left me more concerned than ever. +Without its computers, the company had been financially +and operationally crippled. Half the SJG workforce, +a group of entirely innocent people, had been sorrowfully fired, +deprived of their livelihoods by the seizure. It began to dawn on me +that authors--American writers--might well have their computers seized, +under sealed warrants, without any criminal charge; and that, +as Steve Jackson had discovered, there was no immediate recourse for this. +This was no joke; this wasn't science fiction; this was real. + +I determined to put science fiction aside until I had discovered +what had happened and where this trouble had come from. +It was time to enter the purportedly real world of electronic +free expression and computer crime. Hence, this book. +Hence, the world of the telcos; and the world of the digital underground; +and next, the world of the police. + + + +PART THREE: LAW AND ORDER + + +Of the various anti-hacker activities of 1990, "Operation Sundevil" +had by far the highest public profile. The sweeping, nationwide +computer seizures of May 8, 1990 were unprecedented in scope and highly, +if rather selectively, publicized. + +Unlike the efforts of the Chicago Computer Fraud and Abuse Task Force, +"Operation Sundevil" was not intended to combat "hacking" in the sense +of computer intrusion or sophisticated raids on telco switching stations. +Nor did it have anything to do with hacker misdeeds with AT&T's software, +or with Southern Bell's proprietary documents. + +Instead, "Operation Sundevil" was a crackdown on those traditional scourges +of the digital underground: credit-card theft and telephone code abuse. +The ambitious activities out of Chicago, and the somewhat lesser-known +but vigorous anti-hacker actions of the New York State Police in 1990, +were never a part of "Operation Sundevil" per se, which was based in Arizona. + +Nevertheless, after the spectacular May 8 raids, the public, misled by +police secrecy, hacker panic, and a puzzled national press-corps, +conflated all aspects of the nationwide crackdown in 1990 under +the blanket term "Operation Sundevil." "Sundevil" is still the best-known +synonym for the crackdown of 1990. But the Arizona organizers of "Sundevil" +did not really deserve this reputation--any more, for instance, than all +hackers deserve a reputation as "hackers." + +There was some justice in this confused perception, though. +For one thing, the confusion was abetted by the Washington office +of the Secret Service, who responded to Freedom of Information Act +requests on "Operation Sundevil" by referring investigators +to the publicly known cases of Knight Lightning and the Atlanta Three. +And "Sundevil" was certainly the largest aspect of the Crackdown, +the most deliberate and the best-organized. As a crackdown on electronic +fraud, "Sundevil" lacked the frantic pace of the war on the Legion of Doom; +on the contrary, Sundevil's targets were picked out with cool deliberation +over an elaborate investigation lasting two full years. + +And once again the targets were bulletin board systems. + +Boards can be powerful aids to organized fraud. Underground boards carry +lively, extensive, detailed, and often quite flagrant "discussions" of +lawbreaking techniques and lawbreaking activities. "Discussing" crime +in the abstract, or "discussing" the particulars of criminal cases, +is not illegal--but there are stern state and federal laws against +coldbloodedly conspiring in groups in order to commit crimes. + +In the eyes of police, people who actively conspire to break the law +are not regarded as "clubs," "debating salons," "users' groups," or +"free speech advocates." Rather, such people tend to find themselves +formally indicted by prosecutors as "gangs," "racketeers," "corrupt +organizations" and "organized crime figures." + +What's more, the illicit data contained on outlaw boards goes well beyond +mere acts of speech and/or possible criminal conspiracy. As we have seen, +it was common practice in the digital underground to post purloined telephone +codes on boards, for any phreak or hacker who cared to abuse them. Is posting +digital booty of this sort supposed to be protected by the First Amendment? +Hardly--though the issue, like most issues in cyberspace, is not entirely +resolved. Some theorists argue that to merely RECITE a number publicly +is not illegal--only its USE is illegal. But anti-hacker police point out +that magazines and newspapers (more traditional forms of free expression) +never publish stolen telephone codes (even though this might well +raise their circulation). + +Stolen credit card numbers, being riskier and more valuable, +were less often publicly posted on boards--but there is no question +that some underground boards carried "carding" traffic, +generally exchanged through private mail. + +Underground boards also carried handy programs for "scanning" telephone +codes and raiding credit card companies, as well as the usual obnoxious +galaxy of pirated software, cracked passwords, blue-box schematics, +intrusion manuals, anarchy files, porn files, and so forth. + +But besides their nuisance potential for the spread of illicit knowledge, +bulletin boards have another vitally interesting aspect for the +professional investigator. Bulletin boards are cram-full of EVIDENCE. +All that busy trading of electronic mail, all those hacker boasts, +brags and struts, even the stolen codes and cards, can be neat, +electronic, real-time recordings of criminal activity. +As an investigator, when you seize a pirate board, you have +scored a coup as effective as tapping phones or intercepting mail. +However, you have not actually tapped a phone or intercepted a letter. +The rules of evidence regarding phone-taps and mail interceptions are old, +stern and well-understood by police, prosecutors and defense attorneys alike. +The rules of evidence regarding boards are new, waffling, and understood +by nobody at all. + +Sundevil was the largest crackdown on boards in world history. +On May 7, 8, and 9, 1990, about forty-two computer systems were seized. +Of those forty-two computers, about twenty-five actually were running boards. +(The vagueness of this estimate is attributable to the vagueness of +(a) what a "computer system" is, and (b) what it actually means to +"run a board" with one--or with two computers, or with three.) + +About twenty-five boards vanished into police custody in May 1990. +As we have seen, there are an estimated 30,000 boards in America today. +If we assume that one board in a hundred is up to no good with codes +and cards (which rather flatters the honesty of the board-using community), +then that would leave 2,975 outlaw boards untouched by Sundevil. +Sundevil seized about one tenth of one percent of all computer +bulletin boards in America. Seen objectively, this is something less +than a comprehensive assault. In 1990, Sundevil's organizers-- +the team at the Phoenix Secret Service office, and the Arizona +Attorney General's office-- had a list of at least THREE HUNDRED +boards that they considered fully deserving of search and seizure warrants. +The twenty-five boards actually seized were merely among the most obvious +and egregious of this much larger list of candidates. All these boards +had been examined beforehand--either by informants, who had passed printouts +to the Secret Service, or by Secret Service agents themselves, who not only +come equipped with modems but know how to use them. + +There were a number of motives for Sundevil. First, it offered +a chance to get ahead of the curve on wire-fraud crimes. +Tracking back credit-card ripoffs to their perpetrators +can be appallingly difficult. If these miscreants +have any kind of electronic sophistication, they can snarl +their tracks through the phone network into a mind-boggling, +untraceable mess, while still managing to "reach out and rob someone." +Boards, however, full of brags and boasts, codes and cards, +offer evidence in the handy congealed form. + +Seizures themselves--the mere physical removal of machines-- +tends to take the pressure off. During Sundevil, a large number +of code kids, warez d00dz, and credit card thieves would be deprived +of those boards--their means of community and conspiracy--in one swift blow. +As for the sysops themselves (commonly among the boldest offenders) +they would be directly stripped of their computer equipment, +and rendered digitally mute and blind. + +And this aspect of Sundevil was carried out with great success. +Sundevil seems to have been a complete tactical surprise-- +unlike the fragmentary and continuing seizures of the war on the +Legion of Doom, Sundevil was precisely timed and utterly overwhelming. +At least forty "computers" were seized during May 7, 8 and 9, 1990, +in Cincinnati, Detroit, Los Angeles, Miami, Newark, Phoenix, Tucson, +Richmond, San Diego, San Jose, Pittsburgh and San Francisco. +Some cities saw multiple raids, such as the five separate raids +in the New York City environs. Plano, Texas (essentially a suburb of +the Dallas/Fort Worth metroplex, and a hub of the telecommunications industry) +saw four computer seizures. Chicago, ever in the forefront, saw its own +local Sundevil raid, briskly carried out by Secret Service agents +Timothy Foley and Barbara Golden. + +Many of these raids occurred, not in the cities proper, +but in associated white-middle class suburbs--places like +Mount Lebanon, Pennsylvania and Clark Lake, Michigan. +There were a few raids on offices; most took place in people's homes, +the classic hacker basements and bedrooms. + +The Sundevil raids were searches and seizures, not a group of mass arrests. +There were only four arrests during Sundevil. "Tony the Trashman," +a longtime teenage bete noire of the Arizona Racketeering unit, +was arrested in Tucson on May 9. "Dr. Ripco," sysop of an outlaw board +with the misfortune to exist in Chicago itself, was also arrested-- +on illegal weapons charges. Local units also arrested a 19-year-old +female phone phreak named "Electra" in Pennsylvania, and a male juvenile +in California. Federal agents however were not seeking arrests, but computers. + +Hackers are generally not indicted (if at all) until the evidence +in their seized computers is evaluated--a process that can take weeks, +months--even years. When hackers are arrested on the spot, it's generally +an arrest for other reasons. Drugs and/or illegal weapons show up in a good +third of anti-hacker computer seizures (though not during Sundevil). + +That scofflaw teenage hackers (or their parents) should have marijuana +in their homes is probably not a shocking revelation, but the surprisingly +common presence of illegal firearms in hacker dens is a bit disquieting. +A Personal Computer can be a great equalizer for the techno-cowboy-- +much like that more traditional American "Great Equalizer," +the Personal Sixgun. Maybe it's not all that surprising +that some guy obsessed with power through illicit technology +would also have a few illicit high-velocity-impact devices around. +An element of the digital underground particularly dotes on those +"anarchy philes," and this element tends to shade into the crackpot milieu +of survivalists, gun-nuts, anarcho-leftists and the ultra-libertarian +right-wing. + +This is not to say that hacker raids to date have uncovered any +major crack-dens or illegal arsenals; but Secret Service agents +do not regard "hackers" as "just kids." They regard hackers as +unpredictable people, bright and slippery. It doesn't help matters +that the hacker himself has been "hiding behind his keyboard" +all this time. Commonly, police have no idea what he looks like. +This makes him an unknown quantity, someone best treated with +proper caution. + +To date, no hacker has come out shooting, though they do sometimes brag on +boards that they will do just that. Threats of this sort are taken seriously. +Secret Service hacker raids tend to be swift, comprehensive, well-manned +(even over-manned); and agents generally burst through every door +in the home at once, sometimes with drawn guns. Any potential resistance +is swiftly quelled. Hacker raids are usually raids on people's homes. +It can be a very dangerous business to raid an American home; +people can panic when strangers invade their sanctum. Statistically speaking, +the most dangerous thing a policeman can do is to enter someone's home. +(The second most dangerous thing is to stop a car in traffic.) +People have guns in their homes. More cops are hurt in homes +than are ever hurt in biker bars or massage parlors. + +But in any case, no one was hurt during Sundevil, +or indeed during any part of the Hacker Crackdown. + +Nor were there any allegations of any physical mistreatment of a suspect. +Guns were pointed, interrogations were sharp and prolonged; but no one +in 1990 claimed any act of brutality by any crackdown raider. + +In addition to the forty or so computers, Sundevil reaped floppy disks +in particularly great abundance--an estimated 23,000 of them, which +naturally included every manner of illegitimate data: pirated games, +stolen codes, hot credit card numbers, the complete text and software +of entire pirate bulletin-boards. These floppy disks, which remain +in police custody today, offer a gigantic, almost embarrassingly +rich source of possible criminal indictments. These 23,000 floppy disks +also include a thus-far unknown quantity of legitimate computer games, +legitimate software, purportedly "private" mail from boards, +business records, and personal correspondence of all kinds. + +Standard computer-crime search warrants lay great emphasis on seizing +written documents as well as computers--specifically including photocopies, +computer printouts, telephone bills, address books, logs, notes, +memoranda and correspondence. In practice, this has meant that diaries, +gaming magazines, software documentation, nonfiction books on hacking +and computer security, sometimes even science fiction novels, have all +vanished out the door in police custody. A wide variety of electronic items +have been known to vanish as well, including telephones, televisions, answering +machines, Sony Walkmans, desktop printers, compact disks, and audiotapes. + +No fewer than 150 members of the Secret Service were sent into +the field during Sundevil. They were commonly accompanied by +squads of local and/or state police. Most of these officers-- +especially the locals--had never been on an anti-hacker raid before. +(This was one good reason, in fact, why so many of them were invited along +in the first place.) Also, the presence of a uniformed police officer +assures the raidees that the people entering their homes are, in fact, police. +Secret Service agents wear plain clothes. So do the telco security experts +who commonly accompany the Secret Service on raids (and who make no particular +effort to identify themselves as mere employees of telephone companies). + +A typical hacker raid goes something like this. First, police storm in +rapidly, through every entrance, with overwhelming force, +in the assumption that this tactic will keep casualties to a minimum. +Second, possible suspects are immediately removed from the vicinity +of any and all computer systems, so that they will have no chance +to purge or destroy computer evidence. Suspects are herded into a room +without computers, commonly the living room, and kept under guard-- +not ARMED guard, for the guns are swiftly holstered, but under guard +nevertheless. They are presented with the search warrant and warned +that anything they say may be held against them. Commonly they have +a great deal to say, especially if they are unsuspecting parents. + +Somewhere in the house is the "hot spot"--a computer tied to a phone +line (possibly several computers and several phones). Commonly it's +a teenager's bedroom, but it can be anywhere in the house; +there may be several such rooms. This "hot spot" is put in charge +of a two-agent team, the "finder" and the "recorder." The "finder" +is computer-trained, commonly the case agent who has actually obtained +the search warrant from a judge. He or she understands what is being sought, +and actually carries out the seizures: unplugs machines, opens drawers, +desks, files, floppy-disk containers, etc. The "recorder" photographs +all the equipment, just as it stands--especially the tangle of +wired connections in the back, which can otherwise be a real nightmare +to restore. The recorder will also commonly photograph every room +in the house, lest some wily criminal claim that the police had robbed him +during the search. Some recorders carry videocams or tape recorders; +however, it's more common for the recorder to simply take written notes. +Objects are described and numbered as the finder seizes them, generally +on standard preprinted police inventory forms. + +Even Secret Service agents were not, and are not, expert computer users. +They have not made, and do not make, judgements on the fly about potential +threats posed by various forms of equipment. They may exercise discretion; +they may leave Dad his computer, for instance, but they don't HAVE to. +Standard computer-crime search warrants, which date back to the early 80s, +use a sweeping language that targets computers, most anything attached +to a computer, most anything used to operate a computer--most anything +that remotely resembles a computer--plus most any and all written documents +surrounding it. Computer-crime investigators have strongly urged agents +to seize the works. + +In this sense, Operation Sundevil appears to have been a complete success. +Boards went down all over America, and were shipped en masse to the computer +investigation lab of the Secret Service, in Washington DC, along with the +23,000 floppy disks and unknown quantities of printed material. + +But the seizure of twenty-five boards, and the multi-megabyte mountains +of possibly useful evidence contained in these boards (and in their owners' +other computers, also out the door), were far from the only motives for +Operation Sundevil. An unprecedented action of great ambition and size, +Sundevil's motives can only be described as political. It was a +public-relations effort, meant to pass certain messages, meant to make +certain situations clear: both in the mind of the general public, +and in the minds of various constituencies of the electronic community. + + First --and this motivation was vital--a "message" would be sent from +law enforcement to the digital underground. This very message was recited +in so many words by Garry M. Jenkins, the Assistant Director of the +US Secret Service, at the Sundevil press conference in Phoenix on +May 9, 1990, immediately after the raids. In brief, hackers were +mistaken in their foolish belief that they could hide behind the +"relative anonymity of their computer terminals." On the contrary, +they should fully understand that state and federal cops were +actively patrolling the beat in cyberspace--that they were +on the watch everywhere, even in those sleazy and secretive +dens of cybernetic vice, the underground boards. + +This is not an unusual message for police to publicly convey to crooks. +The message is a standard message; only the context is new. + +In this respect, the Sundevil raids were the digital equivalent +of the standard vice-squad crackdown on massage parlors, porno bookstores, +head-shops, or floating crap-games. There may be few or no arrests in a raid +of this sort; no convictions, no trials, no interrogations. In cases of this +sort, police may well walk out the door with many pounds of sleazy magazines, +X-rated videotapes, sex toys, gambling equipment, baggies of marijuana. . . . + +Of course, if something truly horrendous is discovered by the raiders, +there will be arrests and prosecutions. Far more likely, however, +there will simply be a brief but sharp disruption of the closed +and secretive world of the nogoodniks. There will be "street hassle." +"Heat." "Deterrence." And, of course, the immediate loss of the seized goods. +It is very unlikely that any of this seized material will ever be returned. +Whether charged or not, whether convicted or not, the perpetrators will +almost surely lack the nerve ever to ask for this stuff to be given back. + +Arrests and trials--putting people in jail--may involve all kinds of +formal legalities; but dealing with the justice system is far from the only +task of police. Police do not simply arrest people. They don't simply +put people in jail. That is not how the police perceive their jobs. +Police "protect and serve." Police "keep the peace," they "keep public order." +Like other forms of public relations, keeping public order is not an +exact science. Keeping public order is something of an art-form. + +If a group of tough-looking teenage hoodlums was loitering on a street-corner, +no one would be surprised to see a street-cop arrive and sternly order +them to "break it up." On the contrary, the surprise would come if one +of these ne'er-do-wells stepped briskly into a phone-booth, +called a civil rights lawyer, and instituted a civil suit +in defense of his Constitutional rights of free speech +and free assembly. But something much along this line +was one of the many anomolous outcomes of the Hacker Crackdown. + +Sundevil also carried useful "messages" for other constituents of +the electronic community. These messages may not have been read +aloud from the Phoenix podium in front of the press corps, +but there was little mistaking their meaning. There was a message +of reassurance for the primary victims of coding and carding: +the telcos, and the credit companies. Sundevil was greeted with joy +by the security officers of the electronic business community. +After years of high-tech harassment and spiralling revenue losses, +their complaints of rampant outlawry were being taken seriously by +law enforcement. No more head-scratching or dismissive shrugs; +no more feeble excuses about "lack of computer-trained officers" or +the low priority of "victimless" white-collar telecommunication crimes. + +Computer-crime experts have long believed that computer-related offenses +are drastically under-reported. They regard this as a major open scandal +of their field. Some victims are reluctant to come forth, because they +believe that police and prosecutors are not computer-literate, +and can and will do nothing. Others are embarrassed by +their vulnerabilities, and will take strong measures +to avoid any publicity; this is especially true of banks, +who fear a loss of investor confidence should an embezzlement-case +or wire-fraud surface. And some victims are so helplessly confused +by their own high technology that they never even realize that +a crime has occurred--even when they have been fleeced to the bone. + +The results of this situation can be dire. +Criminals escape apprehension and punishment. +The computer-crime units that do exist, can't get work. +The true scope of computer-crime: its size, its real nature, +the scope of its threats, and the legal remedies for it-- +all remain obscured. + +Another problem is very little publicized, but it is a cause +of genuine concern. Where there is persistent crime, +but no effective police protection, then vigilantism can result. +Telcos, banks, credit companies, the major corporations who +maintain extensive computer networks vulnerable to hacking +--these organizations are powerful, wealthy, and +politically influential. They are disinclined to be +pushed around by crooks (or by most anyone else, +for that matter). They often maintain well-organized +private security forces, commonly run by +experienced veterans of military and police units, +who have left public service for the greener pastures +of the private sector. For police, the corporate +security manager can be a powerful ally; but if this +gentleman finds no allies in the police, and the +pressure is on from his board-of-directors, +he may quietly take certain matters into his own hands. + +Nor is there any lack of disposable hired-help in the +corporate security business. Private security agencies-- +the `security business' generally--grew explosively in the 1980s. +Today there are spooky gumshoed armies of "security consultants," +"rent-a- cops," "private eyes," "outside experts"--every manner +of shady operator who retails in "results" and discretion. +Or course, many of these gentlemen and ladies may be paragons +of professional and moral rectitude. But as anyone +who has read a hard-boiled detective novel knows, +police tend to be less than fond of this sort +of private-sector competition. + +Companies in search of computer-security have even been +known to hire hackers. Police shudder at this prospect. + +Police treasure good relations with the business community. +Rarely will you see a policeman so indiscreet as to allege +publicly that some major employer in his state or city has succumbed +to paranoia and gone off the rails. Nevertheless, +police --and computer police in particular--are aware +of this possibility. Computer-crime police can and do +spend up to half of their business hours just doing +public relations: seminars, "dog and pony shows," +sometimes with parents' groups or computer users, +but generally with their core audience: the likely +victims of hacking crimes. These, of course, are telcos, +credit card companies and large computer-equipped corporations. +The police strongly urge these people, as good citizens, +to report offenses and press criminal charges; +they pass the message that there is someone in authority who cares, +understands, and, best of all, will take useful action +should a computer-crime occur. + +But reassuring talk is cheap. Sundevil offered action. + +The final message of Sundevil was intended for internal consumption +by law enforcement. Sundevil was offered as proof that the community +of American computer-crime police had come of age. Sundevil was +proof that enormous things like Sundevil itself could now be accomplished. +Sundevil was proof that the Secret Service and its local law-enforcement +allies could act like a well-oiled machine--(despite the hampering use +of those scrambled phones). It was also proof that the Arizona Organized +Crime and Racketeering Unit--the sparkplug of Sundevil--ranked with the best +in the world in ambition, organization, and sheer conceptual daring. + +And, as a final fillip, Sundevil was a message from the Secret Service +to their longtime rivals in the Federal Bureau of Investigation. +By Congressional fiat, both USSS and FBI formally share jurisdiction +over federal computer-crimebusting activities. Neither of these groups +has ever been remotely happy with this muddled situation. It seems to +suggest that Congress cannot make up its mind as to which of these groups +is better qualified. And there is scarcely a G-man or a Special Agent +anywhere without a very firm opinion on that topic. + +# + +For the neophyte, one of the most puzzling aspects of the crackdown +on hackers is why the United States Secret Service has anything at all +to do with this matter. + +The Secret Service is best known for its primary public role: +its agents protect the President of the United States. +They also guard the President's family, the Vice President and his family, +former Presidents, and Presidential candidates. They sometimes guard +foreign dignitaries who are visiting the United States, especially foreign +heads of state, and have been known to accompany American officials +on diplomatic missions overseas. + +Special Agents of the Secret Service don't wear uniforms, but the +Secret Service also has two uniformed police agencies. There's the +former White House Police (now known as the Secret Service Uniformed Division, +since they currently guard foreign embassies in Washington, as well as the +White House itself). And there's the uniformed Treasury Police Force. + +The Secret Service has been charged by Congress with a number +of little-known duties. They guard the precious metals in Treasury vaults. +They guard the most valuable historical documents of the United States: +originals of the Constitution, the Declaration of Independence, +Lincoln's Second Inaugural Address, an American-owned copy of +the Magna Carta, and so forth. Once they were assigned to guard +the Mona Lisa, on her American tour in the 1960s. + +The entire Secret Service is a division of the Treasury Department. +Secret Service Special Agents (there are about 1,900 of them) +are bodyguards for the President et al, but they all work for the Treasury. +And the Treasury (through its divisions of the U.S. Mint and the +Bureau of Engraving and Printing) prints the nation's money. + +As Treasury police, the Secret Service guards the nation's currency; +it is the only federal law enforcement agency with direct jurisdiction +over counterfeiting and forgery. It analyzes documents for authenticity, +and its fight against fake cash is still quite lively (especially since +the skilled counterfeiters of Medellin, Columbia have gotten into the act). +Government checks, bonds, and other obligations, which exist in untold +millions and are worth untold billions, are common targets for forgery, +which the Secret Service also battles. It even handles forgery +of postage stamps. + +But cash is fading in importance today as money has become electronic. +As necessity beckoned, the Secret Service moved from fighting the +counterfeiting of paper currency and the forging of checks, +to the protection of funds transferred by wire. + +From wire-fraud, it was a simple skip-and-jump to what is formally +known as "access device fraud." Congress granted the Secret Service +the authority to investigate "access device fraud" under Title 18 +of the United States Code (U.S.C. Section 1029). + +The term "access device" seems intuitively simple. It's some kind +of high-tech gizmo you use to get money with. It makes good sense +to put this sort of thing in the charge of counterfeiting and +wire-fraud experts. + +However, in Section 1029, the term "access device" is very +generously defined. An access device is: "any card, plate, +code, account number, or other means of account access +that can be used, alone or in conjunction with another access device, +to obtain money, goods, services, or any other thing of value, +or that can be used to initiate a transfer of funds." + +"Access device" can therefore be construed to include credit cards +themselves (a popular forgery item nowadays). It also includes credit card +account NUMBERS, those standards of the digital underground. The same goes +for telephone charge cards (an increasingly popular item with telcos, +who are tired of being robbed of pocket change by phone-booth thieves). +And also telephone access CODES, those OTHER standards of the digital +underground. (Stolen telephone codes may not "obtain money," but they +certainly do obtain valuable "services," which is specifically forbidden +by Section 1029.) + +We can now see that Section 1029 already pits the United States Secret Service +directly against the digital underground, without any mention at all of +the word "computer." + +Standard phreaking devices, like "blue boxes," used to steal phone service +from old-fashioned mechanical switches, are unquestionably "counterfeit +access devices." Thanks to Sec.1029, it is not only illegal to USE +counterfeit access devices, but it is even illegal to BUILD them. +"Producing," "designing" "duplicating" or "assembling" blue boxes +are all federal crimes today, and if you do this, the Secret Service +has been charged by Congress to come after you. + +Automatic Teller Machines, which replicated all over America during the 1980s, +are definitely "access devices," too, and an attempt to tamper with their +punch-in codes and plastic bank cards falls directly under Sec. 1029. + +Section 1029 is remarkably elastic. Suppose you find a computer password +in somebody's trash. That password might be a "code"--it's certainly a +"means of account access." Now suppose you log on to a computer +and copy some software for yourself. You've certainly obtained +"service" (computer service) and a "thing of value" (the software). +Suppose you tell a dozen friends about your swiped password, +and let them use it, too. Now you're "trafficking in unauthorized +access devices." And when the Prophet, a member of the Legion of Doom, +passed a stolen telephone company document to Knight Lightning +at Phrack magazine, they were both charged under Sec. 1029! + +There are two limitations on Section 1029. First, the offense must +"affect interstate or foreign commerce" in order to become a matter +of federal jurisdiction. The term "affecting commerce" is not well defined; +but you may take it as a given that the Secret Service can take an interest +if you've done most anything that happens to cross a state line. +State and local police can be touchy about their jurisdictions, +and can sometimes be mulish when the feds show up. But when it comes +to computer-crime, the local police are pathetically grateful +for federal help--in fact they complain that they can't get enough of it. +If you're stealing long-distance service, you're almost certainly crossing +state lines, and you're definitely "affecting the interstate commerce" +of the telcos. And if you're abusing credit cards by ordering stuff +out of glossy catalogs from, say, Vermont, you're in for it. + +The second limitation is money. As a rule, the feds don't pursue +penny-ante offenders. Federal judges will dismiss cases that appear +to waste their time. Federal crimes must be serious; Section 1029 +specifies a minimum loss of a thousand dollars. + +We now come to the very next section of Title 18, which is Section 1030, +"Fraud and related activity in connection with computers." This statute +gives the Secret Service direct jurisdiction over acts of computer intrusion. +On the face of it, the Secret Service would now seem to command the field. +Section 1030, however, is nowhere near so ductile as Section 1029. + +The first annoyance is Section 1030(d), which reads: + +"(d) The United States Secret Service shall, +IN ADDITION TO ANY OTHER AGENCY HAVING SUCH AUTHORITY, +have the authority to investigate offenses under this section. +Such authority of the United States Secret Service shall be +exercised in accordance with an agreement which shall be entered +into by the Secretary of the Treasury AND THE ATTORNEY GENERAL." +(Author's italics.) [Represented by capitals.] + +The Secretary of the Treasury is the titular head of the Secret Service, +while the Attorney General is in charge of the FBI. In Section (d), +Congress shrugged off responsibility for the computer-crime turf-battle +between the Service and the Bureau, and made them fight it out all +by themselves. The result was a rather dire one for the Secret Service, +for the FBI ended up with exclusive jurisdiction over computer break-ins +having to do with national security, foreign espionage, federally insured +banks, and U.S. military bases, while retaining joint jurisdiction over +all the other computer intrusions. Essentially, when it comes to Section 1030, +the FBI not only gets the real glamor stuff for itself, but can peer over the +shoulder of the Secret Service and barge in to meddle whenever it suits them. + +The second problem has to do with the dicey term +"Federal interest computer." Section 1030(a)(2) +makes it illegal to "access a computer without authorization" +if that computer belongs to a financial institution or an issuer +of credit cards (fraud cases, in other words). Congress was quite +willing to give the Secret Service jurisdiction over +money-transferring computers, but Congress balked at +letting them investigate any and all computer intrusions. +Instead, the USSS had to settle for the money machines +and the "Federal interest computers." A "Federal interest computer" +is a computer which the government itself owns, or is using. +Large networks of interstate computers, linked over state lines, +are also considered to be of "Federal interest." (This notion of +"Federal interest" is legally rather foggy and has never been +clearly defined in the courts. The Secret Service has never yet +had its hand slapped for investigating computer break-ins that were NOT +of "Federal interest," but conceivably someday this might happen.) + +So the Secret Service's authority over "unauthorized access" +to computers covers a lot of territory, but by no means the +whole ball of cyberspatial wax. If you are, for instance, +a LOCAL computer retailer, or the owner of a LOCAL bulletin +board system, then a malicious LOCAL intruder can break in, +crash your system, trash your files and scatter viruses, +and the U.S. Secret Service cannot do a single thing about it. + +At least, it can't do anything DIRECTLY. But the Secret Service +will do plenty to help the local people who can. + +The FBI may have dealt itself an ace off the bottom of the deck +when it comes to Section 1030; but that's not the whole story; +that's not the street. What's Congress thinks is one thing, +and Congress has been known to change its mind. The REAL +turf-struggle is out there in the streets where it's happening. +If you're a local street-cop with a computer problem, +the Secret Service wants you to know where you can find +the real expertise. While the Bureau crowd are off having +their favorite shoes polished--(wing-tips)--and making derisive +fun of the Service's favorite shoes--("pansy-ass tassels")-- +the tassel-toting Secret Service has a crew of ready-and-able +hacker-trackers installed in the capital of every state in the Union. +Need advice? They'll give you advice, or at least point you in +the right direction. Need training? They can see to that, too. + +If you're a local cop and you call in the FBI, the FBI +(as is widely and slanderously rumored) will order you around +like a coolie, take all the credit for your busts, +and mop up every possible scrap of reflected glory. +The Secret Service, on the other hand, doesn't brag a lot. +They're the quiet types. VERY quiet. Very cool. Efficient. +High-tech. Mirrorshades, icy stares, radio ear-plugs, +an Uzi machine-pistol tucked somewhere in that well-cut jacket. +American samurai, sworn to give their lives to protect our President. +"The granite agents." Trained in martial arts, absolutely fearless. +Every single one of 'em has a top-secret security clearance. +Something goes a little wrong, you're not gonna hear any whining +and moaning and political buck-passing out of these guys. + +The facade of the granite agent is not, of course, the reality. +Secret Service agents are human beings. And the real glory +in Service work is not in battling computer crime--not yet, +anyway--but in protecting the President. The real glamour +of Secret Service work is in the White House Detail. +If you're at the President's side, then the kids and the wife +see you on television; you rub shoulders with the most powerful +people in the world. That's the real heart of Service work, +the number one priority. More than one computer investigation +has stopped dead in the water when Service agents vanished at +the President's need. + +There's romance in the work of the Service. The intimate access +to circles of great power; the esprit-de-corps of a highly trained +and disciplined elite; the high responsibility of defending the +Chief Executive; the fulfillment of a patriotic duty. And as police +work goes, the pay's not bad. But there's squalor in Service work, too. +You may get spat upon by protesters howling abuse--and if they get violent, +if they get too close, sometimes you have to knock one of them down-- +discreetly. + +The real squalor in Service work is drudgery such as "the quarterlies," +traipsing out four times a year, year in, year out, to interview the various +pathetic wretches, many of them in prisons and asylums, who have seen fit +to threaten the President's life. And then there's the grinding stress +of searching all those faces in the endless bustling crowds, looking for +hatred, looking for psychosis, looking for the tight, nervous face +of an Arthur Bremer, a Squeaky Fromme, a Lee Harvey Oswald. +It's watching all those grasping, waving hands for sudden movements, +while your ears strain at your radio headphone for the long-rehearsed +cry of "Gun!" + +It's poring, in grinding detail, over the biographies of every rotten +loser who ever shot at a President. It's the unsung work of the +Protective Research Section, who study scrawled, anonymous death threats +with all the meticulous tools of anti-forgery techniques. + +And it's maintaining the hefty computerized files on anyone +who ever threatened the President's life. Civil libertarians +have become increasingly concerned at the Government's use +of computer files to track American citizens--but the +Secret Service file of potential Presidential assassins, +which has upward of twenty thousand names, rarely causes +a peep of protest. If you EVER state that you intend to +kill the President, the Secret Service will want to know +and record who you are, where you are, what you are, +and what you're up to. If you're a serious threat-- +if you're officially considered "of protective interest"-- +then the Secret Service may well keep tabs on you +for the rest of your natural life. + +Protecting the President has first call on all the Service's resources. +But there's a lot more to the Service's traditions and history than +standing guard outside the Oval Office. + +The Secret Service is the nation's oldest general federal +law-enforcement agency. Compared to the Secret Service, +the FBI are new-hires and the CIA are temps. The Secret Service +was founded 'way back in 1865, at the suggestion of Hugh McCulloch, +Abraham Lincoln's Secretary of the Treasury. McCulloch wanted +a specialized Treasury police to combat counterfeiting. +Abraham Lincoln agreed that this seemed a good idea, and, +with a terrible irony, Abraham Lincoln was shot that +very night by John Wilkes Booth. + +The Secret Service originally had nothing to do with protecting Presidents. +They didn't take this on as a regular assignment until after the Garfield +assassination in 1881. And they didn't get any Congressional money for it +until President McKinley was shot in 1901. The Service was originally +designed for one purpose: destroying counterfeiters. + +# + +There are interesting parallels between the Service's +nineteenth-century entry into counterfeiting, +and America's twentieth-century entry into computer-crime. + +In 1865, America's paper currency was a terrible muddle. +Security was drastically bad. Currency was printed on the spot +by local banks in literally hundreds of different designs. +No one really knew what the heck a dollar bill was supposed to look like. +Bogus bills passed easily. If some joker told you that a one-dollar bill +from the Railroad Bank of Lowell, Massachusetts had a woman leaning on +a shield, with a locomotive, a cornucopia, a compass, various agricultural +implements, a railroad bridge, and some factories, then you pretty much had +to take his word for it. (And in fact he was telling the truth!) + +SIXTEEN HUNDRED local American banks designed and printed their own +paper currency, and there were no general standards for security. +Like a badly guarded node in a computer network, badly designed bills +were easy to fake, and posed a security hazard for the entire monetary system. + +No one knew the exact extent of the threat to the currency. +There were panicked estimates that as much as a third of +the entire national currency was faked. Counterfeiters-- +known as "boodlers" in the underground slang of the time-- +were mostly technically skilled printers who had gone to the bad. +Many had once worked printing legitimate currency. +Boodlers operated in rings and gangs. Technical experts +engraved the bogus plates--commonly in basements in New York City. +Smooth confidence men passed large wads of high-quality, +high-denomination fakes, including the really sophisticated stuff-- +government bonds, stock certificates, and railway shares. +Cheaper, botched fakes were sold or sharewared to low-level +gangs of boodler wannabes. (The really cheesy lowlife boodlers +merely upgraded real bills by altering face values, +changing ones to fives, tens to hundreds, and so on.) + +The techniques of boodling were little-known and regarded +with a certain awe by the mid- nineteenth-century public. +The ability to manipulate the system for rip-off seemed +diabolically clever. As the skill and daring of the +boodlers increased, the situation became intolerable. +The federal government stepped in, and began offering +its own federal currency, which was printed in fancy green ink, +but only on the back--the original "greenbacks." And at first, +the improved security of the well-designed, well-printed +federal greenbacks seemed to solve the problem; but then +the counterfeiters caught on. Within a few years things were +worse than ever: a CENTRALIZED system where ALL security was bad! + +The local police were helpless. The Government tried offering +blood money to potential informants, but this met with little success. +Banks, plagued by boodling, gave up hope of police help and hired +private security men instead. Merchants and bankers queued up +by the thousands to buy privately-printed manuals on currency security, +slim little books like Laban Heath's INFALLIBLE GOVERNMENT +COUNTERFEIT DETECTOR. The back of the book offered Laban Heath's +patent microscope for five bucks. + +Then the Secret Service entered the picture. The first agents +were a rough and ready crew. Their chief was one William P. Wood, +a former guerilla in the Mexican War who'd won a reputation busting +contractor fraudsters for the War Department during the Civil War. +Wood, who was also Keeper of the Capital Prison, had a sideline +as a counterfeiting expert, bagging boodlers for the federal bounty money. + +Wood was named Chief of the new Secret Service in July 1865. +There were only ten Secret Service agents in all: Wood himself, +a handful who'd worked for him in the War Department, and a few +former private investigators--counterfeiting experts--whom Wood +had won over to public service. (The Secret Service of 1865 was +much the size of the Chicago Computer Fraud Task Force or the +Arizona Racketeering Unit of 1990.) These ten "Operatives" +had an additional twenty or so "Assistant Operatives" and "Informants." +Besides salary and per diem, each Secret Service employee received +a whopping twenty-five dollars for each boodler he captured. + +Wood himself publicly estimated that at least HALF of America's currency +was counterfeit, a perhaps pardonable perception. Within a year the +Secret Service had arrested over 200 counterfeiters. They busted about +two hundred boodlers a year for four years straight. + +Wood attributed his success to travelling fast and light, hitting the +bad-guys hard, and avoiding bureaucratic baggage. "Because my raids +were made without military escort and I did not ask the assistance +of state officers, I surprised the professional counterfeiter." + +Wood's social message to the once-impudent boodlers bore an eerie ring +of Sundevil: "It was also my purpose to convince such characters that +it would no longer be healthy for them to ply their vocation without +being handled roughly, a fact they soon discovered." + +William P. Wood, the Secret Service's guerilla pioneer, +did not end well. He succumbed to the lure of aiming for +the really big score. The notorious Brockway Gang of New York City, +headed by William E. Brockway, the "King of the Counterfeiters," +had forged a number of government bonds. They'd passed these +brilliant fakes on the prestigious Wall Street investment +firm of Jay Cooke and Company. The Cooke firm were frantic +and offered a huge reward for the forgers' plates. + +Laboring diligently, Wood confiscated the plates +(though not Mr. Brockway) and claimed the reward. +But the Cooke company treacherously reneged. +Wood got involved in a down-and-dirty lawsuit +with the Cooke capitalists. Wood's boss, +Secretary of the Treasury McCulloch, felt that +Wood's demands for money and glory were unseemly, +and even when the reward money finally came through, +McCulloch refused to pay Wood anything. +Wood found himself mired in a seemingly endless +round of federal suits and Congressional lobbying. + +Wood never got his money. And he lost his job to boot. +He resigned in 1869. + +Wood's agents suffered, too. On May 12, 1869, the second Chief +of the Secret Service took over, and almost immediately fired +most of Wood's pioneer Secret Service agents: Operatives, +Assistants and Informants alike. The practice of receiving $25 +per crook was abolished. And the Secret Service began the long, +uncertain process of thorough professionalization. + +Wood ended badly. He must have felt stabbed in the back. +In fact his entire organization was mangled. + +On the other hand, William P. Wood WAS the first head of the Secret Service. +William Wood was the pioneer. People still honor his name. Who remembers +the name of the SECOND head of the Secret Service? + +As for William Brockway (also known as "Colonel Spencer"), +he was finally arrested by the Secret Service in 1880. +He did five years in prison, got out, and was still boodling +at the age of seventy-four. + +# + +Anyone with an interest in Operation Sundevil-- +or in American computer-crime generally-- +could scarcely miss the presence of Gail Thackeray, +Assistant Attorney General of the State of Arizona. +Computer-crime training manuals often cited +Thackeray's group and her work; she was the +highest-ranking state official to specialize +in computer-related offenses. Her name had been +on the Sundevil press release (though modestly ranked +well after the local federal prosecuting attorney and +the head of the Phoenix Secret Service office). + +As public commentary, and controversy, began to mount +about the Hacker Crackdown, this Arizonan state official +began to take a higher and higher public profile. +Though uttering almost nothing specific about +the Sundevil operation itself, she coined some +of the most striking soundbites of the growing propaganda war: +"Agents are operating in good faith, and I don't think +you can say that for the hacker community," was one. +Another was the memorable "I am not a mad dog prosecutor" +(Houston Chronicle, Sept 2, 1990.) In the meantime, +the Secret Service maintained its usual extreme discretion; +the Chicago Unit, smarting from the backlash +of the Steve Jackson scandal, had gone completely to earth. + +As I collated my growing pile of newspaper clippings, +Gail Thackeray ranked as a comparative fount of public +knowledge on police operations. + +I decided that I had to get to know Gail Thackeray. +I wrote to her at the Arizona Attorney General's Office. +Not only did she kindly reply to me, but, to my astonishment, +she knew very well what "cyberpunk" science fiction was. + +Shortly after this, Gail Thackeray lost her job. +And I temporarily misplaced my own career as +a science-fiction writer, to become a full-time +computer-crime journalist. In early March, 1991, +I flew to Phoenix, Arizona, to interview Gail Thackeray +for my book on the hacker crackdown. + +# + +"Credit cards didn't used to cost anything to get," +says Gail Thackeray. "Now they cost forty bucks-- +and that's all just to cover the costs from RIP-OFF ARTISTS." + +Electronic nuisance criminals are parasites. +One by one they're not much harm, no big deal. +But they never come just one by one. They come in swarms, +heaps, legions, sometimes whole subcultures. And they bite. +Every time we buy a credit card today, we lose a little financial +vitality to a particular species of bloodsucker. + +What, in her expert opinion, are the worst forms of electronic crime, +I ask, consulting my notes. Is it--credit card fraud? Breaking into +ATM bank machines? Phone-phreaking? Computer intrusions? +Software viruses? Access-code theft? Records tampering? +Software piracy? Pornographic bulletin boards? +Satellite TV piracy? Theft of cable service? +It's a long list. By the time I reach the end +of it I feel rather depressed. + +"Oh no," says Gail Thackeray, leaning forward over the table, +her whole body gone stiff with energetic indignation, +"the biggest damage is telephone fraud. Fake sweepstakes, +fake charities. Boiler-room con operations. You could pay off +the national debt with what these guys steal. . . . +They target old people, they get hold of credit ratings +and demographics, they rip off the old and the weak." +The words come tumbling out of her. + +It's low-tech stuff, your everyday boiler-room fraud. +Grifters, conning people out of money over the phone, +have been around for decades. This is where the word "phony" came from! + +It's just that it's so much EASIER now, horribly facilitated by advances +in technology and the byzantine structure of the modern phone system. +The same professional fraudsters do it over and over, Thackeray tells me, +they hide behind dense onion-shells of fake companies. . . fake holding +corporations nine or ten layers deep, registered all over the map. +They get a phone installed under a false name in an empty safe-house. +And then they call-forward everything out of that phone to yet +another phone, a phone that may even be in another STATE. +And they don't even pay the charges on their phones; +after a month or so, they just split; set up somewhere else +in another Podunkville with the same seedy crew of veteran phone-crooks. +They buy or steal commercial credit card reports, slap them on the PC, +have a program pick out people over sixty-five who pay a lot to charities. +A whole subculture living off this, merciless folks on the con. + +"The `light-bulbs for the blind' people," Thackeray muses, +with a special loathing. "There's just no end to them." + +We're sitting in a downtown diner in Phoenix, Arizona. +It's a tough town, Phoenix. A state capital seeing some hard times. +Even to a Texan like myself, Arizona state politics seem rather baroque. +There was, and remains, endless trouble over the Martin Luther King holiday, +the sort of stiff-necked, foot-shooting incident for which Arizona politics +seem famous. There was Evan Mecham, the eccentric Republican millionaire +governor who was impeached, after reducing state government to a +ludicrous shambles. Then there was the national Keating scandal, +involving Arizona savings and loans, in which both of Arizona's +U.S. senators, DeConcini and McCain, played sadly prominent roles. + +And the very latest is the bizarre AzScam case, +in which state legislators were videotaped, +eagerly taking cash from an informant of the Phoenix city +police department, who was posing as a Vegas mobster. + +"Oh," says Thackeray cheerfully. "These people are amateurs here, +they thought they were finally getting to play with the big boys. +They don't have the least idea how to take a bribe! +It's not institutional corruption. It's not like back in Philly." + +Gail Thackeray was a former prosecutor in Philadelphia. +Now she's a former assistant attorney general of the State of Arizona. +Since moving to Arizona in 1986, she had worked under the aegis +of Steve Twist, her boss in the Attorney General's office. +Steve Twist wrote Arizona's pioneering computer crime laws +and naturally took an interest in seeing them enforced. +It was a snug niche, and Thackeray's Organized Crime and +Racketeering Unit won a national reputation for ambition +and technical knowledgeability. . . . Until the latest +election in Arizona. Thackeray's boss ran for the top +job, and lost. The victor, the new Attorney General, +apparently went to some pains to eliminate the bureaucratic +traces of his rival, including his pet group--Thackeray's group. +Twelve people got their walking papers. + +Now Thackeray's painstakingly assembled computer lab +sits gathering dust somewhere in the glass-and-concrete +Attorney General's HQ on 1275 Washington Street. +Her computer-crime books, her painstakingly garnered +back issues of phreak and hacker zines, all bought +at her own expense--are piled in boxes somewhere. +The State of Arizona is simply not particularly +interested in electronic racketeering at the moment. + +At the moment of our interview, Gail Thackeray, +officially unemployed, is working out of the county +sheriff's office, living on her savings, and prosecuting +several cases--working 60-hour weeks, just as always-- +for no pay at all. "I'm trying to train people," +she mutters. + +Half her life seems to be spent training people--merely pointing out, +to the naive and incredulous (such as myself) that this stuff +is ACTUALLY GOING ON OUT THERE. It's a small world, computer crime. +A young world. Gail Thackeray, a trim blonde Baby-Boomer who favors +Grand Canyon white-water rafting to kill some slow time, +is one of the world's most senior, most veteran "hacker-trackers." +Her mentor was Donn Parker, the California think-tank theorist +who got it all started `way back in the mid-70s, the "grandfather +of the field," "the great bald eagle of computer crime." + +And what she has learned, Gail Thackeray teaches. Endlessly. +Tirelessly. To anybody. To Secret Service agents and state police, +at the Glynco, Georgia federal training center. To local police, +on "roadshows" with her slide projector and notebook. +To corporate security personnel. To journalists. To parents. + +Even CROOKS look to Gail Thackeray for advice. +Phone-phreaks call her at the office. They know very +well who she is. They pump her for information +on what the cops are up to, how much they know. +Sometimes whole CROWDS of phone phreaks, +hanging out on illegal conference calls, will call Gail +Thackeray up. They taunt her. And, as always, +they boast. Phone-phreaks, real stone phone-phreaks, +simply CANNOT SHUT UP. They natter on for hours. + +Left to themselves, they mostly talk about the intricacies +of ripping-off phones; it's about as interesting as listening +to hot-rodders talk about suspension and distributor-caps. +They also gossip cruelly about each other. And when talking +to Gail Thackeray, they incriminate themselves. "I have tapes," +Thackeray says coolly. + +Phone phreaks just talk like crazy. "Dial-Tone" out in Alabama +has been known to spend half-an-hour simply reading stolen +phone-codes aloud into voice-mail answering machines. +Hundreds, thousands of numbers, recited in a monotone, +without a break--an eerie phenomenon. When arrested, +it's a rare phone phreak who doesn't inform at endless length +on everybody he knows. + +Hackers are no better. What other group of criminals, +she asks rhetorically, publishes newsletters and holds conventions? +She seems deeply nettled by the sheer brazenness of this behavior, +though to an outsider, this activity might make one wonder +whether hackers should be considered "criminals" at all. +Skateboarders have magazines, and they trespass a lot. +Hot rod people have magazines and they break speed limits +and sometimes kill people. . . . + +I ask her whether it would be any loss to society if phone phreaking +and computer hacking, as hobbies, simply dried up and blew away, +so that nobody ever did it again. + +She seems surprised. "No," she says swiftly. "Maybe a little. . . +in the old days. . .the MIT stuff. . . . But there's a lot of wonderful, +legal stuff you can do with computers now, you don't have to break into +somebody else's just to learn. You don't have that excuse. +You can learn all you like." + +Did you ever hack into a system? I ask. + +The trainees do it at Glynco. Just to demonstrate system vulnerabilities. +She's cool to the notion. Genuinely indifferent. + +"What kind of computer do you have?" + +"A Compaq 286LE," she mutters. + +"What kind do you WISH you had?" + +At this question, the unmistakable light of true hackerdom flares in +Gail Thackeray's eyes. She becomes tense, animated, the words pour out: +"An Amiga 2000 with an IBM card and Mac emulation! The most common hacker +machines are Amigas and Commodores. And Apples." If she had the Amiga, +she enthuses, she could run a whole galaxy of seized computer-evidence disks +on one convenient multifunctional machine. A cheap one, too. Not like the +old Attorney General lab, where they had an ancient CP/M machine, +assorted Amiga flavors and Apple flavors, a couple IBMS, all the +utility software. . .but no Commodores. The workstations down +at the Attorney General's are Wang dedicated word-processors. +Lame machines tied in to an office net--though at least they get +on- line to the Lexis and Westlaw legal data services. + +I don't say anything. I recognize the syndrome, though. +This computer-fever has been running through segments of +our society for years now. It's a strange kind of lust: +K-hunger, Meg-hunger; but it's a shared disease; +it can kill parties dead, as conversation spirals into +the deepest and most deviant recesses of software releases +and expensive peripherals. . . . The mark of the hacker beast. +I have it too. The whole "electronic community," whatever the hell +that is, has it. Gail Thackeray has it. Gail Thackeray is a hacker cop. +My immediate reaction is a strong rush of indignant pity: +WHY DOESN'T SOMEBODY BUY THIS WOMAN HER AMIGA?! +It's not like she's asking for a Cray X-MP +supercomputer mainframe; an Amiga's a sweet little +cookie-box thing. We're losing zillions in organized fraud; +prosecuting and defending a single hacker case in court can cost +a hundred grand easy. How come nobody can come up with four lousy grand +so this woman can do her job? For a hundred grand we could buy every +computer cop in America an Amiga. There aren't that many of 'em. + +Computers. The lust, the hunger, for computers. +The loyalty they inspire, the intense sense of possessiveness. +The culture they have bred. I myself am sitting in downtown Phoenix, +Arizona because it suddenly occurred to me that the police might-- +just MIGHT--come and take away my computer. The prospect of this, +the mere IMPLIED THREAT, was unbearable. It literally changed my life. +It was changing the lives of many others. Eventually it would change +everybody's life. + +Gail Thackeray was one of the top computer-crime people in America. +And I was just some novelist, and yet I had a better computer than hers. +PRACTICALLY EVERYBODY I KNEW had a better computer than Gail Thackeray +and her feeble laptop 286. It was like sending the sheriff in to clean +up Dodge City and arming her with a slingshot cut from an old rubber tire. + +But then again, you don't need a howitzer to enforce the law. +You can do a lot just with a badge. With a badge alone, +you can basically wreak havoc, take a terrible vengeance on wrongdoers. +Ninety percent of "computer crime investigation" is just "crime investigation:" +names, places, dossiers, modus operandi, search warrants, victims, +complainants, informants. . . . + +What will computer crime look like in ten years? Will it get better? +Did "Sundevil" send 'em reeling back in confusion? + +It'll be like it is now, only worse, she tells me with perfect conviction. +Still there in the background, ticking along, changing with the times: +the criminal underworld. It'll be like drugs are. Like our problems +with alcohol. All the cops and laws in the world never solved our problems +with alcohol. If there's something people want, a certain percentage +of them are just going to take it. Fifteen percent of the populace +will never steal. Fifteen percent will steal most anything not nailed down. +The battle is for the hearts and minds of the remaining seventy percent. + +And criminals catch on fast. If there's not "too steep a learning curve"-- +if it doesn't require a baffling amount of expertise and practice-- +then criminals are often some of the first through the gate of a +new technology. Especially if it helps them to hide. +They have tons of cash, criminals. The new communications tech-- +like pagers, cellular phones, faxes, Federal Express--were pioneered +by rich corporate people, and by criminals. In the early years +of pagers and beepers, dope dealers were so enthralled this technology +that owing a beeper was practically prima facie evidence of cocaine dealing. +CB radio exploded when the speed limit hit 55 and breaking the highway law +became a national pastime. Dope dealers send cash by Federal Express, +despite, or perhaps BECAUSE OF, the warnings in FedEx offices that tell you +never to try this. Fed Ex uses X-rays and dogs on their mail, +to stop drug shipments. That doesn't work very well. + +Drug dealers went wild over cellular phones. +There are simple methods of faking ID on cellular phones, +making the location of the call mobile, free of charge, +and effectively untraceable. Now victimized cellular +companies routinely bring in vast toll-lists of calls +to Colombia and Pakistan. + +Judge Greene's fragmentation of the phone company +is driving law enforcement nuts. Four thousand +telecommunications companies. Fraud skyrocketing. +Every temptation in the world available with a phone +and a credit card number. Criminals untraceable. +A galaxy of "new neat rotten things to do." + +If there were one thing Thackeray would like to have, +it would be an effective legal end-run through this new +fragmentation minefield. + +It would be a new form of electronic search warrant, +an "electronic letter of marque" to be issued by a judge. +It would create a new category of "electronic emergency." +Like a wiretap, its use would be rare, but it would cut +across state lines and force swift cooperation from all concerned. +Cellular, phone, laser, computer network, PBXes, AT&T, Baby Bells, +long-distance entrepreneurs, packet radio. Some document, +some mighty court-order, that could slice through four thousand +separate forms of corporate red-tape, and get her at once to +the source of calls, the source of email threats and viruses, +the sources of bomb threats, kidnapping threats. "From now on," +she says, "the Lindbergh baby will always die." + +Something that would make the Net sit still, if only for a moment. +Something that would get her up to speed. Seven league boots. +That's what she really needs. "Those guys move in nanoseconds +and I'm on the Pony Express." + +And then, too, there's the coming international angle. +Electronic crime has never been easy to localize, +to tie to a physical jurisdiction. And phone-phreaks +and hackers loathe boundaries, they jump them whenever they can. +The English. The Dutch. And the Germans, especially the ubiquitous +Chaos Computer Club. The Australians. They've all learned phone-phreaking +from America. It's a growth mischief industry. The multinational +networks are global, but governments and the police simply aren't. +Neither are the laws. Or the legal frameworks for citizen protection. + +One language is global, though--English. Phone phreaks speak English; +it's their native tongue even if they're Germans. English may have started +in England but now it's the Net language; it might as well be called "CNNese." + +Asians just aren't much into phone phreaking. They're the world masters +at organized software piracy. The French aren't into phone-phreaking either. +The French are into computerized industrial espionage. + +In the old days of the MIT righteous hackerdom, crashing systems +didn't hurt anybody. Not all that much, anyway. Not permanently. +Now the players are more venal. Now the consequences are worse. +Hacking will begin killing people soon. Already there are methods +of stacking calls onto 911 systems, annoying the police, and possibly +causing the death of some poor soul calling in with a genuine emergency. +Hackers in Amtrak computers, or air-traffic control computers, will kill +somebody someday. Maybe a lot of people. Gail Thackeray expects it. + +And the viruses are getting nastier. The "Scud" virus is the latest one out. +It wipes hard-disks. + +According to Thackeray, the idea that phone-phreaks are Robin Hoods is a fraud. +They don't deserve this repute. Basically, they pick on the weak. AT&T now +protects itself with the fearsome ANI (Automatic Number Identification) +trace capability. When AT&T wised up and tightened security generally, +the phreaks drifted into the Baby Bells. The Baby Bells lashed out in 1989 +and 1990, so the phreaks switched to smaller long-distance entrepreneurs. +Today, they are moving into locally owned PBXes and voice-mail systems, +which are full of security holes, dreadfully easy to hack. These victims +aren't the moneybags Sheriff of Nottingham or Bad King John, but small groups +of innocent people who find it hard to protect themselves, and who really +suffer from these depredations. Phone phreaks pick on the weak. They do it +for power. If it were legal, they wouldn't do it. They don't want service, +or knowledge, they want the thrill of power-tripping. There's plenty of +knowledge or service around if you're willing to pay. Phone phreaks don't pay, +they steal. It's because it is illegal that it feels like power, +that it gratifies their vanity. + +I leave Gail Thackeray with a handshake at the door of her office building-- +a vast International-Style office building downtown. The Sheriff's office +is renting part of it. I get the vague impression that quite a lot of the +building is empty--real estate crash. + +In a Phoenix sports apparel store, in a downtown mall, I meet +the "Sun Devil" himself. He is the cartoon mascot of +Arizona State University, whose football stadium, "Sundevil," +is near the local Secret Service HQ--hence the name Operation Sundevil. +The Sun Devil himself is named "Sparky." Sparky the Sun Devil is maroon +and bright yellow, the school colors. Sparky brandishes a three-tined +yellow pitchfork. He has a small mustache, pointed ears, a barbed tail, +and is dashing forward jabbing the air with the pitchfork, +with an expression of devilish glee. + +Phoenix was the home of Operation Sundevil. The Legion of Doom +ran a hacker bulletin board called "The Phoenix Project." +An Australian hacker named "Phoenix" once burrowed through +the Internet to attack Cliff Stoll, then bragged and boasted +about it to The New York Times. This net of coincidence +is both odd and meaningless. + +The headquarters of the Arizona Attorney General, Gail Thackeray's +former workplace, is on 1275 Washington Avenue. Many of the downtown +streets in Phoenix are named after prominent American presidents: +Washington, Jefferson, Madison. . . . + +After dark, all the employees go home to their suburbs. +Washington, Jefferson and Madison--what would be the +Phoenix inner city, if there were an inner city in this +sprawling automobile-bred town--become the haunts +of transients and derelicts. The homeless. The sidewalks +along Washington are lined with orange trees. +Ripe fallen fruit lies scattered like croquet balls +on the sidewalks and gutters. No one seems to be eating them. +I try a fresh one. It tastes unbearably bitter. + +The Attorney General's office, built in 1981 during the +Babbitt administration, is a long low two-story building +of white cement and wall-sized sheets of curtain-glass. +Behind each glass wall is a lawyer's office, quite open +and visible to anyone strolling by. Across the street +is a dour government building labelled simply ECONOMIC SECURITY, +something that has not been in great supply in the American +Southwest lately. + +The offices are about twelve feet square. They feature +tall wooden cases full of red-spined lawbooks; +Wang computer monitors; telephones; Post-it notes galore. +Also framed law diplomas and a general excess of bad +Western landscape art. Ansel Adams photos are a big favorite, +perhaps to compensate for the dismal specter of the parking lot, +two acres of striped black asphalt, which features gravel landscaping +and some sickly-looking barrel cacti. + +It has grown dark. Gail Thackeray has told me that the people +who work late here, are afraid of muggings in the parking lot. +It seems cruelly ironic that a woman tracing electronic racketeers +across the interstate labyrinth of Cyberspace should fear an assault +by a homeless derelict in the parking lot of her own workplace. + +Perhaps this is less than coincidence. Perhaps these two seemingly +disparate worlds are somehow generating one another. The poor and +disenfranchised take to the streets, while the rich and computer-equipped, +safe in their bedrooms, chatter over their modems. Quite often the derelicts +kick the glass out and break in to the lawyers' offices, if they see something +they need or want badly enough. + +I cross the parking lot to the street behind the Attorney General's office. +A pair of young tramps are bedding down on flattened sheets of cardboard, +under an alcove stretching over the sidewalk. One tramp wears a +glitter-covered T-shirt reading "CALIFORNIA" in Coca-Cola cursive. +His nose and cheeks look chafed and swollen; they glisten with +what seems to be Vaseline. The other tramp has a ragged long-sleeved +shirt and lank brown hair parted in the middle. They both wear blue jeans +coated in grime. They are both drunk. + +"You guys crash here a lot?" I ask them. + +They look at me warily. I am wearing black jeans, a black pinstriped +suit jacket and a black silk tie. I have odd shoes and a funny haircut. + +"It's our first time here," says the red-nosed tramp unconvincingly. +There is a lot of cardboard stacked here. More than any two people could use. + +"We usually stay at the Vinnie's down the street," says the brown-haired tramp, +puffing a Marlboro with a meditative air, as he sprawls with his head on +a blue nylon backpack. "The Saint Vincent's." + +"You know who works in that building over there?" I ask, pointing. + +The brown-haired tramp shrugs. "Some kind of attorneys, it says." + +We urge one another to take it easy. I give them five bucks. + +A block down the street I meet a vigorous workman who is wheeling along +some kind of industrial trolley; it has what appears to be a tank of +propane on it. + +We make eye contact. We nod politely. I walk past him. "Hey! +Excuse me sir!" he says. + +"Yes?" I say, stopping and turning. + +"Have you seen," the guy says rapidly, "a black guy, about 6'7", +scars on both his cheeks like this--" he gestures-- "wears a +black baseball cap on backwards, wandering around here anyplace?" + +"Sounds like I don't much WANT to meet him," I say. + +"He took my wallet," says my new acquaintance. +"Took it this morning. Y'know, some people would be +SCARED of a guy like that. But I'm not scared. +I'm from Chicago. I'm gonna hunt him down. +We do things like that in Chicago." + +"Yeah?" + +"I went to the cops and now he's got an APB out on his ass," +he says with satisfaction. "You run into him, you let me know." + +"Okay," I say. "What is your name, sir?" + +"Stanley. . . ." + +"And how can I reach you?" + +"Oh," Stanley says, in the same rapid voice, +"you don't have to reach, uh, me. +You can just call the cops. Go straight to the cops." +He reaches into a pocket and pulls out a greasy piece of pasteboard. +"See, here's my report on him." + +I look. The "report," the size of an index card, is labelled PRO-ACT: +Phoenix Residents Opposing Active Crime Threat. . . . or is it +Organized Against Crime Threat? In the darkening street it's hard +to read. Some kind of vigilante group? Neighborhood watch? +I feel very puzzled. + +"Are you a police officer, sir?" + +He smiles, seems very pleased by the question. + +"No," he says. + +"But you are a `Phoenix Resident?'" + +"Would you believe a homeless person," Stanley says. + +"Really? But what's with the. . . ." For the first time I take a close look +at Stanley's trolley. It's a rubber-wheeled thing of industrial metal, +but the device I had mistaken for a tank of propane is in fact a water-cooler. +Stanley also has an Army duffel-bag, stuffed tight as a sausage with clothing +or perhaps a tent, and, at the base of his trolley, a cardboard box and a +battered leather briefcase. + +"I see," I say, quite at a loss. For the first time I notice that Stanley +has a wallet. He has not lost his wallet at all. It is in his back pocket +and chained to his belt. It's not a new wallet. It seems to have seen +a lot of wear. + +"Well, you know how it is, brother," says Stanley. +Now that I know that he is homeless--A POSSIBLE +THREAT--my entire perception of him has changed +in an instant. His speech, which once seemed just +bright and enthusiastic, now seems to have a +dangerous tang of mania. "I have to do this!" +he assures me. "Track this guy down. . . . +It's a thing I do. . . you know. . .to keep myself together!" +He smiles, nods, lifts his trolley by its decaying rubber handgrips. + +"Gotta work together, y'know," Stanley booms, his face alight +with cheerfulness, "the police can't do everything!" +The gentlemen I met in my stroll in downtown Phoenix +are the only computer illiterates in this book. +To regard them as irrelevant, however, would be a grave mistake. + +As computerization spreads across society, the populace at large +is subjected to wave after wave of future shock. But, as a +necessary converse, the "computer community" itself is subjected +to wave after wave of incoming computer illiterates. +How will those currently enjoying America's digital bounty regard, +and treat, all this teeming refuse yearning to breathe free? +Will the electronic frontier be another Land of Opportunity-- +or an armed and monitored enclave, where the disenfranchised +snuggle on their cardboard at the locked doors of our houses of justice? + +Some people just don't get along with computers. They can't read. +They can't type. They just don't have it in their heads to master +arcane instructions in wirebound manuals. Somewhere, the process +of computerization of the populace will reach a limit. Some people-- +quite decent people maybe, who might have thrived in any other situation-- +will be left irretrievably outside the bounds. What's to be done with +these people, in the bright new shiny electroworld? How will they +be regarded, by the mouse-whizzing masters of cyberspace? With contempt? +Indifference? Fear? + +In retrospect, it astonishes me to realize how quickly poor Stanley +became a perceived threat. Surprise and fear are closely allied feelings. +And the world of computing is full of surprises. + +I met one character in the streets of Phoenix whose role in this book +is supremely and directly relevant. That personage was Stanley's giant +thieving scarred phantom. This phantasm is everywhere in this book. +He is the specter haunting cyberspace. + +Sometimes he's a maniac vandal ready to smash the phone system +for no sane reason at all. Sometimes he's a fascist fed, +coldly programming his mighty mainframes to destroy our Bill of Rights. +Sometimes he's a telco bureaucrat, covertly conspiring to register all modems +in the service of an Orwellian surveillance regime. Mostly, though, +this fearsome phantom is a "hacker." He's strange, he doesn't belong, +he's not authorized, he doesn't smell right, he's not keeping his proper place, +he's not one of us. The focus of fear is the hacker, for much the same +reasons that Stanley's fancied assailant is black. + +Stanley's demon can't go away, because he doesn't exist. +Despite singleminded and tremendous effort, he can't be arrested, +sued, jailed, or fired. The only constructive way to do ANYTHING +about him is to learn more about Stanley himself. This learning process +may be repellent, it may be ugly, it may involve grave elements of paranoiac +confusion, but it's necessary. Knowing Stanley requires something more +than class-crossing condescension. It requires more than steely +legal objectivity. It requires human compassion and sympathy. + +To know Stanley is to know his demon. If you know the other guy's demon, +then maybe you'll come to know some of your own. You'll be able to +separate reality from illusion. And then you won't do your cause, +and yourself, more harm than good. Like poor damned Stanley from Chicago did. + +# + +The Federal Computer Investigations Committee (FCIC) is the most important +and influential organization in the realm of American computer-crime. +Since the police of other countries have largely taken their computer-crime +cues from American methods, the FCIC might well be called the most important +computer crime group in the world. + +It is also, by federal standards, an organization of great unorthodoxy. +State and local investigators mix with federal agents. Lawyers, +financial auditors and computer-security programmers trade notes +with street cops. Industry vendors and telco security people show up +to explain their gadgetry and plead for protection and justice. +Private investigators, think-tank experts and industry pundits throw in +their two cents' worth. The FCIC is the antithesis of a formal bureaucracy. + +Members of the FCIC are obscurely proud of this fact; they recognize their +group as aberrant, but are entirely convinced that this, for them, +outright WEIRD behavior is nevertheless ABSOLUTELY NECESSARY +to get their jobs done. + +FCIC regulars --from the Secret Service, the FBI, the IRS, +the Department of Labor, the offices of federal attorneys, +state police, the Air Force, from military intelligence-- +often attend meetings, held hither and thither across the country, +at their own expense. The FCIC doesn't get grants. It doesn't +charge membership fees. It doesn't have a boss. It has no headquarters-- +just a mail drop in Washington DC, at the Fraud Division of the Secret Service. +It doesn't have a budget. It doesn't have schedules. It meets three times +a year--sort of. Sometimes it issues publications, but the FCIC +has no regular publisher, no treasurer, not even a secretary. +There are no minutes of FCIC meetings. Non-federal people are considered +"non-voting members," but there's not much in the way of elections. +There are no badges, lapel pins or certificates of membership. +Everyone is on a first-name basis. There are about forty of them. +Nobody knows how many, exactly. People come, people go-- +sometimes people "go" formally but still hang around anyway. +Nobody has ever exactly figured out what "membership" of this +"Committee" actually entails. + +Strange as this may seem to some, to anyone familiar with the social world +of computing, the "organization" of the FCIC is very recognizable. + +For years now, economists and management theorists have speculated +that the tidal wave of the information revolution would destroy rigid, +pyramidal bureaucracies, where everything is top-down and +centrally controlled. Highly trained "employees" would take on +much greater autonomy, being self-starting, and self-motivating, +moving from place to place, task to task, with great speed and fluidity. +"Ad-hocracy" would rule, with groups of people spontaneously knitting +together across organizational lines, tackling the problem at hand, +applying intense computer-aided expertise to it, and then vanishing +whence they came. + +This is more or less what has actually happened in the world of +federal computer investigation. With the conspicuous exception +of the phone companies, which are after all over a hundred years old, +practically EVERY organization that plays any important role in this book +functions just like the FCIC. The Chicago Task Force, the Arizona +Racketeering Unit, the Legion of Doom, the Phrack crowd, the +Electronic Frontier Foundation--they ALL look and act like "tiger teams" +or "user's groups." They are all electronic ad-hocracies leaping up +spontaneously to attempt to meet a need. + +Some are police. Some are, by strict definition, criminals. +Some are political interest-groups. But every single group +has that same quality of apparent spontaneity--"Hey, gang! +My uncle's got a barn--let's put on a show!" + +Every one of these groups is embarrassed by this "amateurism," +and, for the sake of their public image in a world of non-computer people, +they all attempt to look as stern and formal and impressive as possible. +These electronic frontier-dwellers resemble groups of nineteenth-century +pioneers hankering after the respectability of statehood. +There are however, two crucial differences in the historical experience +of these "pioneers" of the nineteeth and twenty-first centuries. + +First, powerful information technology DOES play into the hands of small, +fluid, loosely organized groups. There have always been "pioneers," +"hobbyists," "amateurs," "dilettantes," "volunteers," "movements," +"users' groups" and "blue-ribbon panels of experts" around. +But a group of this kind--when technically equipped to ship +huge amounts of specialized information, at lightning speed, +to its members, to government, and to the press--is simply +a different kind of animal. It's like the difference between +an eel and an electric eel. + +The second crucial change is that American society is currently +in a state approaching permanent technological revolution. +In the world of computers particularly, it is practically impossible +to EVER stop being a "pioneer," unless you either drop dead or +deliberately jump off the bus. The scene has never slowed down +enough to become well-institutionalized. And after twenty, thirty, +forty years the "computer revolution" continues to spread, +to permeate new corners of society. Anything that really works +is already obsolete. + +If you spend your entire working life as a "pioneer," the word "pioneer" +begins to lose its meaning. Your way of life looks less and less like +an introduction to something else" more stable and organized, +and more and more like JUST THE WAY THINGS ARE. A "permanent revolution" +is really a contradiction in terms. If "turmoil" lasts long enough, +it simply becomes A NEW KIND OF SOCIETY--still the same game of history, +but new players, new rules. + +Apply this to the world of late twentieth-century law enforcement, +and the implications are novel and puzzling indeed. Any bureaucratic +rulebook you write about computer-crime will be flawed when you write it, +and almost an antique by the time it sees print. The fluidity and fast +reactions of the FCIC give them a great advantage in this regard, +which explains their success. Even with the best will in the world +(which it does not, in fact, possess) it is impossible for an organization +the size of the U.S. Federal Bureau of Investigation to get up to speed +on the theory and practice of computer crime. If they tried to train all +their agents to do this, it would be SUICIDAL, as they would NEVER BE ABLE +TO DO ANYTHING ELSE. + +The FBI does try to train its agents in the basics of electronic crime, +at their base in Quantico, Virginia. And the Secret Service, along with +many other law enforcement groups, runs quite successful and well-attended +training courses on wire fraud, business crime, and computer intrusion +at the Federal Law Enforcement Training Center (FLETC, pronounced "fletsy") +in Glynco, Georgia. But the best efforts of these bureaucracies does not +remove the absolute need for a "cutting-edge mess" like the FCIC. + +For you see--the members of FCIC ARE the trainers of the rest +of law enforcement. Practically and literally speaking, +they are the Glynco computer-crime faculty by another name. +If the FCIC went over a cliff on a bus, the U.S. law enforcement +community would be rendered deaf dumb and blind in the world +of computer crime, and would swiftly feel a desperate need +to reinvent them. And this is no time to go starting from scratch. + +On June 11, 1991, I once again arrived in Phoenix, Arizona, +for the latest meeting of the Federal Computer Investigations Committee. +This was more or less the twentieth meeting of this stellar group. +The count was uncertain, since nobody could figure out whether to +include the meetings of "the Colluquy," which is what the FCIC +was called in the mid-1980s before it had even managed to obtain +the dignity of its own acronym. + +Since my last visit to Arizona, in May, the local AzScam bribery scandal +had resolved itself in a general muddle of humiliation. The Phoenix chief +of police, whose agents had videotaped nine state legislators up to no good, +had resigned his office in a tussle with the Phoenix city council over +the propriety of his undercover operations. + +The Phoenix Chief could now join Gail Thackeray and eleven of her closest +associates in the shared experience of politically motivated unemployment. +As of June, resignations were still continuing at the Arizona Attorney +General's office, which could be interpreted as either a New Broom +Sweeping Clean or a Night of the Long Knives Part II, depending on +your point of view. + +The meeting of FCIC was held at the Scottsdale Hilton Resort. +Scottsdale is a wealthy suburb of Phoenix, known as "Scottsdull" +to scoffing local trendies, but well-equipped with posh shopping-malls +and manicured lawns, while conspicuously undersupplied with homeless derelicts. +The Scottsdale Hilton Resort was a sprawling hotel in postmodern +crypto-Southwestern style. It featured a "mission bell tower" +plated in turquoise tile and vaguely resembling a Saudi minaret. + +Inside it was all barbarically striped Santa Fe Style decor. +There was a health spa downstairs and a large oddly-shaped +pool in the patio. A poolside umbrella-stand offered Ben and Jerry's +politically correct Peace Pops. + +I registered as a member of FCIC, attaining a handy discount rate, +then went in search of the Feds. Sure enough, at the back of the +hotel grounds came the unmistakable sound of Gail Thackeray +holding forth. + +Since I had also attended the Computers Freedom and Privacy conference +(about which more later), this was the second time I had seen Thackeray +in a group of her law enforcement colleagues. Once again I was struck +by how simply pleased they seemed to see her. It was natural that she'd +get SOME attention, as Gail was one of two women in a group of some thirty men; +but there was a lot more to it than that. + +Gail Thackeray personifies the social glue of the FCIC. They could give +a damn about her losing her job with the Attorney General. They were sorry +about it, of course, but hell, they'd all lost jobs. If they were the kind +of guys who liked steady boring jobs, they would never have gotten into +computer work in the first place. + +I wandered into her circle and was immediately introduced to five strangers. +The conditions of my visit at FCIC were reviewed. I would not quote +anyone directly. I would not tie opinions expressed to the agencies +of the attendees. I would not (a purely hypothetical example) +report the conversation of a guy from the Secret Service talking +quite civilly to a guy from the FBI, as these two agencies NEVER +talk to each other, and the IRS (also present, also hypothetical) +NEVER TALKS TO ANYBODY. + +Worse yet, I was forbidden to attend the first conference. And I didn't. +I have no idea what the FCIC was up to behind closed doors that afternoon. +I rather suspect that they were engaging in a frank and thorough confession +of their errors, goof-ups and blunders, as this has been a feature of every +FCIC meeting since their legendary Memphis beer-bust of 1986. Perhaps the +single greatest attraction of FCIC is that it is a place where you can go, +let your hair down, and completely level with people who actually comprehend +what you are talking about. Not only do they understand you, but they +REALLY PAY ATTENTION, they are GRATEFUL FOR YOUR INSIGHTS, and they +FORGIVE YOU, which in nine cases out of ten is something even your +boss can't do, because as soon as you start talking "ROM," "BBS," +or "T-1 trunk," his eyes glaze over. + +I had nothing much to do that afternoon. The FCIC were beavering away +in their conference room. Doors were firmly closed, windows too dark +to peer through. I wondered what a real hacker, a computer intruder, +would do at a meeting like this. + +The answer came at once. He would "trash" the place. Not reduce the place +to trash in some orgy of vandalism; that's not the use of the term in the +hacker milieu. No, he would quietly EMPTY THE TRASH BASKETS and silently +raid any valuable data indiscreetly thrown away. + +Journalists have been known to do this. (Journalists hunting information +have been known to do almost every single unethical thing that hackers +have ever done. They also throw in a few awful techniques all their own.) +The legality of `trashing' is somewhat dubious but it is not in fact +flagrantly illegal. It was, however, absurd to contemplate trashing the FCIC. +These people knew all about trashing. I wouldn't last fifteen seconds. + +The idea sounded interesting, though. I'd been hearing a lot about +the practice lately. On the spur of the moment, I decided I would try +trashing the office ACROSS THE HALL from the FCIC, an area which had +nothing to do with the investigators. + +The office was tiny; six chairs, a table. . . . Nevertheless, it was open, +so I dug around in its plastic trash can. + +To my utter astonishment, I came up with the torn scraps of a SPRINT +long-distance phone bill. More digging produced a bank statement +and the scraps of a hand-written letter, along with gum, cigarette ashes, +candy wrappers and a day-old-issue of USA TODAY. + +The trash went back in its receptacle while the scraps of data went into +my travel bag. I detoured through the hotel souvenir shop for some +Scotch tape and went up to my room. + +Coincidence or not, it was quite true. Some poor soul had, in fact, +thrown a SPRINT bill into the hotel's trash. Date May 1991, +total amount due: $252.36. Not a business phone, either, +but a residential bill, in the name of someone called Evelyn +(not her real name). Evelyn's records showed a ## PAST DUE BILL ##! +Here was her nine-digit account ID. Here was a stern computer-printed warning: + +"TREAT YOUR FONCARD AS YOU WOULD ANY CREDIT CARD. TO SECURE AGAINST FRAUD, +NEVER GIVE YOUR FONCARD NUMBER OVER THE PHONE UNLESS YOU INITIATED THE CALL. +IF YOU RECEIVE SUSPICIOUS CALLS PLEASE NOTIFY CUSTOMER SERVICE IMMEDIATELY!" + +I examined my watch. Still plenty of time left for the FCIC to carry on. +I sorted out the scraps of Evelyn's SPRINT bill and re-assembled them with +fresh Scotch tape. Here was her ten-digit FONCARD number. Didn't seem +to have the ID number necessary to cause real fraud trouble. + +I did, however, have Evelyn's home phone number. And the phone numbers +for a whole crowd of Evelyn's long-distance friends and acquaintances. +In San Diego, Folsom, Redondo, Las Vegas, La Jolla, Topeka, and Northampton +Massachusetts. Even somebody in Australia! + +I examined other documents. Here was a bank statement. It was Evelyn's +IRA account down at a bank in San Mateo California (total balance $1877.20). +Here was a charge-card bill for $382.64. She was paying it off bit by bit. + +Driven by motives that were completely unethical and prurient, +I now examined the handwritten notes. They had been torn fairly +thoroughly, so much so that it took me almost an entire five minutes +to reassemble them. + +They were drafts of a love letter. They had been written on +the lined stationery of Evelyn's employer, a biomedical company. +Probably written at work when she should have been doing something else. + +"Dear Bob," (not his real name) "I guess in everyone's life there comes +a time when hard decisions have to be made, and this is a difficult one +for me--very upsetting. Since you haven't called me, and I don't understand +why, I can only surmise it's because you don't want to. I thought I would +have heard from you Friday. I did have a few unusual problems with my phone +and possibly you tried, I hope so. + +"Robert, you asked me to `let go'. . . ." + +The first note ended. UNUSUAL PROBLEMS WITH HER PHONE? +I looked swiftly at the next note. + +"Bob, not hearing from you for the whole weekend has left me very perplexed. . . ." + +Next draft. + +"Dear Bob, there is so much I don't understand right now, and I wish I did. +I wish I could talk to you, but for some unknown reason you have elected not +to call--this is so difficult for me to understand. . . ." + +She tried again. + +"Bob, Since I have always held you in such high esteem, I had every hope that +we could remain good friends, but now one essential ingredient is missing-- +respect. Your ability to discard people when their purpose is served is +appalling to me. The kindest thing you could do for me now is to leave me +alone. You are no longer welcome in my heart or home. . . ." + +Try again. + +"Bob, I wrote a very factual note to you to say how much respect I had lost +for you, by the way you treat people, me in particular, so uncaring and cold. +The kindest thing you can do for me is to leave me alone entirely, +as you are no longer welcome in my heart or home. I would appreciate it +if you could retire your debt to me as soon as possible--I wish no link +to you in any way. Sincerely, Evelyn." + +Good heavens, I thought, the bastard actually owes her money! +I turned to the next page. + +"Bob: very simple. GOODBYE! No more mind games--no more fascination-- +no more coldness--no more respect for you! It's over--Finis. Evie" + +There were two versions of the final brushoff letter, but they read about +the same. Maybe she hadn't sent it. The final item in my illicit and +shameful booty was an envelope addressed to "Bob" at his home address, +but it had no stamp on it and it hadn't been mailed. + +Maybe she'd just been blowing off steam because her rascal boyfriend +had neglected to call her one weekend. Big deal. Maybe they'd kissed +and made up, maybe she and Bob were down at Pop's Chocolate Shop now, +sharing a malted. Sure. + +Easy to find out. All I had to do was call Evelyn up. With a half-clever +story and enough brass-plated gall I could probably trick the truth out of her. +Phone-phreaks and hackers deceive people over the phone all the time. +It's called "social engineering." Social engineering is a very common practice +in the underground, and almost magically effective. Human beings are almost +always the weakest link in computer security. The simplest way to learn +Things You Are Not Meant To Know is simply to call up and exploit the +knowledgeable people. With social engineering, you use the bits of specialized +knowledge you already have as a key, to manipulate people into believing +that you are legitimate. You can then coax, flatter, or frighten them into +revealing almost anything you want to know. Deceiving people (especially +over the phone) is easy and fun. Exploiting their gullibility is very +gratifying; it makes you feel very superior to them. + +If I'd been a malicious hacker on a trashing raid, I would now have Evelyn +very much in my power. Given all this inside data, it wouldn't take much +effort at all to invent a convincing lie. If I were ruthless enough, +and jaded enough, and clever enough, this momentary indiscretion of hers-- +maybe committed in tears, who knows--could cause her a whole world of +confusion and grief. + +I didn't even have to have a MALICIOUS motive. Maybe I'd be "on her side," +and call up Bob instead, and anonymously threaten to break both his kneecaps +if he didn't take Evelyn out for a steak dinner pronto. It was still +profoundly NONE OF MY BUSINESS. To have gotten this knowledge at all +was a sordid act and to use it would be to inflict a sordid injury. + +To do all these awful things would require exactly zero high-tech expertise. +All it would take was the willingness to do it and a certain amount +of bent imagination. + +I went back downstairs. The hard-working FCIC, who had labored forty-five +minutes over their schedule, were through for the day, and adjourned to the +hotel bar. We all had a beer. + +I had a chat with a guy about "Isis," or rather IACIS, +the International Association of Computer Investigation Specialists. +They're into "computer forensics," the techniques of picking computer- +systems apart without destroying vital evidence. IACIS, currently run +out of Oregon, is comprised of investigators in the U.S., Canada, Taiwan +and Ireland. "Taiwan and Ireland?" I said. Are TAIWAN and IRELAND +really in the forefront of this stuff? Well not exactly, my informant +admitted. They just happen to have been the first ones to have caught +on by word of mouth. Still, the international angle counts, because this +is obviously an international problem. Phone-lines go everywhere. + +There was a Mountie here from the Royal Canadian Mounted Police. +He seemed to be having quite a good time. Nobody had flung this +Canadian out because he might pose a foreign security risk. +These are cyberspace cops. They still worry a lot about "jurisdictions," +but mere geography is the least of their troubles. + +NASA had failed to show. NASA suffers a lot from computer intrusions, +in particular from Australian raiders and a well-trumpeted Chaos +Computer Club case, and in 1990 there was a brief press flurry +when it was revealed that one of NASA's Houston branch-exchanges +had been systematically ripped off by a gang of phone-phreaks. +But the NASA guys had had their funding cut. They were stripping everything. + +Air Force OSI, its Office of Special Investigations, is the ONLY federal +entity dedicated full-time to computer security. They'd been expected +to show up in force, but some of them had cancelled--a Pentagon budget pinch. + +As the empties piled up, the guys began joshing around and telling war-stories. +"These are cops," Thackeray said tolerantly. "If they're not talking shop +they talk about women and beer." + +I heard the story about the guy who, asked for "a copy" of a computer disk, +PHOTOCOPIED THE LABEL ON IT. He put the floppy disk onto the glass plate +of a photocopier. The blast of static when the copier worked completely +erased all the real information on the disk. + +Some other poor souls threw a whole bag of confiscated diskettes +into the squad-car trunk next to the police radio. The powerful radio +signal blasted them, too. + +We heard a bit about Dave Geneson, the first computer prosecutor, +a mainframe-runner in Dade County, turned lawyer. Dave Geneson +was one guy who had hit the ground running, a signal virtue +in making the transition to computer-crime. It was generally +agreed that it was easier to learn the world of computers first, +then police or prosecutorial work. You could take certain computer +people and train 'em to successful police work--but of course they +had to have the COP MENTALITY. They had to have street smarts. +Patience. Persistence. And discretion. You've got to make sure +they're not hot-shots, show-offs, "cowboys." + +Most of the folks in the bar had backgrounds in military intelligence, +or drugs, or homicide. It was rudely opined that "military intelligence" +was a contradiction in terms, while even the grisly world of homicide +was considered cleaner than drug enforcement. One guy had been 'way +undercover doing dope-work in Europe for four years straight. +"I'm almost recovered now," he said deadpan, with the acid black humor +that is pure cop. "Hey, now I can say FUCKER without putting MOTHER +in front of it." + +"In the cop world," another guy said earnestly, "everything is good and bad, +black and white. In the computer world everything is gray." + +One guy--a founder of the FCIC, who'd been with the group +since it was just the Colluquy--described his own introduction +to the field. He'd been a Washington DC homicide guy called in +on a "hacker" case. From the word "hacker," he naturally assumed +he was on the trail of a knife-wielding marauder, and went to the +computer center expecting blood and a body. When he finally figured +out what was happening there (after loudly demanding, in vain, +that the programmers "speak English"), he called headquarters +and told them he was clueless about computers. They told him nobody +else knew diddly either, and to get the hell back to work. + +So, he said, he had proceeded by comparisons. By analogy. By metaphor. +"Somebody broke in to your computer, huh?" Breaking and entering; +I can understand that. How'd he get in? "Over the phone-lines." +Harassing phone-calls, I can understand that! What we need here +is a tap and a trace! + +It worked. It was better than nothing. And it worked a lot faster +when he got hold of another cop who'd done something similar. +And then the two of them got another, and another, and pretty soon +the Colluquy was a happening thing. It helped a lot that everybody +seemed to know Carlton Fitzpatrick, the data-processing trainer in Glynco. + +The ice broke big-time in Memphis in '86. The Colluquy had attracted +a bunch of new guys--Secret Service, FBI, military, other feds, heavy guys. +Nobody wanted to tell anybody anything. They suspected that if word got back +to the home office they'd all be fired. They passed an uncomfortably +guarded afternoon. + +The formalities got them nowhere. But after the formal session was over, +the organizers brought in a case of beer. As soon as the participants +knocked it off with the bureaucratic ranks and turf-fighting, everything +changed. "I bared my soul," one veteran reminisced proudly. By nightfall +they were building pyramids of empty beer-cans and doing everything +but composing a team fight song. + +FCIC were not the only computer-crime people around. There was DATTA +(District Attorneys' Technology Theft Association), though they mostly +specialized in chip theft, intellectual property, and black-market cases. +There was HTCIA (High Tech Computer Investigators Association), +also out in Silicon Valley, a year older than FCIC and featuring +brilliant people like Donald Ingraham. There was LEETAC +(Law Enforcement Electronic Technology Assistance Committee) +in Florida, and computer-crime units in Illinois and Maryland +and Texas and Ohio and Colorado and Pennsylvania. But these were +local groups. FCIC were the first to really network nationally +and on a federal level. + +FCIC people live on the phone lines. Not on bulletin board systems-- +they know very well what boards are, and they know that boards aren't secure. +Everyone in the FCIC has a voice-phone bill like you wouldn't believe. +FCIC people have been tight with the telco people for a long time. +Telephone cyberspace is their native habitat. + +FCIC has three basic sub-tribes: the trainers, the security people, +and the investigators. That's why it's called an "Investigations +Committee" with no mention of the term "computer-crime"--the dreaded +"C-word." FCIC, officially, is "an association of agencies rather +than individuals;" unofficially, this field is small enough that +the influence of individuals and individual expertise is paramount. +Attendance is by invitation only, and most everyone in FCIC considers +himself a prophet without honor in his own house. + +Again and again I heard this, with different terms but identical +sentiments. "I'd been sitting in the wilderness talking to myself." +"I was totally isolated." "I was desperate." "FCIC is the best +thing there is about computer crime in America." "FCIC is what +really works." "This is where you hear real people telling you +what's really happening out there, not just lawyers picking nits." +"We taught each other everything we knew." + +The sincerity of these statements convinces me that this is true. +FCIC is the real thing and it is invaluable. It's also very sharply +at odds with the rest of the traditions and power structure +in American law enforcement. There probably hasn't been anything +around as loose and go-getting as the FCIC since the start of the +U.S. Secret Service in the 1860s. FCIC people are living like +twenty-first-century people in a twentieth-century environment, +and while there's a great deal to be said for that, there's also +a great deal to be said against it, and those against it happen +to control the budgets. + +I listened to two FCIC guys from Jersey compare life histories. +One of them had been a biker in a fairly heavy-duty gang in the 1960s. +"Oh, did you know so-and-so?" said the other guy from Jersey. +"Big guy, heavyset?" + +"Yeah, I knew him." + +"Yeah, he was one of ours. He was our plant in the gang." + +"Really? Wow! Yeah, I knew him. Helluva guy." + +Thackeray reminisced at length about being tear-gassed blind +in the November 1969 antiwar protests in Washington Circle, +covering them for her college paper. "Oh yeah, I was there," +said another cop. "Glad to hear that tear gas hit somethin'. +Haw haw haw." He'd been so blind himself, he confessed, +that later that day he'd arrested a small tree. + +FCIC are an odd group, sifted out by coincidence and necessity, +and turned into a new kind of cop. There are a lot of specialized +cops in the world--your bunco guys, your drug guys, your tax guys, +but the only group that matches FCIC for sheer isolation are probably +the child-pornography people. Because they both deal with conspirators +who are desperate to exchange forbidden data and also desperate to hide; +and because nobody else in law enforcement even wants to hear about it. + +FCIC people tend to change jobs a lot. They tend not to get the equipment +and training they want and need. And they tend to get sued quite often. + +As the night wore on and a band set up in the bar, the talk grew darker. +Nothing ever gets done in government, someone opined, until there's +a DISASTER. Computing disasters are awful, but there's no denying +that they greatly help the credibility of FCIC people. The Internet Worm, +for instance. "For years we'd been warning about that--but it's nothing +compared to what's coming." They expect horrors, these people. +They know that nothing will really get done until there is a horror. + +# + +Next day we heard an extensive briefing from a guy who'd been a computer cop, +gotten into hot water with an Arizona city council, and now installed +computer networks for a living (at a considerable rise in pay). +He talked about pulling fiber-optic networks apart. + +Even a single computer, with enough peripherals, is a literal +"network"--a bunch of machines all cabled together, generally +with a complexity that puts stereo units to shame. FCIC people +invent and publicize methods of seizing computers and maintaining +their evidence. Simple things, sometimes, but vital rules of thumb +for street cops, who nowadays often stumble across a busy computer +in the midst of a drug investigation or a white-collar bust. +For instance: Photograph the system before you touch it. +Label the ends of all the cables before you detach anything. +"Park" the heads on the disk drives before you move them. +Get the diskettes. Don't put the diskettes in magnetic fields. +Don't write on diskettes with ballpoint pens. Get the manuals. +Get the printouts. Get the handwritten notes. Copy data before +you look at it, and then examine the copy instead of the original. + +Now our lecturer distributed copied diagrams of a typical LAN +or "Local Area Network", which happened to be out of Connecticut. +ONE HUNDRED AND FIFTY-NINE desktop computers, each with its own +peripherals. Three "file servers." Five "star couplers" +each with thirty-two ports. One sixteen-port coupler +off in the corner office. All these machines talking to each other, +distributing electronic mail, distributing software, distributing, +quite possibly, criminal evidence. All linked by high-capacity +fiber-optic cable. A bad guy--cops talk a about "bad guys" +--might be lurking on PC #47 lot or #123 and distributing +his ill doings onto some dupe's "personal" machine in +another office--or another floor--or, quite possibly, +two or three miles away! Or, conceivably, the evidence might +be "data-striped"--split up into meaningless slivers stored, +one by one, on a whole crowd of different disk drives. + +The lecturer challenged us for solutions. I for one was utterly clueless. +As far as I could figure, the Cossacks were at the gate; there were probably +more disks in this single building than were seized during the entirety +of Operation Sundevil. + +"Inside informant," somebody said. Right. There's always the human angle, +something easy to forget when contemplating the arcane recesses of high +technology. Cops are skilled at getting people to talk, and computer people, +given a chair and some sustained attention, will talk about their computers +till their throats go raw. There's a case on record of a single question-- +"How'd you do it?"--eliciting a forty-five-minute videotaped confession +from a computer criminal who not only completely incriminated himself +but drew helpful diagrams. + +Computer people talk. Hackers BRAG. Phone-phreaks +talk PATHOLOGICALLY--why else are they stealing phone-codes, +if not to natter for ten hours straight to their friends +on an opposite seaboard? Computer-literate people do +in fact possess an arsenal of nifty gadgets and techniques +that would allow them to conceal all kinds of exotic skullduggery, +and if they could only SHUT UP about it, they could probably +get away with all manner of amazing information-crimes. +But that's just not how it works--or at least, +that's not how it's worked SO FAR. + +Most every phone-phreak ever busted has swiftly implicated his mentors, +his disciples, and his friends. Most every white-collar computer-criminal, +smugly convinced that his clever scheme is bulletproof, swiftly learns +otherwise when, for the first time in his life, an actual no-kidding +policeman leans over, grabs the front of his shirt, looks him right +in the eye and says: "All right, ASSHOLE--you and me are going downtown!" +All the hardware in the world will not insulate your nerves from +these actual real-life sensations of terror and guilt. + +Cops know ways to get from point A to point Z without thumbing +through every letter in some smart-ass bad-guy's alphabet. +Cops know how to cut to the chase. Cops know a lot of things +other people don't know. + +Hackers know a lot of things other people don't know, too. +Hackers know, for instance, how to sneak into your computer +through the phone-lines. But cops can show up RIGHT ON YOUR DOORSTEP +and carry off YOU and your computer in separate steel boxes. +A cop interested in hackers can grab them and grill them. +A hacker interested in cops has to depend on hearsay, +underground legends, and what cops are willing to publicly reveal. +And the Secret Service didn't get named "the SECRET Service" +because they blab a lot. + +Some people, our lecturer informed us, were under the mistaken +impression that it was "impossible" to tap a fiber-optic line. +Well, he announced, he and his son had just whipped up a +fiber-optic tap in his workshop at home. He passed it around +the audience, along with a circuit-covered LAN plug-in card +so we'd all recognize one if we saw it on a case. We all had a look. + +The tap was a classic "Goofy Prototype"--a thumb-length rounded +metal cylinder with a pair of plastic brackets on it. +From one end dangled three thin black cables, each of which ended +in a tiny black plastic cap. When you plucked the safety-cap +off the end of a cable, you could see the glass fiber-- +no thicker than a pinhole. + +Our lecturer informed us that the metal cylinder was a +"wavelength division multiplexer." Apparently, what one did +was to cut the fiber-optic cable, insert two of the legs into +the cut to complete the network again, and then read any passing data +on the line by hooking up the third leg to some kind of monitor. +Sounded simple enough. I wondered why nobody had thought of it before. +I also wondered whether this guy's son back at the workshop had any +teenage friends. + +We had a break. The guy sitting next to me was wearing a giveaway +baseball cap advertising the Uzi submachine gun. We had a desultory chat +about the merits of Uzis. Long a favorite of the Secret Service, +it seems Uzis went out of fashion with the advent of the Persian Gulf War, +our Arab allies taking some offense at Americans toting Israeli weapons. +Besides, I was informed by another expert, Uzis jam. The equivalent weapon +of choice today is the Heckler & Koch, manufactured in Germany. + +The guy with the Uzi cap was a forensic photographer. He also did a lot +of photographic surveillance work in computer crime cases. He used to, +that is, until the firings in Phoenix. He was now a private investigator and, +with his wife, ran a photography salon specializing in weddings and portrait +photos. At--one must repeat--a considerable rise in income. + +He was still FCIC. If you were FCIC, and you needed to talk +to an expert about forensic photography, well, there he was, +willing and able. If he hadn't shown up, people would have missed him. + +Our lecturer had raised the point that preliminary investigation +of a computer system is vital before any seizure is undertaken. +It's vital to understand how many machines are in there, what kinds +there are, what kind of operating system they use, how many people +use them, where the actual data itself is stored. To simply barge into +an office demanding "all the computers" is a recipe for swift disaster. + +This entails some discreet inquiries beforehand. In fact, what it +entails is basically undercover work. An intelligence operation. +SPYING, not to put too fine a point on it. + +In a chat after the lecture, I asked an attendee whether "trashing" might work. + +I received a swift briefing on the theory and practice of "trash covers." +Police "trash covers," like "mail covers" or like wiretaps, require the +agreement of a judge. This obtained, the "trashing" work of cops is just +like that of hackers, only more so and much better organized. So much so, +I was informed, that mobsters in Phoenix make extensive use of locked +garbage cans picked up by a specialty high-security trash company. + +In one case, a tiger team of Arizona cops had trashed a local residence +for four months. Every week they showed up on the municipal garbage truck, +disguised as garbagemen, and carried the contents of the suspect cans off +to a shade tree, where they combed through the garbage--a messy task, +especially considering that one of the occupants was undergoing +kidney dialysis. All useful documents were cleaned, dried and examined. +A discarded typewriter-ribbon was an especially valuable source of data, +as its long one-strike ribbon of film contained the contents of every +letter mailed out of the house. The letters were neatly retyped by +a police secretary equipped with a large desk-mounted magnifying glass. + +There is something weirdly disquieting about the whole subject of +"trashing"-- an unsuspected and indeed rather disgusting mode of +deep personal vulnerability. Things that we pass by every day, +that we take utterly for granted, can be exploited with so little work. +Once discovered, the knowledge of these vulnerabilities tend to spread. + +Take the lowly subject of MANHOLE COVERS. The humble manhole cover +reproduces many of the dilemmas of computer-security in miniature. +Manhole covers are, of course, technological artifacts, access-points +to our buried urban infrastructure. To the vast majority of us, +manhole covers are invisible. They are also vulnerable. For many years now, +the Secret Service has made a point of caulking manhole covers along all routes +of the Presidential motorcade. This is, of course, to deter terrorists from +leaping out of underground ambush or, more likely, planting remote-control +car-smashing bombs beneath the street. + +Lately, manhole covers have seen more and more criminal exploitation, +especially in New York City. Recently, a telco in New York City +discovered that a cable television service had been sneaking into +telco manholes and installing cable service alongside the phone-lines-- +WITHOUT PAYING ROYALTIES. New York companies have also suffered a +general plague of (a) underground copper cable theft; (b) dumping of garbage, +including toxic waste, and (c) hasty dumping of murder victims. + +Industry complaints reached the ears of an innovative New England +industrial-security company, and the result was a new product known +as "the Intimidator," a thick titanium-steel bolt with a precisely machined +head that requires a special device to unscrew. All these "keys" have registered +serial numbers kept on file with the manufacturer. There are now some +thousands of these "Intimidator" bolts being sunk into American pavements +wherever our President passes, like some macabre parody of strewn roses. +They are also spreading as fast as steel dandelions around US military bases +and many centers of private industry. + +Quite likely it has never occurred to you to peer under a manhole cover, +perhaps climb down and walk around down there with a flashlight, just to see +what it's like. Formally speaking, this might be trespassing, but if you +didn't hurt anything, and didn't make an absolute habit of it, nobody would +really care. The freedom to sneak under manholes was likely a freedom +you never intended to exercise. + +You now are rather less likely to have that freedom at all. +You may never even have missed it until you read about it here, +but if you're in New York City it's gone, and elsewhere it's likely going. +This is one of the things that crime, and the reaction to +crime, does to us. + +The tenor of the meeting now changed as the Electronic Frontier Foundation +arrived. The EFF, whose personnel and history will be examined in detail +in the next chapter, are a pioneering civil liberties group who arose in +direct response to the Hacker Crackdown of 1990. + +Now Mitchell Kapor, the Foundation's president, and Michael Godwin, +its chief attorney, were confronting federal law enforcement MANO A MANO +for the first time ever. Ever alert to the manifold uses of publicity, +Mitch Kapor and Mike Godwin had brought their own journalist in tow: +Robert Draper, from Austin, whose recent well-received book about +ROLLING STONE magazine was still on the stands. Draper was on assignment +for TEXAS MONTHLY. + +The Steve Jackson/EFF civil lawsuit against the Chicago Computer Fraud +and Abuse Task Force was a matter of considerable regional interest in Texas. +There were now two Austinite journalists here on the case. In fact, +counting Godwin (a former Austinite and former journalist) there were +three of us. Lunch was like Old Home Week. + +Later, I took Draper up to my hotel room. We had a long frank talk +about the case, networking earnestly like a miniature freelance-journo +version of the FCIC: privately confessing the numerous blunders +of journalists covering the story, and trying hard to figure out +who was who and what the hell was really going on out there. +I showed Draper everything I had dug out of the Hilton trashcan. +We pondered the ethics of "trashing" for a while, and agreed +that they were dismal. We also agreed that finding a SPRINT +bill on your first time out was a heck of a coincidence. + +First I'd "trashed"--and now, mere hours later, I'd bragged to someone else. +Having entered the lifestyle of hackerdom, I was now, unsurprisingly, +following its logic. Having discovered something remarkable through +a surreptitious action, I of course HAD to "brag," and to drag the passing +Draper into my iniquities. I felt I needed a witness. Otherwise nobody +would have believed what I'd discovered. . . . + +Back at the meeting, Thackeray cordially, if rather tentatively, +introduced Kapor and Godwin to her colleagues. Papers were distributed. +Kapor took center stage. The brilliant Bostonian high-tech entrepreneur, +normally the hawk in his own administration and quite an effective +public speaker, seemed visibly nervous, and frankly admitted as much. +He began by saying he consided computer-intrusion to be morally wrong, +and that the EFF was not a "hacker defense fund," despite what had appeared +in print. Kapor chatted a bit about the basic motivations of his group, +emphasizing their good faith and willingness to listen and seek common ground +with law enforcement--when, er, possible. + +Then, at Godwin's urging, Kapor suddenly remarked that EFF's own Internet +machine had been "hacked" recently, and that EFF did not consider +this incident amusing. + +After this surprising confession, things began to loosen up +quite rapidly. Soon Kapor was fielding questions, parrying objections, +challenging definitions, and juggling paradigms with something akin +to his usual gusto. + +Kapor seemed to score quite an effect with his shrewd and skeptical analysis +of the merits of telco "Caller-ID" services. (On this topic, FCIC and EFF +have never been at loggerheads, and have no particular established earthworks +to defend.) Caller-ID has generally been promoted as a privacy service +for consumers, a presentation Kapor described as a "smokescreen," +the real point of Caller-ID being to ALLOW CORPORATE CUSTOMERS TO BUILD +EXTENSIVE COMMERCIAL DATABASES ON EVERYBODY WHO PHONES OR FAXES THEM. +Clearly, few people in the room had considered this possibility, +except perhaps for two late-arrivals from US WEST RBOC security, +who chuckled nervously. + +Mike Godwin then made an extensive presentation on +"Civil Liberties Implications of Computer Searches and Seizures." +Now, at last, we were getting to the real nitty-gritty here, +real political horse-trading. The audience listened with close +attention, angry mutters rising occasionally: "He's trying to +teach us our jobs!" "We've been thinking about this for years! +We think about these issues every day!" "If I didn't seize the works, +I'd be sued by the guy's victims!" "I'm violating the law if I leave +ten thousand disks full of illegal PIRATED SOFTWARE and STOLEN CODES!" +"It's our job to make sure people don't trash the Constitution-- +we're the DEFENDERS of the Constitution!" "We seize stuff when +we know it will be forfeited anyway as restitution for the victim!" + +"If it's forfeitable, then don't get a search warrant, get a +forfeiture warrant," Godwin suggested coolly. He further remarked +that most suspects in computer crime don't WANT to see their computers +vanish out the door, headed God knew where, for who knows how long. +They might not mind a search, even an extensive search, but they want +their machines searched on-site. + +"Are they gonna feed us?" somebody asked sourly. + +"How about if you take copies of the data?" Godwin parried. + +"That'll never stand up in court." + +"Okay, you make copies, give THEM the copies, and take the originals." + +Hmmm. + +Godwin championed bulletin-board systems as repositories of First Amendment +protected free speech. He complained that federal computer-crime training +manuals gave boards a bad press, suggesting that they are hotbeds of crime +haunted by pedophiles and crooks, whereas the vast majority of the nation's +thousands of boards are completely innocuous, and nowhere near so +romantically suspicious. + +People who run boards violently resent it when their systems are seized, +and their dozens (or hundreds) of users look on in abject horror. +Their rights of free expression are cut short. Their right to associate +with other people is infringed. And their privacy is violated as their +private electronic mail becomes police property. + +Not a soul spoke up to defend the practice of seizing boards. +The issue passed in chastened silence. Legal principles aside-- +(and those principles cannot be settled without laws passed or +court precedents)--seizing bulletin boards has become public-relations +poison for American computer police. + +And anyway, it's not entirely necessary. If you're a cop, you can get 'most +everything you need from a pirate board, just by using an inside informant. +Plenty of vigilantes--well, CONCERNED CITIZENS--will inform police the moment +they see a pirate board hit their area (and will tell the police all about it, +in such technical detail, actually, that you kinda wish they'd shut up). +They will happily supply police with extensive downloads or printouts. +It's IMPOSSIBLE to keep this fluid electronic information out of the +hands of police. + +Some people in the electronic community become enraged at the prospect +of cops "monitoring" bulletin boards. This does have touchy aspects, +as Secret Service people in particular examine bulletin boards with +some regularity. But to expect electronic police to be deaf dumb +and blind in regard to this particular medium rather flies in the face +of common sense. Police watch television, listen to radio, read newspapers +and magazines; why should the new medium of boards be different? +Cops can exercise the same access to electronic information +as everybody else. As we have seen, quite a few computer +police maintain THEIR OWN bulletin boards, including anti-hacker +"sting" boards, which have generally proven quite effective. + +As a final clincher, their Mountie friends in Canada (and colleagues +in Ireland and Taiwan) don't have First Amendment or American +constitutional restrictions, but they do have phone lines, +and can call any bulletin board in America whenever they please. +The same technological determinants that play into the hands of hackers, +phone phreaks and software pirates can play into the hands of police. +"Technological determinants" don't have ANY human allegiances. +They're not black or white, or Establishment or Underground, +or pro-or-anti anything. + +Godwin complained at length about what he called "the Clever Hobbyist +hypothesis" --the assumption that the "hacker" you're busting is clearly +a technical genius, and must therefore by searched with extreme thoroughness. +So: from the law's point of view, why risk missing anything? Take the works. +Take the guy's computer. Take his books. Take his notebooks. +Take the electronic drafts of his love letters. Take his Walkman. +Take his wife's computer. Take his dad's computer. Take his kid +sister's computer. Take his employer's computer. Take his compact disks-- +they MIGHT be CD-ROM disks, cunningly disguised as pop music. +Take his laser printer--he might have hidden something vital in the +printer's 5meg of memory. Take his software manuals and hardware +documentation. Take his science-fiction novels and his simulation- +gaming books. Take his Nintendo Game-Boy and his Pac-Man arcade game. +Take his answering machine, take his telephone out of the wall. +Take anything remotely suspicious. + +Godwin pointed out that most "hackers" are not, in fact, clever +genius hobbyists. Quite a few are crooks and grifters who don't +have much in the way of technical sophistication; just some rule-of-thumb +rip-off techniques. The same goes for most fifteen-year-olds who've +downloaded a code-scanning program from a pirate board. There's no +real need to seize everything in sight. It doesn't require an entire +computer system and ten thousand disks to prove a case in court. + +What if the computer is the instrumentality of a crime? someone demanded. + +Godwin admitted quietly that the doctrine of seizing the instrumentality +of a crime was pretty well established in the American legal system. + +The meeting broke up. Godwin and Kapor had to leave. Kapor was testifying +next morning before the Massachusetts Department Of Public Utility, +about ISDN narrowband wide-area networking. + +As soon as they were gone, Thackeray seemed elated. +She had taken a great risk with this. Her colleagues had not, +in fact, torn Kapor and Godwin's heads off. She was very proud of them, +and told them so. + +"Did you hear what Godwin said about INSTRUMENTALITY OF A CRIME?" +she exulted, to nobody in particular. "Wow, that means +MITCH ISN'T GOING TO SUE ME." + +# + +America's computer police are an interesting group. +As a social phenomenon they are far more interesting, +and far more important, than teenage phone phreaks +and computer hackers. First, they're older and wiser; +not dizzy hobbyists with leaky morals, but seasoned adult +professionals with all the responsibilities of public service. +And, unlike hackers, they possess not merely TECHNICAL +power alone, but heavy-duty legal and social authority. + +And, very interestingly, they are just as much at +sea in cyberspace as everyone else. They are not +happy about this. Police are authoritarian by nature, +and prefer to obey rules and precedents. (Even those police +who secretly enjoy a fast ride in rough territory will soberly +disclaim any "cowboy" attitude.) But in cyberspace there ARE +no rules and precedents. They are groundbreaking pioneers, +Cyberspace Rangers, whether they like it or not. + +In my opinion, any teenager enthralled by computers, +fascinated by the ins and outs of computer security, +and attracted by the lure of specialized forms of knowledge and power, +would do well to forget all about "hacking" and set his (or her) +sights on becoming a fed. Feds can trump hackers at almost every +single thing hackers do, including gathering intelligence, +undercover disguise, trashing, phone-tapping, building dossiers, +networking, and infiltrating computer systems--CRIMINAL computer systems. +Secret Service agents know more about phreaking, coding and carding +than most phreaks can find out in years, and when it comes to viruses, +break-ins, software bombs and trojan horses, Feds have direct access to red-hot +confidential information that is only vague rumor in the underground. + +And if it's an impressive public rep you're after, there are few people +in the world who can be so chillingly impressive as a well-trained, +well-armed United States Secret Service agent. + +Of course, a few personal sacrifices are necessary in order to obtain +that power and knowledge. First, you'll have the galling discipline +of belonging to a large organization; but the world of computer crime +is still so small, and so amazingly fast-moving, that it will remain +spectacularly fluid for years to come. The second sacrifice is that +you'll have to give up ripping people off. This is not a great loss. +Abstaining from the use of illegal drugs, also necessary, will be a boon +to your health. + +A career in computer security is not a bad choice for a young man +or woman today. The field will almost certainly expand drastically +in years to come. If you are a teenager today, by the time you +become a professional, the pioneers you have read about in this book +will be the grand old men and women of the field, swamped by their many +disciples and successors. Of course, some of them, like William P. Wood +of the 1865 Secret Service, may well be mangled in the whirring machinery +of legal controversy; but by the time you enter the computer-crime field, +it may have stabilized somewhat, while remaining entertainingly challenging. + +But you can't just have a badge. You have to win it. First, there's the +federal law enforcement training. And it's hard--it's a challenge. +A real challenge--not for wimps and rodents. + +Every Secret Service agent must complete gruelling courses at the +Federal Law Enforcement Training Center. (In fact, Secret Service +agents are periodically re-trained during their entire careers.) + +In order to get a glimpse of what this might be like, +I myself travelled to FLETC. + +# + +The Federal Law Enforcement Training Center is a 1500-acre facility +on Georgia's Atlantic coast. It's a milieu of marshgrass, seabirds, +damp, clinging sea-breezes, palmettos, mosquitos, and bats. +Until 1974, it was a Navy Air Base, and still features a working runway, +and some WWII vintage blockhouses and officers' quarters. +The Center has since benefitted by a forty-million-dollar retrofit, +but there's still enough forest and swamp on the facility for the +Border Patrol to put in tracking practice. + +As a town, "Glynco" scarcely exists. The nearest real town is Brunswick, +a few miles down Highway 17, where I stayed at the aptly named Marshview +Holiday Inn. I had Sunday dinner at a seafood restaurant called "Jinright's," +where I feasted on deep-fried alligator tail. This local favorite was +a heaped basket of bite-sized chunks of white, tender, almost fluffy +reptile meat, steaming in a peppered batter crust. Alligator makes +a culinary experience that's hard to forget, especially when liberally +basted with homemade cocktail sauce from a Jinright squeeze-bottle. + +The crowded clientele were tourists, fishermen, local black folks +in their Sunday best, and white Georgian locals who all seemed +to bear an uncanny resemblance to Georgia humorist Lewis Grizzard. + +The 2,400 students from 75 federal agencies who make up the FLETC +population scarcely seem to make a dent in the low-key local scene. +The students look like tourists, and the teachers seem to have taken +on much of the relaxed air of the Deep South. My host was Mr. Carlton +Fitzpatrick, the Program Coordinator of the Financial Fraud Institute. +Carlton Fitzpatrick is a mustached, sinewy, well-tanned Alabama native +somewhere near his late forties, with a fondness for chewing tobacco, +powerful computers, and salty, down-home homilies. We'd met before, +at FCIC in Arizona. + +The Financial Fraud Institute is one of the nine divisions at FLETC. +Besides Financial Fraud, there's Driver & Marine, Firearms, +and Physical Training. These are specialized pursuits. +There are also five general training divisions: Basic Training, +Operations, Enforcement Techniques, Legal Division, and Behavioral Science. + +Somewhere in this curriculum is everything necessary to turn green college +graduates into federal agents. First they're given ID cards. Then they get +the rather miserable-looking blue coveralls known as "smurf suits." +The trainees are assigned a barracks and a cafeteria, and immediately +set on FLETC's bone-grinding physical training routine. Besides the +obligatory daily jogging--(the trainers run up danger flags beside +the track when the humidity rises high enough to threaten heat stroke)-- +here's the Nautilus machines, the martial arts, the survival skills. . . . + +The eighteen federal agencies who maintain on-site academies at FLETC +employ a wide variety of specialized law enforcement units, some of them +rather arcane. There's Border Patrol, IRS Criminal Investigation Division, +Park Service, Fish and Wildlife, Customs, Immigration, Secret Service and +the Treasury's uniformed subdivisions. . . . If you're a federal cop +and you don't work for the FBI, you train at FLETC. This includes people +as apparently obscure as the agents of the Railroad Retirement Board +Inspector General. Or the Tennessee Valley Authority Police, +who are in fact federal police officers, and can and do arrest criminals +on the federal property of the Tennessee Valley Authority. + +And then there are the computer-crime people. All sorts, all backgrounds. +Mr. Fitzpatrick is not jealous of his specialized knowledge. Cops all over, +in every branch of service, may feel a need to learn what he can teach. +Backgrounds don't matter much. Fitzpatrick himself was originally a +Border Patrol veteran, then became a Border Patrol instructor at FLETC. +His Spanish is still fluent--but he found himself strangely fascinated +when the first computers showed up at the Training Center. Fitzpatrick +did have a background in electrical engineering, and though he never +considered himself a computer hacker, he somehow found himself writing +useful little programs for this new and promising gizmo. + +He began looking into the general subject of computers and crime, +reading Donn Parker's books and articles, keeping an ear cocked +for war stories, useful insights from the field, the up-and-coming +people of the local computer-crime and high-technology units. . . . +Soon he got a reputation around FLETC as the resident "computer expert," +and that reputation alone brought him more exposure, more experience-- +until one day he looked around, and sure enough he WAS a federal +computer-crime expert. + +In fact, this unassuming, genial man may be THE federal computer-crime expert. +There are plenty of very good computer people, and plenty of very good +federal investigators, but the area where these worlds of expertise overlap +is very slim. And Carlton Fitzpatrick has been right at the center of that +since 1985, the first year of the Colluquy, a group which owes much to +his influence. + +He seems quite at home in his modest, acoustic-tiled office, +with its Ansel Adams-style Western photographic art, a gold-framed +Senior Instructor Certificate, and a towering bookcase crammed with +three-ring binders with ominous titles such as Datapro Reports on +Information Security and CFCA Telecom Security '90. + +The phone rings every ten minutes; colleagues show up at the door +to chat about new developments in locksmithing or to shake their heads +over the latest dismal developments in the BCCI global banking scandal. + +Carlton Fitzpatrick is a fount of computer-crime war-stories, +related in an acerbic drawl. He tells me the colorful tale +of a hacker caught in California some years back. He'd been +raiding systems, typing code without a detectable break, +for twenty, twenty-four, thirty-six hours straight. Not just +logged on--TYPING. Investigators were baffled. Nobody +could do that. Didn't he have to go to the bathroom? +Was it some kind of automatic keyboard-whacking device +that could actually type code? + +A raid on the suspect's home revealed a situation of astonishing squalor. +The hacker turned out to be a Pakistani computer-science student who had +flunked out of a California university. He'd gone completely underground +as an illegal electronic immigrant, and was selling stolen phone-service +to stay alive. The place was not merely messy and dirty, but in a state +of psychotic disorder. Powered by some weird mix of culture shock, +computer addiction, and amphetamines, the suspect had in fact been sitting +in front of his computer for a day and a half straight, with snacks and +drugs at hand on the edge of his desk and a chamber-pot under his chair. + +Word about stuff like this gets around in the hacker-tracker community. + +Carlton Fitzpatrick takes me for a guided tour by car around the +FLETC grounds. One of our first sights is the biggest indoor +firing range in the world. There are federal trainees in there, +Fitzpatrick assures me politely, blasting away with a wide variety +of automatic weapons: Uzis, Glocks, AK-47s. . . . He's willing to +take me inside. I tell him I'm sure that's really interesting, +but I'd rather see his computers. Carlton Fitzpatrick seems quite +surprised and pleased. I'm apparently the first journalist he's ever +seen who has turned down the shooting gallery in favor of microchips. + +Our next stop is a favorite with touring Congressmen: the three-mile +long FLETC driving range. Here trainees of the Driver & Marine Division +are taught high-speed pursuit skills, setting and breaking road-blocks, +diplomatic security driving for VIP limousines. . . . A favorite FLETC +pastime is to strap a passing Senator into the passenger seat beside a +Driver & Marine trainer, hit a hundred miles an hour, then take it right into +"the skid-pan," a section of greased track where two tons of Detroit iron +can whip and spin like a hockey puck. + +Cars don't fare well at FLETC. First they're rifled again and again +for search practice. Then they do 25,000 miles of high-speed +pursuit training; they get about seventy miles per set +of steel-belted radials. Then it's off to the skid pan, +where sometimes they roll and tumble headlong in the grease. +When they're sufficiently grease-stained, dented, and creaky, +they're sent to the roadblock unit, where they're battered without pity. +And finally then they're sacrificed to the Bureau of Alcohol, +Tobacco and Firearms, whose trainees learn the ins and outs +of car-bomb work by blowing them into smoking wreckage. + +There's a railroad box-car on the FLETC grounds, and a large +grounded boat, and a propless plane; all training-grounds for searches. +The plane sits forlornly on a patch of weedy tarmac next to an eerie +blockhouse known as the "ninja compound," where anti-terrorism specialists +practice hostage rescues. As I gaze on this creepy paragon of modern +low-intensity warfare, my nerves are jangled by a sudden staccato outburst +of automatic weapons fire, somewhere in the woods to my right. +"Nine-millimeter," Fitzpatrick judges calmly. + +Even the eldritch ninja compound pales somewhat compared +to the truly surreal area known as "the raid-houses." +This is a street lined on both sides with nondescript +concrete-block houses with flat pebbled roofs. +They were once officers' quarters. Now they are training grounds. +The first one to our left, Fitzpatrick tells me, has been specially +adapted for computer search-and-seizure practice. Inside it has been +wired for video from top to bottom, with eighteen pan-and-tilt +remotely controlled videocams mounted on walls and in corners. +Every movement of the trainee agent is recorded live by teachers, +for later taped analysis. Wasted movements, hesitations, possibly lethal +tactical mistakes--all are gone over in detail. + +Perhaps the weirdest single aspect of this building is its front door, +scarred and scuffed all along the bottom, from the repeated impact, +day after day, of federal shoe-leather. + +Down at the far end of the row of raid-houses some people are practicing +a murder. We drive by slowly as some very young and rather nervous-looking +federal trainees interview a heavyset bald man on the raid-house lawn. +Dealing with murder takes a lot of practice; first you have to learn +to control your own instinctive disgust and panic, then you have to learn +to control the reactions of a nerve-shredded crowd of civilians, +some of whom may have just lost a loved one, some of whom may be murderers-- +quite possibly both at once. + +A dummy plays the corpse. The roles of the bereaved, the morbidly curious, +and the homicidal are played, for pay, by local Georgians: waitresses, +musicians, most anybody who needs to moonlight and can learn a script. +These people, some of whom are FLETC regulars year after year, +must surely have one of the strangest jobs in the world. + +Something about the scene: "normal" people in a weird situation, +standing around talking in bright Georgia sunshine, unsuccessfully +pretending that something dreadful has gone on, while a dummy lies +inside on faked bloodstains. . . . While behind this weird masquerade, +like a nested set of Russian dolls, are grim future realities of real death, +real violence, real murders of real people, that these young agents +will really investigate, many times during their careers. . . . +Over and over. . . . Will those anticipated murders look like this, +feel like this--not as "real" as these amateur actors are trying to +make it seem, but both as "real," and as numbingly unreal, as watching +fake people standing around on a fake lawn? Something about this scene +unhinges me. It seems nightmarish to me, Kafkaesque. I simply don't +know how to take it; my head is turned around; I don't know whether to laugh, +cry, or just shudder. + +When the tour is over, Carlton Fitzpatrick and I talk about computers. +For the first time cyberspace seems like quite a comfortable place. +It seems very real to me suddenly, a place where I know what I'm talking about, +a place I'm used to. It's real. "Real." Whatever. + +Carlton Fitzpatrick is the only person I've met in cyberspace circles +who is happy with his present equipment. He's got a 5 Meg RAM PC with +a 112 meg hard disk; a 660 meg's on the way. He's got a Compaq 386 desktop, +and a Zenith 386 laptop with 120 meg. Down the hall is a NEC Multi-Sync 2A +with a CD-ROM drive and a 9600 baud modem with four com-lines. +There's a training minicomputer, and a 10-meg local mini just for the Center, +and a lab-full of student PC clones and half-a-dozen Macs or so. +There's a Data General MV 2500 with 8 meg on board and a 370 meg disk. + +Fitzpatrick plans to run a UNIX board on the Data General when he's +finished beta-testing the software for it, which he wrote himself. +It'll have E-mail features, massive files on all manner of computer-crime +and investigation procedures, and will follow the computer-security +specifics of the Department of Defense "Orange Book." He thinks +it will be the biggest BBS in the federal government. + +Will it have Phrack on it? I ask wryly. + +Sure, he tells me. Phrack, TAP, Computer Underground Digest, +all that stuff. With proper disclaimers, of course. + +I ask him if he plans to be the sysop. Running a system that size is very +time-consuming, and Fitzpatrick teaches two three-hour courses every day. + +No, he says seriously, FLETC has to get its money worth out of the instructors. +He thinks he can get a local volunteer to do it, a high-school student. + +He says a bit more, something I think about an Eagle Scout law-enforcement +liaison program, but my mind has rocketed off in disbelief. + +"You're going to put a TEENAGER in charge of a federal security BBS?" +I'm speechless. It hasn't escaped my notice that the FLETC Financial +Fraud Institute is the ULTIMATE hacker-trashing target; there is stuff in here, +stuff of such utter and consummate cool by every standard of the +digital underground. . . . + +I imagine the hackers of my acquaintance, fainting dead-away from +forbidden-knowledge greed-fits, at the mere prospect of cracking +the superultra top-secret computers used to train the Secret Service +in computer-crime. . . . + +"Uhm, Carlton," I babble, "I'm sure he's a really nice kid and all, +but that's a terrible temptation to set in front of somebody who's, +you know, into computers and just starting out. . . ." + +"Yeah," he says, "that did occur to me." For the first time I begin +to suspect that he's pulling my leg. + +He seems proudest when he shows me an ongoing project called JICC, +Joint Intelligence Control Council. It's based on the services provided +by EPIC, the El Paso Intelligence Center, which supplies data and intelligence +to the Drug Enforcement Administration, the Customs Service, the Coast Guard, +and the state police of the four southern border states. Certain EPIC files +can now be accessed by drug-enforcement police of Central America, +South America and the Caribbean, who can also trade information +among themselves. Using a telecom program called "White Hat," +written by two brothers named Lopez from the Dominican Republic, +police can now network internationally on inexpensive PCs. +Carlton Fitzpatrick is teaching a class of drug-war agents +from the Third World, and he's very proud of their progress. +Perhaps soon the sophisticated smuggling networks of the +Medellin Cartel will be matched by a sophisticated computer +network of the Medellin Cartel's sworn enemies. They'll track boats, +track contraband, track the international drug-lords who now leap over +borders with great ease, defeating the police through the clever use +of fragmented national jurisdictions. + +JICC and EPIC must remain beyond the scope of this book. +They seem to me to be very large topics fraught with complications +that I am not fit to judge. I do know, however, that the international, +computer-assisted networking of police, across national boundaries, +is something that Carlton Fitzpatrick considers very important, +a harbinger of a desirable future. I also know that networks +by their nature ignore physical boundaries. And I also know +that where you put communications you put a community, +and that when those communities become self-aware +they will fight to preserve themselves and to expand their influence. +I make no judgements whether this is good or bad. +It's just cyberspace; it's just the way things are. + +I asked Carlton Fitzpatrick what advice he would have for +a twenty-year-old who wanted to shine someday in the world +of electronic law enforcement. + +He told me that the number one rule was simply not to be +scared of computers. You don't need to be an obsessive +"computer weenie," but you mustn't be buffaloed just because +some machine looks fancy. The advantages computers give +smart crooks are matched by the advantages they give smart cops. +Cops in the future will have to enforce the law "with their heads, +not their holsters." Today you can make good cases without ever +leaving your office. In the future, cops who resist the computer +revolution will never get far beyond walking a beat. + +I asked Carlton Fitzpatrick if he had some single message for the public; +some single thing that he would most like the American public to know +about his work. + +He thought about it while. "Yes," he said finally. "TELL me the rules, +and I'll TEACH those rules!" He looked me straight in the eye. +"I do the best that I can." + + + +PART FOUR: THE CIVIL LIBERTARIANS + + +The story of the Hacker Crackdown, as we have followed it thus far, +has been technological, subcultural, criminal and legal. +The story of the Civil Libertarians, though it partakes +of all those other aspects, is profoundly and thoroughly POLITICAL. + +In 1990, the obscure, long-simmering struggle over the ownership +and nature of cyberspace became loudly and irretrievably public. +People from some of the oddest corners of American society suddenly +found themselves public figures. Some of these people found this +situation much more than they had ever bargained for. They backpedalled, +and tried to retreat back to the mandarin obscurity of their cozy +subcultural niches. This was generally to prove a mistake. + +But the civil libertarians seized the day in 1990. They found themselves +organizing, propagandizing, podium-pounding, persuading, touring, +negotiating, posing for publicity photos, submitting to interviews, +squinting in the limelight as they tried a tentative, but growingly +sophisticated, buck-and-wing upon the public stage. + +It's not hard to see why the civil libertarians should have +this competitive advantage. + +The hackers of the digital underground are an hermetic elite. +They find it hard to make any remotely convincing case for +their actions in front of the general public. Actually, +hackers roundly despise the "ignorant" public, and have never +trusted the judgement of "the system." Hackers do propagandize, +but only among themselves, mostly in giddy, badly spelled manifestos +of class warfare, youth rebellion or naive techie utopianism. +Hackers must strut and boast in order to establish and preserve +their underground reputations. But if they speak out too loudly +and publicly, they will break the fragile surface-tension of the underground, +and they will be harrassed or arrested. Over the longer term, +most hackers stumble, get busted, get betrayed, or simply give up. +As a political force, the digital underground is hamstrung. + +The telcos, for their part, are an ivory tower under protracted seige. +They have plenty of money with which to push their calculated public image, +but they waste much energy and goodwill attacking one another with +slanderous and demeaning ad campaigns. The telcos have suffered +at the hands of politicians, and, like hackers, they don't trust +the public's judgement. And this distrust may be well-founded. +Should the general public of the high-tech 1990s come to understand +its own best interests in telecommunications, that might well pose +a grave threat to the specialized technical power and authority +that the telcos have relished for over a century. The telcos do +have strong advantages: loyal employees, specialized expertise, +influence in the halls of power, tactical allies in law enforcement, +and unbelievably vast amounts of money. But politically speaking, they lack +genuine grassroots support; they simply don't seem to have many friends. + +Cops know a lot of things other people don't know. +But cops willingly reveal only those aspects of their +knowledge that they feel will meet their institutional +purposes and further public order. Cops have respect, +they have responsibilities, they have power in the streets +and even power in the home, but cops don't do particularly +well in limelight. When pressed, they will step out in the +public gaze to threaten bad-guys, or to cajole prominent citizens, +or perhaps to sternly lecture the naive and misguided. +But then they go back within their time-honored fortress +of the station-house, the courtroom and the rule-book. + +The electronic civil libertarians, however, have proven to be +born political animals. They seemed to grasp very early on +the postmodern truism that communication is power. Publicity is power. +Soundbites are power. The ability to shove one's issue onto the public +agenda--and KEEP IT THERE--is power. Fame is power. Simple personal +fluency and eloquence can be power, if you can somehow catch the +public's eye and ear. + +The civil libertarians had no monopoly on "technical power"-- +though they all owned computers, most were not particularly +advanced computer experts. They had a good deal of money, +but nowhere near the earthshaking wealth and the galaxy +of resources possessed by telcos or federal agencies. +They had no ability to arrest people. They carried +out no phreak and hacker covert dirty-tricks. + +But they really knew how to network. + +Unlike the other groups in this book, the civil libertarians +have operated very much in the open, more or less right +in the public hurly-burly. They have lectured audiences galore +and talked to countless journalists, and have learned to +refine their spiels. They've kept the cameras clicking, +kept those faxes humming, swapped that email, +run those photocopiers on overtime, licked envelopes +and spent small fortunes on airfare and long-distance. +In an information society, this open, overt, obvious activity +has proven to be a profound advantage. + +In 1990, the civil libertarians of cyberspace assembled +out of nowhere in particular, at warp speed. This "group" +(actually, a networking gaggle of interested parties +which scarcely deserves even that loose term) has almost nothing +in the way of formal organization. Those formal civil libertarian +organizations which did take an interest in cyberspace issues, +mainly the Computer Professionals for Social Responsibility +and the American Civil Liberties Union, were carried along +by events in 1990, and acted mostly as adjuncts, +underwriters or launching-pads. + +The civil libertarians nevertheless enjoyed the greatest success +of any of the groups in the Crackdown of 1990. At this writing, +their future looks rosy and the political initiative is firmly in their hands. +This should be kept in mind as we study the highly unlikely lives +and lifestyles of the people who actually made this happen. + +# + +In June 1989, Apple Computer, Inc., of Cupertino, +California, had a problem. Someone had illicitly copied +a small piece of Apple's proprietary software, software +which controlled an internal chip driving the Macintosh +screen display. This Color QuickDraw source code was +a closely guarded piece of Apple's intellectual property. +Only trusted Apple insiders were supposed to possess it. + +But the "NuPrometheus League" wanted things otherwise. +This person (or persons) made several illicit copies +of this source code, perhaps as many as two dozen. +He (or she, or they) then put those illicit floppy disks +into envelopes and mailed them to people all over America: +people in the computer industry who were associated with, +but not directly employed by, Apple Computer. + +The NuPrometheus caper was a complex, highly ideological, +and very hacker-like crime. Prometheus, it will be recalled, +stole the fire of the Gods and gave this potent gift to the +general ranks of downtrodden mankind. A similar god-in-the-manger +attitude was implied for the corporate elite of Apple Computer, +while the "Nu" Prometheus had himself cast in the role of rebel demigod. +The illicitly copied data was given away for free. + +The new Prometheus, whoever he was, escaped the +fate of the ancient Greek Prometheus, who was chained +to a rock for centuries by the vengeful gods while an eagle +tore and ate his liver. On the other hand, NuPrometheus +chickened out somewhat by comparison with his role model. +The small chunk of Color QuickDraw code he had filched +and replicated was more or less useless to Apple's +industrial rivals (or, in fact, to anyone else). +Instead of giving fire to mankind, it was more as if +NuPrometheus had photocopied the schematics for part of a Bic lighter. +The act was not a genuine work of industrial espionage. +It was best interpreted as a symbolic, deliberate slap +in the face for the Apple corporate heirarchy. + +Apple's internal struggles were well-known in the industry. Apple's founders, +Jobs and Wozniak, had both taken their leave long since. Their raucous core +of senior employees had been a barnstorming crew of 1960s Californians, +many of them markedly less than happy with the new button-down multimillion +dollar regime at Apple. Many of the programmers and developers who had +invented the Macintosh model in the early 1980s had also taken their leave of +the company. It was they, not the current masters of Apple's corporate fate, +who had invented the stolen Color QuickDraw code. The NuPrometheus stunt +was well-calculated to wound company morale. + +Apple called the FBI. The Bureau takes an interest in high-profile +intellectual-property theft cases, industrial espionage and theft +of trade secrets. These were likely the right people to call, +and rumor has it that the entities responsible were in fact discovered +by the FBI, and then quietly squelched by Apple management. NuPrometheus +was never publicly charged with a crime, or prosecuted, or jailed. +But there were no further illicit releases of Macintosh internal software. +Eventually the painful issue of NuPrometheus was allowed to fade. + +In the meantime, however, a large number of puzzled bystanders +found themselves entertaining surprise guests from the FBI. + +One of these people was John Perry Barlow. Barlow is a most unusual man, +difficult to describe in conventional terms. He is perhaps best known as +a songwriter for the Grateful Dead, for he composed lyrics for +"Hell in a Bucket," "Picasso Moon," "Mexicali Blues," "I Need a Miracle," +and many more; he has been writing for the band since 1970. + +Before we tackle the vexing question as to why a rock lyricist +should be interviewed by the FBI in a computer-crime case, +it might be well to say a word or two about the Grateful Dead. +The Grateful Dead are perhaps the most successful and long-lasting +of the numerous cultural emanations from the Haight-Ashbury district +of San Francisco, in the glory days of Movement politics and +lysergic transcendance. The Grateful Dead are a nexus, a veritable +whirlwind, of applique decals, psychedelic vans, tie-dyed T-shirts, +earth-color denim, frenzied dancing and open and unashamed drug use. +The symbols, and the realities, of Californian freak power surround +the Grateful Dead like knotted macrame. + +The Grateful Dead and their thousands of Deadhead devotees +are radical Bohemians. This much is widely understood. +Exactly what this implies in the 1990s is rather more problematic. + +The Grateful Dead are among the world's most popular +and wealthy entertainers: number 20, according to Forbes magazine, +right between M.C. Hammer and Sean Connery. In 1990, this jeans-clad +group of purported raffish outcasts earned seventeen million dollars. +They have been earning sums much along this line for quite some time now. + +And while the Dead are not investment bankers or three-piece-suit +tax specialists--they are, in point of fact, hippie musicians-- +this money has not been squandered in senseless Bohemian excess. +The Dead have been quietly active for many years, funding various +worthy activities in their extensive and widespread cultural community. + +The Grateful Dead are not conventional players in the American +power establishment. They nevertheless are something of a force +to be reckoned with. They have a lot of money and a lot of friends +in many places, both likely and unlikely. + +The Dead may be known for back-to-the-earth environmentalist rhetoric, +but this hardly makes them anti-technological Luddites. On the contrary, +like most rock musicians, the Grateful Dead have spent their entire adult +lives in the company of complex electronic equipment. They have funds to burn +on any sophisticated tool and toy that might happen to catch their fancy. +And their fancy is quite extensive. + +The Deadhead community boasts any number of recording engineers, +lighting experts, rock video mavens, electronic technicians +of all descriptions. And the drift goes both ways. Steve Wozniak, +Apple's co-founder, used to throw rock festivals. Silicon Valley rocks out. + +These are the 1990s, not the 1960s. Today, for a surprising number of people +all over America, the supposed dividing line between Bohemian and technician +simply no longer exists. People of this sort may have a set of windchimes +and a dog with a knotted kerchief 'round its neck, but they're also quite +likely to own a multimegabyte Macintosh running MIDI synthesizer software +and trippy fractal simulations. These days, even Timothy Leary himself, +prophet of LSD, does virtual-reality computer-graphics demos in +his lecture tours. + +John Perry Barlow is not a member of the Grateful Dead. He is, however, +a ranking Deadhead. + +Barlow describes himself as a "techno-crank." A vague term like +"social activist" might not be far from the mark, either. +But Barlow might be better described as a "poet"--if one keeps in mind +Percy Shelley's archaic definition of poets as "unacknowledged legislators +of the world." + +Barlow once made a stab at acknowledged legislator status. In 1987, +he narrowly missed the Republican nomination for a seat in the +Wyoming State Senate. Barlow is a Wyoming native, the third-generation +scion of a well-to-do cattle-ranching family. He is in his early forties, +married and the father of three daughters. + +Barlow is not much troubled by other people's narrow notions of consistency. +In the late 1980s, this Republican rock lyricist cattle rancher sold his ranch +and became a computer telecommunications devotee. + +The free-spirited Barlow made this transition with ease. He genuinely +enjoyed computers. With a beep of his modem, he leapt from small-town +Pinedale, Wyoming, into electronic contact with a large and lively crowd +of bright, inventive, technological sophisticates from all over the world. +Barlow found the social milieu of computing attractive: its fast-lane pace, +its blue-sky rhetoric, its open-endedness. Barlow began dabbling in +computer journalism, with marked success, as he was a quick study, +and both shrewd and eloquent. He frequently travelled to San Francisco +to network with Deadhead friends. There Barlow made extensive contacts +throughout the Californian computer community, including friendships +among the wilder spirits at Apple. + +In May 1990, Barlow received a visit from a local Wyoming agent of the FBI. +The NuPrometheus case had reached Wyoming. + +Barlow was troubled to find himself under investigation in an +area of his interests once quite free of federal attention. +He had to struggle to explain the very nature of computer-crime +to a headscratching local FBI man who specialized in cattle-rustling. +Barlow, chatting helpfully and demonstrating the wonders of his modem +to the puzzled fed, was alarmed to find all "hackers" generally under +FBI suspicion as an evil influence in the electronic community. +The FBI, in pursuit of a hacker called "NuPrometheus," were tracing +attendees of a suspect group called the Hackers Conference. + +The Hackers Conference, which had been started in 1984, was a +yearly Californian meeting of digital pioneers and enthusiasts. +The hackers of the Hackers Conference had little if anything to do +with the hackers of the digital underground. On the contrary, +the hackers of this conference were mostly well-to-do Californian +high-tech CEOs, consultants, journalists and entrepreneurs. +(This group of hackers were the exact sort of "hackers" +most likely to react with militant fury at any criminal +degradation of the term "hacker.") + +Barlow, though he was not arrested or accused of a crime, +and though his computer had certainly not gone out the door, +was very troubled by this anomaly. He carried the word to the Well. + +Like the Hackers Conference, "the Well" was an emanation of the +Point Foundation. Point Foundation, the inspiration of a wealthy +Californian 60s radical named Stewart Brand, was to be a major +launch-pad of the civil libertarian effort. + +Point Foundation's cultural efforts, like those of their fellow Bay Area +Californians the Grateful Dead, were multifaceted and multitudinous. +Rigid ideological consistency had never been a strong suit of the +Whole Earth Catalog. This Point publication had enjoyed a strong +vogue during the late 60s and early 70s, when it offered hundreds +of practical (and not so practical) tips on communitarian living, +environmentalism, and getting back-to-the-land. The Whole Earth Catalog, +and its sequels, sold two and half million copies and won a +National Book Award. + +With the slow collapse of American radical dissent, the Whole Earth Catalog +had slipped to a more modest corner of the cultural radar; but in its +magazine incarnation, CoEvolution Quarterly, the Point Foundation +continued to offer a magpie potpourri of "access to tools and ideas." + +CoEvolution Quarterly, which started in 1974, was never a widely +popular magazine. Despite periodic outbreaks of millenarian fervor, +CoEvolution Quarterly failed to revolutionize Western civilization +and replace leaden centuries of history with bright new Californian paradigms. +Instead, this propaganda arm of Point Foundation cakewalked a fine line between +impressive brilliance and New Age flakiness. CoEvolution Quarterly carried +no advertising, cost a lot, and came out on cheap newsprint with modest +black-and-white graphics. It was poorly distributed, and spread mostly +by subscription and word of mouth. + +It could not seem to grow beyond 30,000 subscribers. +And yet--it never seemed to shrink much, either. +Year in, year out, decade in, decade out, some strange +demographic minority accreted to support the magazine. +The enthusiastic readership did not seem to have much +in the way of coherent politics or ideals. It was sometimes +hard to understand what held them together (if the often bitter +debate in the letter-columns could be described as "togetherness"). + +But if the magazine did not flourish, it was resilient; it got by. +Then, in 1984, the birth-year of the Macintosh computer, +CoEvolution Quarterly suddenly hit the rapids. Point Foundation +had discovered the computer revolution. Out came the Whole Earth +Software Catalog of 1984, arousing headscratching doubts among +the tie-dyed faithful, and rabid enthusiasm among the nascent +"cyberpunk" milieu, present company included. Point Foundation +started its yearly Hackers Conference, and began to take an +extensive interest in the strange new possibilities of +digital counterculture. CoEvolution Quarterlyfolded its teepee, +replaced by Whole Earth Software Review and eventually by Whole Earth +Review (the magazine's present incarnation, currently under +the editorship of virtual-reality maven Howard Rheingold). + +1985 saw the birth of the "WELL"--the "Whole Earth 'Lectronic Link." +The Well was Point Foundation's bulletin board system. + +As boards went, the Well was an anomaly from the beginning, +and remained one. It was local to San Francisco. +It was huge, with multiple phonelines and enormous files +of commentary. Its complex UNIX-based software might be +most charitably described as "user-opaque." It was run on +a mainframe out of the rambling offices of a non-profit +cultural foundation in Sausalito. And it was crammed with +fans of the Grateful Dead. + +Though the Well was peopled by chattering hipsters of the Bay Area +counterculture, it was by no means a "digital underground" board. +Teenagers were fairly scarce; most Well users (known as "Wellbeings") +were thirty- and forty-something Baby Boomers. They tended to work +in the information industry: hardware, software, telecommunications, +media, entertainment. Librarians, academics, and journalists were +especially common on the Well, attracted by Point Foundation's +open-handed distribution of "tools and ideas." + +There were no anarchy files on the Well, scarcely a +dropped hint about access codes or credit-card theft. +No one used handles. Vicious "flame-wars" were held to +a comparatively civilized rumble. Debates were sometimes sharp, +but no Wellbeing ever claimed that a rival had disconnected his phone, +trashed his house, or posted his credit card numbers. + +The Well grew slowly as the 1980s advanced. It charged a modest sum +for access and storage, and lost money for years--but not enough to hamper +the Point Foundation, which was nonprofit anyway. By 1990, the Well +had about five thousand users. These users wandered about a gigantic +cyberspace smorgasbord of "Conferences", each conference itself consisting +of a welter of "topics," each topic containing dozens, sometimes hundreds +of comments, in a tumbling, multiperson debate that could last for months +or years on end. + + +In 1991, the Well's list of conferences looked like this: + + +CONFERENCES ON THE WELL + +WELL "Screenzine" Digest (g zine) + +Best of the WELL - vintage material - (g best) + +Index listing of new topics in all conferences - (g newtops) + +Business - Education +---------------------- + +Apple Library Users Group(g alug) Agriculture (g agri) +Brainstorming (g brain) Classifieds (g cla) +Computer Journalism (g cj) Consultants (g consult) +Consumers (g cons) Design (g design) +Desktop Publishing (g desk) Disability (g disability) +Education (g ed) Energy (g energy91) +Entrepreneurs (g entre) Homeowners (g home) +Indexing (g indexing) Investments (g invest) +Kids91 (g kids) Legal (g legal) +One Person Business (g one) +Periodical/newsletter (g per) +Telecomm Law (g tcl) The Future (g fut) +Translators (g trans) Travel (g tra) +Work (g work) + +Electronic Frontier Foundation (g eff) +Computers, Freedom & Privacy (g cfp) +Computer Professionals for Social Responsibility (g cpsr) + +Social - Political - Humanities +--------------------------------- + +Aging (g gray) AIDS (g aids) +Amnesty International (g amnesty) Archives (g arc) +Berkeley (g berk) Buddhist (g wonderland) +Christian (g cross) Couples (g couples) +Current Events (g curr) Dreams (g dream) +Drugs (g dru) East Coast (g east) +Emotional Health@@@@ (g private) Erotica (g eros) +Environment (g env) Firearms (g firearms) +First Amendment (g first) Fringes of Reason (g fringes) +Gay (g gay) Gay (Private)# (g gaypriv) +Geography (g geo) German (g german) +Gulf War (g gulf) Hawaii (g aloha) +Health (g heal) History (g hist) +Holistic (g holi) Interview (g inter) +Italian (g ital) Jewish (g jew) +Liberty (g liberty) Mind (g mind) +Miscellaneous (g misc) Men on the WELL@@ (g mow) +Network Integration (g origin) Nonprofits (g non) +North Bay (g north) Northwest (g nw) +Pacific Rim (g pacrim) Parenting (g par) +Peace (g pea) Peninsula (g pen) +Poetry (g poetry) Philosophy (g phi) +Politics (g pol) Psychology (g psy) +Psychotherapy (g therapy) Recovery## (g recovery) +San Francisco (g sanfran) Scams (g scam) +Sexuality (g sex) Singles (g singles) +Southern (g south) Spanish (g spanish) +Spirituality (g spirit) Tibet (g tibet) +Transportation (g transport) True Confessions (g tru) +Unclear (g unclear) WELL Writer's Workshop@@@(g www) +Whole Earth (g we) Women on the WELL@(g wow) +Words (g words) Writers (g wri) + +@@@@Private Conference - mail wooly for entry +@@@Private conference - mail sonia for entry +@@Private conference - mail flash for entry +@ Private conference - mail reva for entry +# Private Conference - mail hudu for entry +## Private Conference - mail dhawk for entry + +Arts - Recreation - Entertainment +----------------------------------- +ArtCom Electronic Net (g acen) +Audio-Videophilia (g aud) +Bicycles (g bike) Bay Area Tonight@@(g bat) +Boating (g wet) Books (g books) +CD's (g cd) Comics (g comics) +Cooking (g cook) Flying (g flying) +Fun (g fun) Games (g games) +Gardening (g gard) Kids (g kids) +Nightowls@ (g owl) Jokes (g jokes) +MIDI (g midi) Movies (g movies) +Motorcycling (g ride) Motoring (g car) +Music (g mus) On Stage (g onstage) +Pets (g pets) Radio (g rad) +Restaurant (g rest) Science Fiction (g sf) +Sports (g spo) Star Trek (g trek) +Television (g tv) Theater (g theater) +Weird (g weird) Zines/Factsheet Five(g f5) +@Open from midnight to 6am +@@Updated daily + +Grateful Dead +------------- +Grateful Dead (g gd) Deadplan@ (g dp) +Deadlit (g deadlit) Feedback (g feedback) +GD Hour (g gdh) Tapes (g tapes) +Tickets (g tix) Tours (g tours) + +@Private conference - mail tnf for entry + +Computers +----------- +AI/Forth/Realtime (g realtime) Amiga (g amiga) +Apple (g app) Computer Books (g cbook) +Art & Graphics (g gra) Hacking (g hack) +HyperCard (g hype) IBM PC (g ibm) +LANs (g lan) Laptop (g lap) +Macintosh (g mac) Mactech (g mactech) +Microtimes (g microx) Muchomedia (g mucho) +NeXt (g next) OS/2 (g os2) +Printers (g print) Programmer's Net (g net) +Siggraph (g siggraph) Software Design (g sdc) +Software/Programming (g software) +Software Support (g ssc) +Unix (g unix) Windows (g windows) +Word Processing (g word) + +Technical - Communications +---------------------------- +Bioinfo (g bioinfo) Info (g boing) +Media (g media) NAPLPS (g naplps) +Netweaver (g netweaver) Networld (g networld) +Packet Radio (g packet) Photography (g pho) +Radio (g rad) Science (g science) +Technical Writers (g tec) Telecommunications(g tele) +Usenet (g usenet) Video (g vid) +Virtual Reality (g vr) + +The WELL Itself +--------------- +Deeper (g deeper) Entry (g ent) +General (g gentech) Help (g help) +Hosts (g hosts) Policy (g policy) +System News (g news) Test (g test) + +The list itself is dazzling, bringing to the untutored eye +a dizzying impression of a bizarre milieu of mountain-climbing +Hawaiian holistic photographers trading true-life confessions +with bisexual word-processing Tibetans. + +But this confusion is more apparent than real. Each of these conferences +was a little cyberspace world in itself, comprising dozens and perhaps +hundreds of sub-topics. Each conference was commonly frequented by +a fairly small, fairly like-minded community of perhaps a few dozen people. +It was humanly impossible to encompass the entire Well (especially since +access to the Well's mainframe computer was billed by the hour). +Most long-time users contented themselves with a few favorite +topical neighborhoods, with the occasional foray elsewhere +for a taste of exotica. But especially important news items, +and hot topical debates, could catch the attention of the entire +Well community. + +Like any community, the Well had its celebrities, and John Perry Barlow, +the silver-tongued and silver-modemed lyricist of the Grateful Dead, +ranked prominently among them. It was here on the Well that Barlow +posted his true-life tale of computer-crime encounter with the FBI. + +The story, as might be expected, created a great stir. The Well was +already primed for hacker controversy. In December 1989, Harper's magazine +had hosted a debate on the Well about the ethics of illicit computer intrusion. +While over forty various computer-mavens took part, Barlow proved a star +in the debate. So did "Acid Phreak" and "Phiber Optik," a pair of young +New York hacker-phreaks whose skills at telco switching-station intrusion +were matched only by their apparently limitless hunger for fame. +The advent of these two boldly swaggering outlaws in the precincts +of the Well created a sensation akin to that of Black Panthers +at a cocktail party for the radically chic. + +Phiber Optik in particular was to seize the day in 1990. +A devotee of the 2600 circle and stalwart of the New York +hackers' group "Masters of Deception," Phiber Optik was +a splendid exemplar of the computer intruder as committed dissident. +The eighteen-year-old Optik, a high-school dropout and part-time +computer repairman, was young, smart, and ruthlessly obsessive, +a sharp-dressing, sharp-talking digital dude who was utterly +and airily contemptuous of anyone's rules but his own. +By late 1991, Phiber Optik had appeared in Harper's, +Esquire, The New York Times, in countless public debates +and conventions, even on a television show hosted by Geraldo Rivera. + +Treated with gingerly respect by Barlow and other Well mavens, +Phiber Optik swiftly became a Well celebrity. Strangely, despite +his thorny attitude and utter single-mindedness, Phiber Optik seemed +to arouse strong protective instincts in most of the people who met him. +He was great copy for journalists, always fearlessly ready to swagger, +and, better yet, to actually DEMONSTRATE some off-the-wall digital stunt. +He was a born media darling. + +Even cops seemed to recognize that there was something peculiarly unworldly +and uncriminal about this particular troublemaker. He was so bold, +so flagrant, so young, and so obviously doomed, that even those +who strongly disapproved of his actions grew anxious for his welfare, +and began to flutter about him as if he were an endangered seal pup. + +In January 24, 1990 (nine days after the Martin Luther King Day Crash), +Phiber Optik, Acid Phreak, and a third NYC scofflaw named Scorpion were +raided by the Secret Service. Their computers went out the door, +along with the usual blizzard of papers, notebooks, compact disks, +answering machines, Sony Walkmans, etc. Both Acid Phreak and +Phiber Optik were accused of having caused the Crash. + +The mills of justice ground slowly. The case eventually fell into +the hands of the New York State Police. Phiber had lost his machinery +in the raid, but there were no charges filed against him for over a year. +His predicament was extensively publicized on the Well, where it caused +much resentment for police tactics. It's one thing to merely hear about +a hacker raided or busted; it's another to see the police attacking someone +you've come to know personally, and who has explained his motives at length. +Through the Harper's debate on the Well, it had become clear to the +Wellbeings that Phiber Optik was not in fact going to "hurt anything." +In their own salad days, many Wellbeings had tasted tear-gas in pitched +street-battles with police. They were inclined to indulgence for +acts of civil disobedience. + +Wellbeings were also startled to learn of the draconian thoroughness +of a typical hacker search-and-seizure. It took no great stretch of +imagination for them to envision themselves suffering much the same treatment. + +As early as January 1990, sentiment on the Well had already begun to sour, +and people had begun to grumble that "hackers" were getting a raw deal +from the ham-handed powers-that-be. The resultant issue of Harper's +magazine posed the question as to whether computer-intrusion was a "crime" +at all. As Barlow put it later: "I've begun to wonder if we wouldn't +also regard spelunkers as desperate criminals if AT&T owned all the caves." + +In February 1991, more than a year after the raid on his home, +Phiber Optik was finally arrested, and was charged with first-degree +Computer Tampering and Computer Trespass, New York state offenses. +He was also charged with a theft-of-service misdemeanor, involving a complex +free-call scam to a 900 number. Phiber Optik pled guilty to the misdemeanor +charge, and was sentenced to 35 hours of community service. + +This passing harassment from the unfathomable world of straight people +seemed to bother Optik himself little if at all. Deprived of his computer +by the January search-and-seizure, he simply bought himself a portable +computer so the cops could no longer monitor the phone where he lived +with his Mom, and he went right on with his depredations, sometimes on +live radio or in front of television cameras. + +The crackdown raid may have done little to dissuade Phiber Optik, +but its galling affect on the Wellbeings was profound. As 1990 rolled on, +the slings and arrows mounted: the Knight Lightning raid, +the Steve Jackson raid, the nation-spanning Operation Sundevil. +The rhetoric of law enforcement made it clear that there was, +in fact, a concerted crackdown on hackers in progress. + +The hackers of the Hackers Conference, the Wellbeings, and their ilk, +did not really mind the occasional public misapprehension of "hacking;" +if anything, this membrane of differentiation from straight society +made the "computer community" feel different, smarter, better. +They had never before been confronted, however, by a concerted +vilification campaign. + +Barlow's central role in the counter-struggle was one of the major +anomalies of 1990. Journalists investigating the controversy +often stumbled over the truth about Barlow, but they commonly +dusted themselves off and hurried on as if nothing had happened. +It was as if it were TOO MUCH TO BELIEVE that a 1960s freak +from the Grateful Dead had taken on a federal law enforcement operation +head-to-head and ACTUALLY SEEMED TO BE WINNING! + +Barlow had no easily detectable power-base for a political struggle +of this kind. He had no formal legal or technical credentials. +Barlow was, however, a computer networker of truly stellar brilliance. +He had a poet's gift of concise, colorful phrasing. He also had a +journalist's shrewdness, an off-the-wall, self-deprecating wit, +and a phenomenal wealth of simple personal charm. + +The kind of influence Barlow possessed is fairly common currency +in literary, artistic, or musical circles. A gifted critic can +wield great artistic influence simply through defining +the temper of the times, by coining the catch-phrases +and the terms of debate that become the common currency of the period. +(And as it happened, Barlow WAS a part-time art critic, +with a special fondness for the Western art of Frederic Remington.) + +Barlow was the first commentator to adopt William Gibson's +striking science-fictional term "cyberspace" as a synonym +for the present-day nexus of computer and telecommunications networks. +Barlow was insistent that cyberspace should be regarded as +a qualitatively new world, a "frontier." According to Barlow, +the world of electronic communications, now made visible through +the computer screen, could no longer be usefully regarded +as just a tangle of high-tech wiring. Instead, it had become +a PLACE, cyberspace, which demanded a new set of metaphors, +a new set of rules and behaviors. The term, as Barlow employed it, +struck a useful chord, and this concept of cyberspace was picked up +by Time, Scientific American, computer police, hackers, and even +Constitutional scholars. "Cyberspace" now seems likely to become +a permanent fixture of the language. + +Barlow was very striking in person: a tall, craggy-faced, bearded, +deep-voiced Wyomingan in a dashing Western ensemble of jeans, jacket, +cowboy boots, a knotted throat-kerchief and an ever-present Grateful Dead +cloisonne lapel pin. + +Armed with a modem, however, Barlow was truly in his element. +Formal hierarchies were not Barlow's strong suit; he rarely missed +a chance to belittle the "large organizations and their drones," +with their uptight, institutional mindset. Barlow was very much +of the free-spirit persuasion, deeply unimpressed by brass-hats +and jacks-in-office. But when it came to the digital grapevine, +Barlow was a cyberspace ad-hocrat par excellence. + +There was not a mighty army of Barlows. There was only one Barlow, +and he was a fairly anomolous individual. However, the situation only +seemed to REQUIRE a single Barlow. In fact, after 1990, many people +must have concluded that a single Barlow was far more than +they'd ever bargained for. + +Barlow's querulous mini-essay about his encounter with the FBI +struck a strong chord on the Well. A number of other free spirits +on the fringes of Apple Computing had come under suspicion, +and they liked it not one whit better than he did. + +One of these was Mitchell Kapor, the co-inventor of the spreadsheet +program "Lotus 1-2-3" and the founder of Lotus Development Corporation. +Kapor had written-off the passing indignity of being fingerprinted +down at his own local Boston FBI headquarters, but Barlow's post +made the full national scope of the FBI's dragnet clear to Kapor. +The issue now had Kapor's full attention. As the Secret Service +swung into anti-hacker operation nationwide in 1990, Kapor watched +every move with deep skepticism and growing alarm. + +As it happened, Kapor had already met Barlow, who had interviewed Kapor +for a California computer journal. Like most people who met Barlow, +Kapor had been very taken with him. Now Kapor took it upon himself +to drop in on Barlow for a heart-to-heart talk about the situation. + +Kapor was a regular on the Well. Kapor had been a devotee of the +Whole Earth Catalogsince the beginning, and treasured a complete run +of the magazine. And Kapor not only had a modem, but a private jet. +In pursuit of the scattered high-tech investments of Kapor Enterprises Inc., +his personal, multi-million dollar holding company, Kapor commonly crossed +state lines with about as much thought as one might give to faxing a letter. + +The Kapor-Barlow council of June 1990, in Pinedale, Wyoming, was the start +of the Electronic Frontier Foundation. Barlow swiftly wrote a manifesto, +"Crime and Puzzlement," which announced his, and Kapor's, intention +to form a political organization to "raise and disburse funds for education, +lobbying, and litigation in the areas relating to digital speech and the +extension of the Constitution into Cyberspace." + +Furthermore, proclaimed the manifesto, the foundation would +"fund, conduct, and support legal efforts to demonstrate +that the Secret Service has exercised prior restraint on publications, +limited free speech, conducted improper seizure of equipment and data, +used undue force, and generally conducted itself in a fashion which +is arbitrary, oppressive, and unconstitutional." + +"Crime and Puzzlement" was distributed far and wide through computer +networking channels, and also printed in the Whole Earth Review. +The sudden declaration of a coherent, politicized counter-strike +from the ranks of hackerdom electrified the community. Steve Wozniak +(perhaps a bit stung by the NuPrometheus scandal) swiftly offered +to match any funds Kapor offered the Foundation. + +John Gilmore, one of the pioneers of Sun Microsystems, immediately offered +his own extensive financial and personal support. Gilmore, an ardent +libertarian, was to prove an eloquent advocate of electronic privacy issues, +especially freedom from governmental and corporate computer-assisted +surveillance of private citizens. + +A second meeting in San Francisco rounded up further allies: +Stewart Brand of the Point Foundation, virtual-reality pioneers +Jaron Lanier and Chuck Blanchard, network entrepreneur and venture +capitalist Nat Goldhaber. At this dinner meeting, the activists settled on +a formal title: the Electronic Frontier Foundation, Incorporated. +Kapor became its president. A new EFF Conference was opened on +the Point Foundation's Well, and the Well was declared +"the home of the Electronic Frontier Foundation." + +Press coverage was immediate and intense. Like their +nineteenth-century spiritual ancestors, Alexander Graham Bell +and Thomas Watson, the high-tech computer entrepreneurs +of the 1970s and 1980s--people such as Wozniak, Jobs, Kapor, +Gates, and H. Ross Perot, who had raised themselves by their bootstraps +to dominate a glittering new industry--had always made very good copy. + +But while the Wellbeings rejoiced, the press in general seemed +nonplussed by the self-declared "civilizers of cyberspace." +EFF's insistence that the war against "hackers" involved grave +Constitutional civil liberties issues seemed somewhat farfetched, +especially since none of EFF's organizers were lawyers +or established politicians. The business press in particular +found it easier to seize on the apparent core of the story-- +that high-tech entrepreneur Mitchell Kapor had established +a "defense fund for hackers." Was EFF a genuinely important +political development--or merely a clique of wealthy eccentrics, +dabbling in matters better left to the proper authorities? +The jury was still out. + +But the stage was now set for open confrontation. +And the first and the most critical battle was the +hacker show-trial of "Knight Lightning." + +# + +It has been my practice throughout this book to refer to hackers +only by their "handles." There is little to gain by giving +the real names of these people, many of whom are juveniles, +many of whom have never been convicted of any crime, and many +of whom had unsuspecting parents who have already suffered enough. + +But the trial of Knight Lightning on July 24-27, 1990, +made this particular "hacker" a nationally known public figure. +It can do no particular harm to himself or his family if I repeat +the long-established fact that his name is Craig Neidorf (pronounced NYE-dorf). + +Neidorf's jury trial took place in the United States District Court, +Northern District of Illinois, Eastern Division, with the +Honorable Nicholas J. Bua presiding. The United States of America +was the plaintiff, the defendant Mr. Neidorf. The defendant's attorney +was Sheldon T. Zenner of the Chicago firm of Katten, Muchin and Zavis. + +The prosecution was led by the stalwarts of the Chicago Computer Fraud +and Abuse Task Force: William J. Cook, Colleen D. Coughlin, and +David A. Glockner, all Assistant United States Attorneys. +The Secret Service Case Agent was Timothy M. Foley. + +It will be recalled that Neidorf was the co-editor of an underground hacker +"magazine" called Phrack. Phrack was an entirely electronic publication, +distributed through bulletin boards and over electronic networks. +It was amateur publication given away for free. Neidorf had never made +any money for his work in Phrack. Neither had his unindicted co-editor +"Taran King" or any of the numerous Phrack contributors. + +The Chicago Computer Fraud and Abuse Task Force, however, +had decided to prosecute Neidorf as a fraudster. +To formally admit that Phrack was a "magazine" +and Neidorf a "publisher" was to open a prosecutorial +Pandora's Box of First Amendment issues. To do this +was to play into the hands of Zenner and his EFF advisers, +which now included a phalanx of prominent New York civil rights +lawyers as well as the formidable legal staff of Katten, Muchin and Zavis. +Instead, the prosecution relied heavily on the issue of access device fraud: +Section 1029 of Title 18, the section from which the Secret Service drew +its most direct jurisdiction over computer crime. + +Neidorf's alleged crimes centered around the E911 Document. +He was accused of having entered into a fraudulent scheme with the Prophet, +who, it will be recalled, was the Atlanta LoD member who had illicitly +copied the E911 Document from the BellSouth AIMSX system. + +The Prophet himself was also a co-defendant in the Neidorf case, +part-and-parcel of the alleged "fraud scheme" to "steal" BellSouth's +E911 Document (and to pass the Document across state lines, +which helped establish the Neidorf trial as a federal case). +The Prophet, in the spirit of full co-operation, had agreed +to testify against Neidorf. + +In fact, all three of the Atlanta crew stood ready to testify against Neidorf. +Their own federal prosecutors in Atlanta had charged the Atlanta Three with: +(a) conspiracy, (b) computer fraud, (c) wire fraud, (d) access device fraud, +and (e) interstate transportation of stolen property (Title 18, Sections 371, +1030, 1343, 1029, and 2314). + +Faced with this blizzard of trouble, Prophet and Leftist had ducked +any public trial and had pled guilty to reduced charges--one conspiracy +count apiece. Urvile had pled guilty to that odd bit of Section 1029 +which makes it illegal to possess "fifteen or more" illegal access devices +(in his case, computer passwords). And their sentences were scheduled +for September 14, 1990--well after the Neidorf trial. As witnesses, +they could presumably be relied upon to behave. + +Neidorf, however, was pleading innocent. Most everyone else caught up +in the crackdown had "cooperated fully" and pled guilty in hope +of reduced sentences. (Steve Jackson was a notable exception, +of course, and had strongly protested his innocence from the +very beginning. But Steve Jackson could not get a day in court-- +Steve Jackson had never been charged with any crime in the first place.) + +Neidorf had been urged to plead guilty. But Neidorf was a political science +major and was disinclined to go to jail for "fraud" when he had not made +any money, had not broken into any computer, and had been publishing +a magazine that he considered protected under the First Amendment. + +Neidorf's trial was the ONLY legal action of the entire Crackdown +that actually involved bringing the issues at hand out for a public test +in front of a jury of American citizens. + +Neidorf, too, had cooperated with investigators. He had voluntarily +handed over much of the evidence that had led to his own indictment. +He had already admitted in writing that he knew that the E911 Document +had been stolen before he had "published" it in Phrack--or, from the +prosecution's point of view, illegally transported stolen property by wire +in something purporting to be a "publication." + +But even if the "publication" of the E911 Document was not held to be a crime, +that wouldn't let Neidorf off the hook. Neidorf had still received +the E911 Document when Prophet had transferred it to him from Rich Andrews' +Jolnet node. On that occasion, it certainly hadn't been "published"-- +it was hacker booty, pure and simple, transported across state lines. + +The Chicago Task Force led a Chicago grand jury to indict Neidorf +on a set of charges that could have put him in jail for thirty years. +When some of these charges were successfully challenged before Neidorf +actually went to trial, the Chicago Task Force rearranged his +indictment so that he faced a possible jail term of over sixty years! +As a first offender, it was very unlikely that Neidorf would in fact +receive a sentence so drastic; but the Chicago Task Force clearly +intended to see Neidorf put in prison, and his conspiratorial "magazine" +put permanently out of commission. This was a federal case, and Neidorf +was charged with the fraudulent theft of property worth almost +eighty thousand dollars. + +William Cook was a strong believer in high-profile prosecutions +with symbolic overtones. He often published articles on his work +in the security trade press, arguing that "a clear message had +to be sent to the public at large and the computer community +in particular that unauthorized attacks on computers and the theft +of computerized information would not be tolerated by the courts." + +The issues were complex, the prosecution's tactics somewhat unorthodox, +but the Chicago Task Force had proved sure-footed to date. "Shadowhawk" +had been bagged on the wing in 1989 by the Task Force, and sentenced +to nine months in prison, and a $10,000 fine. The Shadowhawk case involved +charges under Section 1030, the "federal interest computer" section. + +Shadowhawk had not in fact been a devotee of "federal-interest" computers +per se. On the contrary, Shadowhawk, who owned an AT&T home computer, +seemed to cherish a special aggression toward AT&T. He had bragged on +the underground boards "Phreak Klass 2600" and "Dr. Ripco" of his skills +at raiding AT&T, and of his intention to crash AT&T's national phone system. +Shadowhawk's brags were noticed by Henry Kluepfel of Bellcore Security, +scourge of the outlaw boards, whose relations with the Chicago Task Force +were long and intimate. + +The Task Force successfully established that Section 1030 applied to +the teenage Shadowhawk, despite the objections of his defense attorney. +Shadowhawk had entered a computer "owned" by U.S. Missile Command +and merely "managed" by AT&T. He had also entered an AT&T computer +located at Robbins Air Force Base in Georgia. Attacking AT&T was +of "federal interest" whether Shadowhawk had intended it or not. + +The Task Force also convinced the court that a piece of AT&T +software that Shadowhawk had illicitly copied from Bell Labs, +the "Artificial Intelligence C5 Expert System," was worth a cool +one million dollars. Shadowhawk's attorney had argued that +Shadowhawk had not sold the program and had made no profit from +the illicit copying. And in point of fact, the C5 Expert System +was experimental software, and had no established market value +because it had never been on the market in the first place. +AT&T's own assessment of a "one million dollar" figure for its +own intangible property was accepted without challenge +by the court, however. And the court concurred with +the government prosecutors that Shadowhawk showed clear +"intent to defraud" whether he'd gotten any money or not. +Shadowhawk went to jail. + +The Task Force's other best-known triumph had been the conviction +and jailing of "Kyrie." Kyrie, a true denizen of the digital +criminal underground, was a 36-year-old Canadian woman, +convicted and jailed for telecommunications fraud in Canada. +After her release from prison, she had fled the wrath of Canada Bell +and the Royal Canadian Mounted Police, and eventually settled, +very unwisely, in Chicago. + +"Kyrie," who also called herself "Long Distance Information," +specialized in voice-mail abuse. She assembled large numbers +of hot long-distance codes, then read them aloud into a series +of corporate voice-mail systems. Kyrie and her friends were +electronic squatters in corporate voice-mail systems, +using them much as if they were pirate bulletin boards, +then moving on when their vocal chatter clogged the system +and the owners necessarily wised up. Kyrie's camp followers +were a loose tribe of some hundred and fifty phone-phreaks, +who followed her trail of piracy from machine to machine, +ardently begging for her services and expertise. + +Kyrie's disciples passed her stolen credit-card numbers, +in exchange for her stolen "long distance information." +Some of Kyrie's clients paid her off in cash, by scamming +credit-card cash advances from Western Union. + +Kyrie travelled incessantly, mostly through airline tickets +and hotel rooms that she scammed through stolen credit cards. +Tiring of this, she found refuge with a fellow female phone +phreak in Chicago. Kyrie's hostess, like a surprising number +of phone phreaks, was blind. She was also physically disabled. +Kyrie allegedly made the best of her new situation by applying for, +and receiving, state welfare funds under a false identity as +a qualified caretaker for the handicapped. + +Sadly, Kyrie's two children by a former marriage had also vanished +underground with her; these pre-teen digital refugees had no legal +American identity, and had never spent a day in school. + +Kyrie was addicted to technical mastery and enthralled by her own +cleverness and the ardent worship of her teenage followers. +This foolishly led her to phone up Gail Thackeray in Arizona, +to boast, brag, strut, and offer to play informant. +Thackeray, however, had already learned far more +than enough about Kyrie, whom she roundly despised +as an adult criminal corrupting minors, a "female Fagin." +Thackeray passed her tapes of Kyrie's boasts to the Secret Service. + +Kyrie was raided and arrested in Chicago in May 1989. +She confessed at great length and pled guilty. + +In August 1990, Cook and his Task Force colleague Colleen Coughlin +sent Kyrie to jail for 27 months, for computer and telecommunications fraud. +This was a markedly severe sentence by the usual wrist-slapping standards +of "hacker" busts. Seven of Kyrie's foremost teenage disciples were also +indicted and convicted. The Kyrie "high-tech street gang," as Cook +described it, had been crushed. Cook and his colleagues had been +the first ever to put someone in prison for voice-mail abuse. +Their pioneering efforts had won them attention and kudos. + +In his article on Kyrie, Cook drove the message home to the readers +of Security Management magazine, a trade journal for corporate +security professionals. The case, Cook said, and Kyrie's stiff sentence, +"reflect a new reality for hackers and computer crime victims in the +'90s. . . . Individuals and corporations who report computer +and telecommunications crimes can now expect that their cooperation +with federal law enforcement will result in meaningful punishment. +Companies and the public at large must report computer-enhanced +crimes if they want prosecutors and the course to protect their rights +to the tangible and intangible property developed and stored on computers." + +Cook had made it his business to construct this "new reality for hackers." +He'd also made it his business to police corporate property rights +to the intangible. + +Had the Electronic Frontier Foundation been a "hacker defense fund" +as that term was generally understood, they presumably would have stood up +for Kyrie. Her 1990 sentence did indeed send a "message" that federal heat +was coming down on "hackers." But Kyrie found no defenders at EFF, +or anywhere else, for that matter. EFF was not a bail-out fund +for electronic crooks. + +The Neidorf case paralleled the Shadowhawk case in certain ways. +The victim once again was allowed to set the value of the "stolen" property. +Once again Kluepfel was both investigator and technical advisor. +Once again no money had changed hands, but the "intent to defraud" was central. + +The prosecution's case showed signs of weakness early on. The Task Force +had originally hoped to prove Neidorf the center of a nationwide +Legion of Doom criminal conspiracy. The Phrack editors threw physical +get-togethers every summer, which attracted hackers from across the country; +generally two dozen or so of the magazine's favorite contributors and readers. +(Such conventions were common in the hacker community; 2600 Magazine, +for instance, held public meetings of hackers in New York, every month.) +LoD heavy-dudes were always a strong presence at these Phrack-sponsored +"Summercons." + +In July 1988, an Arizona hacker named "Dictator" attended Summercon +in Neidorf's home town of St. Louis. Dictator was one of Gail Thackeray's +underground informants; Dictator's underground board in Phoenix was +a sting operation for the Secret Service. Dictator brought an undercover +crew of Secret Service agents to Summercon. The agents bored spyholes +through the wall of Dictator's hotel room in St Louis, and videotaped +the frolicking hackers through a one-way mirror. As it happened, +however, nothing illegal had occurred on videotape, other than the +guzzling of beer by a couple of minors. Summercons were social events, +not sinister cabals. The tapes showed fifteen hours of raucous laughter, +pizza-gobbling, in-jokes and back-slapping. + +Neidorf's lawyer, Sheldon Zenner, saw the Secret Service tapes +before the trial. Zenner was shocked by the complete harmlessness +of this meeting, which Cook had earlier characterized as a sinister +interstate conspiracy to commit fraud. Zenner wanted to show the +Summercon tapes to the jury. It took protracted maneuverings +by the Task Force to keep the tapes from the jury as "irrelevant." + +The E911 Document was also proving a weak reed. It had originally +been valued at $79,449. Unlike Shadowhawk's arcane Artificial Intelligence +booty, the E911 Document was not software--it was written in English. +Computer-knowledgeable people found this value--for a twelve-page +bureaucratic document--frankly incredible. In his "Crime and Puzzlement" +manifesto for EFF, Barlow commented: "We will probably never know how +this figure was reached or by whom, though I like to imagine an appraisal +team consisting of Franz Kafka, Joseph Heller, and Thomas Pynchon." + +As it happened, Barlow was unduly pessimistic. The EFF did, in fact, +eventually discover exactly how this figure was reached, and by whom-- +but only in 1991, long after the Neidorf trial was over. + +Kim Megahee, a Southern Bell security manager, +had arrived at the document's value by simply adding up +the "costs associated with the production" of the E911 Document. +Those "costs" were as follows: + +1. A technical writer had been hired to research and write the E911 Document. + 200 hours of work, at $35 an hour, cost : $7,000. A Project Manager had + overseen the technical writer. 200 hours, at $31 an hour, made: $6,200. + +2. A week of typing had cost $721 dollars. A week of formatting had + cost $721. A week of graphics formatting had cost $742. + +3. Two days of editing cost $367. + +4. A box of order labels cost five dollars. + +5. Preparing a purchase order for the Document, including typing + and the obtaining of an authorizing signature from within the + BellSouth bureaucracy, cost $129. + +6. Printing cost $313. Mailing the Document to fifty people + took fifty hours by a clerk, and cost $858. + +7. Placing the Document in an index took two clerks an hour each, + totalling $43. + +Bureaucratic overhead alone, therefore, was alleged to have cost +a whopping $17,099. According to Mr. Megahee, the typing +of a twelve-page document had taken a full week. Writing it +had taken five weeks, including an overseer who apparently +did nothing else but watch the author for five weeks. +Editing twelve pages had taken two days. Printing and mailing +an electronic document (which was already available on the +Southern Bell Data Network to any telco employee who needed it), +had cost over a thousand dollars. + +But this was just the beginning. There were also the HARDWARE EXPENSES. +Eight hundred fifty dollars for a VT220 computer monitor. +THIRTY-ONE THOUSAND DOLLARS for a sophisticated VAXstation II computer. +Six thousand dollars for a computer printer. TWENTY-TWO THOUSAND DOLLARS +for a copy of "Interleaf" software. Two thousand five hundred dollars +for VMS software. All this to create the twelve-page Document. + +Plus ten percent of the cost of the software and the hardware, for maintenance. +(Actually, the ten percent maintenance costs, though mentioned, had been left +off the final $79,449 total, apparently through a merciful oversight). + +Mr. Megahee's letter had been mailed directly to William Cook himself, +at the office of the Chicago federal attorneys. The United States Government +accepted these telco figures without question. + +As incredulity mounted, the value of the E911 Document was officially +revised downward. This time, Robert Kibler of BellSouth Security +estimated the value of the twelve pages as a mere $24,639.05--based, +purportedly, on "R&D costs." But this specific estimate, +right down to the nickel, did not move the skeptics at all; +in fact it provoked open scorn and a torrent of sarcasm. + +The financial issues concerning theft of proprietary information +have always been peculiar. It could be argued that BellSouth +had not "lost" its E911 Document at all in the first place, +and therefore had not suffered any monetary damage from this "theft." +And Sheldon Zenner did in fact argue this at Neidorf's trial-- +that Prophet's raid had not been "theft," but was better understood +as illicit copying. + +The money, however, was not central to anyone's true purposes in this trial. +It was not Cook's strategy to convince the jury that the E911 Document +was a major act of theft and should be punished for that reason alone. +His strategy was to argue that the E911 Document was DANGEROUS. +It was his intention to establish that the E911 Document was "a road-map" +to the Enhanced 911 System. Neidorf had deliberately and recklessly +distributed a dangerous weapon. Neidorf and the Prophet did not care +(or perhaps even gloated at the sinister idea) that the E911 Document +could be used by hackers to disrupt 911 service, "a life line for every +person certainly in the Southern Bell region of the United States, +and indeed, in many communities throughout the United States," +in Cook's own words. Neidorf had put people's lives in danger. + +In pre-trial maneuverings, Cook had established that the E911 Document +was too hot to appear in the public proceedings of the Neidorf trial. +The JURY ITSELF would not be allowed to ever see this Document, +lest it slip into the official court records, and thus into the hands +of the general public, and, thus, somehow, to malicious hackers +who might lethally abuse it. + +Hiding the E911 Document from the jury may have been a +clever legal maneuver, but it had a severe flaw. There were, +in point of fact, hundreds, perhaps thousands, of people, +already in possession of the E911 Document, just as Phrack +had published it. Its true nature was already obvious +to a wide section of the interested public (all of whom, +by the way, were, at least theoretically, party to +a gigantic wire-fraud conspiracy). Most everyone +in the electronic community who had a modem and any +interest in the Neidorf case already had a copy of the Document. +It had already been available in Phrack for over a year. + +People, even quite normal people without any particular +prurient interest in forbidden knowledge, did not shut their eyes +in terror at the thought of beholding a "dangerous" document +from a telephone company. On the contrary, they tended to trust +their own judgement and simply read the Document for themselves. +And they were not impressed. + +One such person was John Nagle. Nagle was a forty-one-year-old +professional programmer with a masters' degree in computer science +from Stanford. He had worked for Ford Aerospace, where he had invented +a computer-networking technique known as the "Nagle Algorithm," +and for the prominent Californian computer-graphics firm "Autodesk," +where he was a major stockholder. + +Nagle was also a prominent figure on the Well, much respected +for his technical knowledgeability. + +Nagle had followed the civil-liberties debate closely, +for he was an ardent telecommunicator. He was no particular friend +of computer intruders, but he believed electronic publishing +had a great deal to offer society at large, and attempts +to restrain its growth, or to censor free electronic expression, +strongly roused his ire. + +The Neidorf case, and the E911 Document, were both being discussed +in detail on the Internet, in an electronic publication called Telecom Digest. +Nagle, a longtime Internet maven, was a regular reader of Telecom Digest. +Nagle had never seen a copy of Phrack, but the implications of the case +disturbed him. + +While in a Stanford bookstore hunting books on robotics, +Nagle happened across a book called The Intelligent Network. +Thumbing through it at random, Nagle came across an entire chapter +meticulously detailing the workings of E911 police emergency systems. +This extensive text was being sold openly, and yet in Illinois +a young man was in danger of going to prison for publishing +a thin six-page document about 911 service. + +Nagle made an ironic comment to this effect in Telecom Digest. +From there, Nagle was put in touch with Mitch Kapor, +and then with Neidorf's lawyers. + +Sheldon Zenner was delighted to find a computer telecommunications expert +willing to speak up for Neidorf, one who was not a wacky teenage "hacker." +Nagle was fluent, mature, and respectable; he'd once had a federal +security clearance. + +Nagle was asked to fly to Illinois to join the defense team. + +Having joined the defense as an expert witness, Nagle read the entire +E911 Document for himself. He made his own judgement about its potential +for menace. + +The time has now come for you yourself, the reader, to have a look +at the E911 Document. This six-page piece of work was the pretext +for a federal prosecution that could have sent an electronic publisher +to prison for thirty, or even sixty, years. It was the pretext +for the search and seizure of Steve Jackson Games, a legitimate publisher +of printed books. It was also the formal pretext for the search +and seizure of the Mentor's bulletin board, "Phoenix Project," +and for the raid on the home of Erik Bloodaxe. It also had much +to do with the seizure of Richard Andrews' Jolnet node +and the shutdown of Charles Boykin's AT&T node. +The E911 Document was the single most important piece +of evidence in the Hacker Crackdown. There can be no real +and legitimate substitute for the Document itself. + + +==Phrack Inc.== + +Volume Two, Issue 24, File 5 of 13 + +Control Office Administration +Of Enhanced 911 Services For +Special Services and Account Centers + +by the Eavesdropper + +March, 1988 + + +Description of Service +~~~~~~~~~~~~~~~~~~~~~ +The control office for Emergency 911 service is assigned in +accordance with the existing standard guidelines to one of +the following centers: + +o Special Services Center (SSC) +o Major Accounts Center (MAC) +o Serving Test Center (STC) +o Toll Control Center (TCC) + +The SSC/MAC designation is used in this document interchangeably +for any of these four centers. The Special Services Centers (SSCs) +or Major Account Centers (MACs) have been designated as the trouble +reporting contact for all E911 customer (PSAP) reported troubles. +Subscribers who have trouble on an E911 call will continue +to contact local repair service (CRSAB) who will refer the +trouble to the SSC/MAC, when appropriate. + +Due to the critical nature of E911 service, the control +and timely repair of troubles is demanded. As the primary +E911 customer contact, the SSC/MAC is in the unique position +to monitor the status of the trouble and insure its resolution. + +System Overview +~~~~~~~~~~~~~~ +The number 911 is intended as a nationwide universal +telephone number which provides the public with direct +access to a Public Safety Answering Point (PSAP). A PSAP +is also referred to as an Emergency Service Bureau (ESB). +A PSAP is an agency or facility which is authorized by a +municipality to receive and respond to police, fire and/or +ambulance services. One or more attendants are located +at the PSAP facilities to receive and handle calls of an +emergency nature in accordance with the local municipal +requirements. + +An important advantage of E911 emergency service is +improved (reduced) response times for emergency +services. Also close coordination among agencies +providing various emergency services is a valuable +capability provided by E911 service. + +1A ESS is used as the tandem office for the E911 network to +route all 911 calls to the correct (primary) PSAP designated +to serve the calling station. The E911 feature was +developed primarily to provide routing to the correct PSAP +for all 911 calls. Selective routing allows a 911 call +originated from a particular station located in a particular +district, zone, or town, to be routed to the primary PSAP +designated to serve that customer station regardless of +wire center boundaries. Thus, selective routing eliminates +the problem of wire center boundaries not coinciding with +district or other political boundaries. + +The services available with the E911 feature include: + +Forced Disconnect Default Routing +Alternative Routing Night Service +Selective Routing Automatic Number +Identification (ANI) +Selective Transfer Automatic Location +Identification (ALI) + + +Preservice/Installation Guidelines +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +When a contract for an E911 system has been signed, it is +the responsibility of Network Marketing to establish an +implementation/cutover committee which should include +a representative from the SSC/MAC. Duties of the E911 +Implementation Team include coordination of all phases +of the E911 system deployment and the formation of an +on-going E911 maintenance subcommittee. + +Marketing is responsible for providing the following +customer specific information to the SSC/MAC prior to +the start of call through testing: + +o All PSAP's (name, address, local contact) +o All PSAP circuit ID's +o 1004 911 service request including PSAP details on each PSAP + (1004 Section K, L, M) +o Network configuration +o Any vendor information (name, telephone number, equipment) + +The SSC/MAC needs to know if the equipment and sets +at the PSAP are maintained by the BOCs, an independent +company, or an outside vendor, or any combination. +This information is then entered on the PSAP profile sheets +and reviewed quarterly for changes, additions and deletions. + +Marketing will secure the Major Account Number (MAN) +and provide this number to Corporate Communications +so that the initial issue of the service orders carry +the MAN and can be tracked by the SSC/MAC via CORDNET. +PSAP circuits are official services by definition. + +All service orders required for the installation of the E911 +system should include the MAN assigned to the city/county +which has purchased the system. + +In accordance with the basic SSC/MAC strategy for provisioning, +the SSC/MAC will be Overall Control Office (OCO) for all Node +to PSAP circuits (official services) and any other services +for this customer. Training must be scheduled for all SSC/MAC +involved personnel during the pre-service stage of the project. + +The E911 Implementation Team will form the on-going +maintenance subcommittee prior to the initial +implementation of the E911 system. This sub-committee +will establish post implementation quality assurance +procedures to ensure that the E911 system continues to +provide quality service to the customer. +Customer/Company training, trouble reporting interfaces +for the customer, telephone company and any involved +independent telephone companies needs to be addressed +and implemented prior to E911 cutover. These functions +can be best addressed by the formation of a sub- +committee of the E911 Implementation Team to set up +guidelines for and to secure service commitments of +interfacing organizations. A SSC/MAC supervisor should +chair this subcommittee and include the following +organizations: + +1) Switching Control Center + - E911 translations + - Trunking + - End office and Tandem office hardware/software +2) Recent Change Memory Administration Center + - Daily RC update activity for TN/ESN translations + - Processes validity errors and rejects +3) Line and Number Administration + - Verification of TN/ESN translations +4) Special Service Center/Major Account Center + - Single point of contact for all PSAP and Node to host troubles + - Logs, tracks & statusing of all trouble reports + - Trouble referral, follow up, and escalation + - Customer notification of status and restoration + - Analyzation of "chronic" troubles + - Testing, installation and maintenance of E911 circuits +5) Installation and Maintenance (SSIM/I&M) + - Repair and maintenance of PSAP equipment and Telco owned sets +6) Minicomputer Maintenance Operations Center + - E911 circuit maintenance (where applicable) +7) Area Maintenance Engineer + - Technical assistance on voice (CO-PSAP) network related E911 troubles + + +Maintenance Guidelines +~~~~~~~~~~~~~~~~~~~~~ +The CCNC will test the Node circuit from the 202T at the +Host site to the 202T at the Node site. Since Host to Node +(CCNC to MMOC) circuits are official company services, +the CCNC will refer all Node circuit troubles to the +SSC/MAC. The SSC/MAC is responsible for the testing +and follow up to restoration of these circuit troubles. + +Although Node to PSAP circuit are official services, the +MMOC will refer PSAP circuit troubles to the appropriate +SSC/MAC. The SSC/MAC is responsible for testing and +follow up to restoration of PSAP circuit troubles. + +The SSC/MAC will also receive reports from +CRSAB/IMC(s) on subscriber 911 troubles when they are +not line troubles. The SSC/MAC is responsible for testing +and restoration of these troubles. + +Maintenance responsibilities are as follows: + +SCC@ Voice Network (ANI to PSAP) +@SCC responsible for tandem switch + +SSIM/I&M PSAP Equipment (Modems, CIU's, sets) +Vendor PSAP Equipment (when CPE) +SSC/MAC PSAP to Node circuits, and tandem to + PSAP voice circuits (EMNT) +MMOC Node site (Modems, cables, etc) + +Note: All above work groups are required to resolve troubles +by interfacing with appropriate work groups for resolution. + +The Switching Control Center (SCC) is responsible for +E911/1AESS translations in tandem central offices. +These translations route E911 calls, selective transfer, +default routing, speed calling, etc., for each PSAP. +The SCC is also responsible for troubleshooting on +the voice network (call originating to end office tandem equipment). + +For example, ANI failures in the originating offices would +be a responsibility of the SCC. + +Recent Change Memory Administration Center (RCMAC) performs +the daily tandem translation updates (recent change) +for routing of individual telephone numbers. + +Recent changes are generated from service order activity +(new service, address changes, etc.) and compiled into +a daily file by the E911 Center (ALI/DMS E911 Computer). + +SSIM/I&M is responsible for the installation and repair of +PSAP equipment. PSAP equipment includes ANI Controller, +ALI Controller, data sets, cables, sets, and other peripheral +equipment that is not vendor owned. SSIM/I&M is responsible +for establishing maintenance test kits, complete with spare parts +for PSAP maintenance. This includes test gear, data sets, +and ANI/ALI Controller parts. + +Special Services Center (SSC) or Major Account Center +(MAC) serves as the trouble reporting contact for all +(PSAP) troubles reported by customer. The SSC/MAC +refers troubles to proper organizations for handling and +tracks status of troubles, escalating when necessary. +The SSC/MAC will close out troubles with customer. +The SSC/MAC will analyze all troubles and tracks "chronic" +PSAP troubles. + +Corporate Communications Network Center (CCNC) will +test and refer troubles on all node to host circuits. +All E911 circuits are classified as official company property. + +The Minicomputer Maintenance Operations Center +(MMOC) maintains the E911 (ALI/DMS) computer +hardware at the Host site. This MMOC is also responsible +for monitoring the system and reporting certain PSAP +and system problems to the local MMOC's, SCC's or +SSC/MAC's. The MMOC personnel also operate software +programs that maintain the TN data base under the +direction of the E911 Center. The maintenance of the +NODE computer (the interface between the PSAP and the +ALI/DMS computer) is a function of the MMOC at the +NODE site. The MMOC's at the NODE sites may also be +involved in the testing of NODE to Host circuits. +The MMOC will also assist on Host to PSAP and data network +related troubles not resolved through standard trouble +clearing procedures. + +Installation And Maintenance Center (IMC) is responsible +for referral of E911 subscriber troubles that are not subscriber +line problems. + +E911 Center - Performs the role of System Administration +and is responsible for overall operation of the E911 +computer software. The E911 Center does A-Z trouble +analysis and provides statistical information on the +performance of the system. + +This analysis includes processing PSAP inquiries (trouble +reports) and referral of network troubles. The E911 Center +also performs daily processing of tandem recent change +and provides information to the RCMAC for tandem input. +The E911 Center is responsible for daily processing +of the ALI/DMS computer data base and provides error files, +etc. to the Customer Services department for investigation and correction. +The E911 Center participates in all system implementations and on-going +maintenance effort and assists in the development of procedures, +training and education of information to all groups. + +Any group receiving a 911 trouble from the SSC/MAC should +close out the trouble with the SSC/MAC or provide a status +if the trouble has been referred to another group. +This will allow the SSC/MAC to provide a status back +to the customer or escalate as appropriate. + +Any group receiving a trouble from the Host site (MMOC +or CCNC) should close the trouble back to that group. + +The MMOC should notify the appropriate SSC/MAC +when the Host, Node, or all Node circuits are down so that +the SSC/MAC can reply to customer reports that may be +called in by the PSAPs. This will eliminate duplicate +reporting of troubles. On complete outages the MMOC +will follow escalation procedures for a Node after two (2) +hours and for a PSAP after four (4) hours. Additionally the +MMOC will notify the appropriate SSC/MAC when the +Host, Node, or all Node circuits are down. + +The PSAP will call the SSC/MAC to report E911 troubles. +The person reporting the E911 trouble may not have a +circuit I.D. and will therefore report the PSAP name and +address. Many PSAP troubles are not circuit specific. In +those instances where the caller cannot provide a circuit +I.D., the SSC/MAC will be required to determine the +circuit I.D. using the PSAP profile. Under no circumstances +will the SSC/MAC Center refuse to take the trouble. +The E911 trouble should be handled as quickly as possible, +with the SSC/MAC providing as much assistance as +possible while taking the trouble report from the caller. + +The SSC/MAC will screen/test the trouble to determine the +appropriate handoff organization based on the following criteria: + +PSAP equipment problem: SSIM/I&M +Circuit problem: SSC/MAC +Voice network problem: SCC (report trunk group number) +Problem affecting multiple PSAPs (No ALI report from +all PSAPs): Contact the MMOC to check for NODE or +Host computer problems before further testing. + +The SSC/MAC will track the status of reported troubles +and escalate as appropriate. The SSC/MAC will close out +customer/company reports with the initiating contact. +Groups with specific maintenance responsibilities, +defined above, will investigate "chronic" troubles upon +request from the SSC/MAC and the ongoing maintenance subcommittee. + +All "out of service" E911 troubles are priority one type reports. +One link down to a PSAP is considered a priority one trouble +and should be handled as if the PSAP was isolated. + +The PSAP will report troubles with the ANI controller, ALI +controller or set equipment to the SSC/MAC. + +NO ANI: Where the PSAP reports NO ANI (digital +display screen is blank) ask if this condition exists on all +screens and on all calls. It is important to differentiate +between blank screens and screens displaying 911-00XX, +or all zeroes. + +When the PSAP reports all screens on all calls, ask if there +is any voice contact with callers. If there is no voice +contact the trouble should be referred to the SCC +immediately since 911 calls are not getting through which +may require alternate routing of calls to another PSAP. + +When the PSAP reports this condition on all screens +but not all calls and has voice contact with callers, +the report should be referred to SSIM/I&M for dispatch. +The SSC/MAC should verify with the SCC that ANI +is pulsing before dispatching SSIM. + +When the PSAP reports this condition on one screen for +all calls (others work fine) the trouble should be referred +to SSIM/I&M for dispatch, because the trouble is isolated to +one piece of equipment at the customer premise. + +An ANI failure (i.e. all zeroes) indicates that the ANI has +not been received by the PSAP from the tandem office or +was lost by the PSAP ANI controller. The PSAP may +receive "02" alarms which can be caused by the ANI +controller logging more than three all zero failures on the +same trunk. The PSAP has been instructed to report this +condition to the SSC/MAC since it could indicate an +equipment trouble at the PSAP which might be affecting +all subscribers calling into the PSAP. When all zeroes are +being received on all calls or "02" alarms continue, a tester +should analyze the condition to determine the appropriate +action to be taken. The tester must perform cooperative +testing with the SCC when there appears to be a problem +on the Tandem-PSAP trunks before requesting dispatch. + +When an occasional all zero condition is reported, +the SSC/MAC should dispatch SSIM/I&M to routine +equipment on a "chronic" troublesweep. + +The PSAPs are instructed to report incidental ANI failures +to the BOC on a PSAP inquiry trouble ticket (paper) that +is sent to the Customer Services E911 group and forwarded +to E911 center when required. This usually involves only a +particular telephone number and is not a condition that +would require a report to the SSC/MAC. Multiple ANI +failures which our from the same end office (XX denotes +end office), indicate a hard trouble condition may exist +in the end office or end office tandem trunks. The PSAP will +report this type of condition to the SSC/MAC and the +SSC/MAC should refer the report to the SCC responsible +for the tandem office. NOTE: XX is the ESCO (Emergency +Service Number) associated with the incoming 911 trunks +into the tandem. It is important that the C/MAC tell the +SCC what is displayed at the PSAP (i.e. 911-0011) which +indicates to the SCC which end office is in trouble. + +Note: It is essential that the PSAP fill out inquiry form +on every ANI failure. + +The PSAP will report a trouble any time an address is not +received on an address display (screen blank) E911 call. +(If a record is not in the 911 data base or an ANI failure +is encountered, the screen will provide a display noticing +such condition). The SSC/MAC should verify with the PSAP +whether the NO ALI condition is on one screen or all screens. + +When the condition is on one screen (other screens +receive ALI information) the SSC/MAC will request +SSIM/I&M to dispatch. + +If no screens are receiving ALI information, there is usually +a circuit trouble between the PSAP and the Host computer. +The SSC/MAC should test the trouble and refer for restoral. + +Note: If the SSC/MAC receives calls from multiple +PSAP's, all of which are receiving NO ALI, there is a +problem with the Node or Node to Host circuits or the +Host computer itself. Before referring the trouble the +SSC/MAC should call the MMOC to inquire if the Node +or Host is in trouble. + +Alarm conditions on the ANI controller digital display at +the PSAP are to be reported by the PSAP's. These alarms +can indicate various trouble conditions so the SSC/MAC +should ask the PSAP if any portion of the E911 system +is not functioning properly. + +The SSC/MAC should verify with the PSAP attendant that +the equipment's primary function is answering E911 calls. +If it is, the SSC/MAC should request a dispatch SSIM/I&M. +If the equipment is not primarily used for E911, +then the SSC/MAC should advise PSAP to contact their CPE vendor. + +Note: These troubles can be quite confusing when the +PSAP has vendor equipment mixed in with equipment +that the BOC maintains. The Marketing representative +should provide the SSC/MAC information concerning any +unusual or exception items where the PSAP should +contact their vendor. This information should be included +in the PSAP profile sheets. + +ANI or ALI controller down: When the host computer sees +the PSAP equipment down and it does not come back up, +the MMOC will report the trouble to the SSC/MAC; +the equipment is down at the PSAP, a dispatch will be required. + +PSAP link (circuit) down: The MMOC will provide the +SSC/MAC with the circuit ID that the Host computer +indicates in trouble. Although each PSAP has two circuits, +when either circuit is down the condition must be treated +as an emergency since failure of the second circuit will +cause the PSAP to be isolated. + +Any problems that the MMOC identifies from the Node +location to the Host computer will be handled directly +with the appropriate MMOC(s)/CCNC. + +Note: The customer will call only when a problem is +apparent to the PSAP. When only one circuit is down to +the PSAP, the customer may not be aware there is a +trouble, even though there is one link down, +notification should appear on the PSAP screen. +Troubles called into the SSC/MAC from the MMOC +or other company employee should not be closed out +by calling the PSAP since it may result in the +customer responding that they do not have a trouble. +These reports can only be closed out by receiving +information that the trouble was fixed and by checking +with the company employee that reported the trouble. +The MMOC personnel will be able to verify that the +trouble has cleared by reviewing a printout from the host. + +When the CRSAB receives a subscriber complaint +(i.e., cannot dial 911) the RSA should obtain as much +information as possible while the customer is on the line. + +For example, what happened when the subscriber dialed 911? +The report is automatically directed to the IMC for subscriber line testing. +When no line trouble is found, the IMC will refer the trouble condition +to the SSC/MAC. The SSC/MAC will contact Customer Services E911 Group +and verify that the subscriber should be able to call 911 and obtain the ESN. +The SSC/MAC will verify the ESN via 2SCCS. When both verifications match, +the SSC/MAC will refer the report to the SCC responsible for the 911 tandem +office for investigation and resolution. The MAC is responsible for tracking +the trouble and informing the IMC when it is resolved. + + +For more information, please refer to E911 Glossary of Terms. +End of Phrack File +_____________________________________ + + +The reader is forgiven if he or she was entirely unable to read +this document. John Perry Barlow had a great deal of fun at its expense, +in "Crime and Puzzlement:" "Bureaucrat-ese of surpassing opacity. . . . +To read the whole thing straight through without entering coma requires +either a machine or a human who has too much practice thinking like one. +Anyone who can understand it fully and fluidly had altered his consciousness +beyond the ability to ever again read Blake, Whitman, or Tolstoy. . . . +the document contains little of interest to anyone who is not a student +of advanced organizational sclerosis." + +With the Document itself to hand, however, exactly as it was published +(in its six-page edited form) in Phrack, the reader may be able to verify +a few statements of fact about its nature. First, there is no software, +no computer code, in the Document. It is not computer-programming language +like FORTRAN or C++, it is English; all the sentences have nouns and verbs +and punctuation. It does not explain how to break into the E911 system. +It does not suggest ways to destroy or damage the E911 system. + +There are no access codes in the Document. There are no computer passwords. +It does not explain how to steal long distance service. It does not explain +how to break in to telco switching stations. There is nothing in it about +using a personal computer or a modem for any purpose at all, good or bad. + +Close study will reveal that this document is not about machinery. +The E911 Document is about ADMINISTRATION. It describes how one creates +and administers certain units of telco bureaucracy: +Special Service Centers and Major Account Centers (SSC/MAC). +It describes how these centers should distribute responsibility +for the E911 service, to other units of telco bureaucracy, +in a chain of command, a formal hierarchy. It describes +who answers customer complaints, who screens calls, +who reports equipment failures, who answers those reports, +who handles maintenance, who chairs subcommittees, +who gives orders, who follows orders, WHO tells WHOM what to do. +The Document is not a "roadmap" to computers. +The Document is a roadmap to PEOPLE. + +As an aid to breaking into computer systems, the Document is USELESS. +As an aid to harassing and deceiving telco people, however, the Document +might prove handy (especially with its Glossary, which I have not included). +An intense and protracted study of this Document and its Glossary, +combined with many other such documents, might teach one to speak like +a telco employee. And telco people live by SPEECH--they live by phone +communication. If you can mimic their language over the phone, +you can "social-engineer" them. If you can con telco people, you can +wreak havoc among them. You can force them to no longer trust one another; +you can break the telephonic ties that bind their community; you can make +them paranoid. And people will fight harder to defend their community +than they will fight to defend their individual selves. + +This was the genuine, gut-level threat posed by Phrack magazine. +The real struggle was over the control of telco language, +the control of telco knowledge. It was a struggle to defend the social +"membrane of differentiation" that forms the walls of the telco +community's ivory tower --the special jargon that allows telco +professionals to recognize one another, and to exclude charlatans, +thieves, and upstarts. And the prosecution brought out this fact. +They repeatedly made reference to the threat posed to telco professionals +by hackers using "social engineering." + +However, Craig Neidorf was not on trial for learning to speak like +a professional telecommunications expert. Craig Neidorf was on trial +for access device fraud and transportation of stolen property. +He was on trial for stealing a document that was purportedly +highly sensitive and purportedly worth tens of thousands of dollars. + +# + +John Nagle read the E911 Document. He drew his own conclusions. +And he presented Zenner and his defense team with an overflowing box +of similar material, drawn mostly from Stanford University's +engineering libraries. During the trial, the defense team--Zenner, +half-a-dozen other attorneys, Nagle, Neidorf, and computer-security +expert Dorothy Denning, all pored over the E911 Document line-by-line. + +On the afternoon of July 25, 1990, Zenner began to cross-examine +a woman named Billie Williams, a service manager for Southern Bell +in Atlanta. Ms. Williams had been responsible for the E911 Document. +(She was not its author--its original "author" was a Southern Bell +staff manager named Richard Helms. However, Mr. Helms should not bear +the entire blame; many telco staff people and maintenance personnel +had amended the Document. It had not been so much "written" by a +single author, as built by committee out of concrete-blocks of jargon.) + +Ms. Williams had been called as a witness for the prosecution, +and had gamely tried to explain the basic technical structure +of the E911 system, aided by charts. + +Now it was Zenner's turn. He first established that the +"proprietary stamp" that BellSouth had used on the E911 Document +was stamped on EVERY SINGLE DOCUMENT that BellSouth wrote-- +THOUSANDS of documents. "We do not publish anything other +than for our own company," Ms. Williams explained. +"Any company document of this nature is considered proprietary." +Nobody was in charge of singling out special high-security publications +for special high-security protection. They were ALL special, +no matter how trivial, no matter what their subject matter-- +the stamp was put on as soon as any document was written, +and the stamp was never removed. + +Zenner now asked whether the charts she had been using to explain +the mechanics of E911 system were "proprietary," too. +Were they PUBLIC INFORMATION, these charts, all about PSAPs, +ALIs, nodes, local end switches? Could he take the charts out +in the street and show them to anybody, "without violating +some proprietary notion that BellSouth has?" + +Ms Williams showed some confusion, but finally areed that the charts were, +in fact, public. + +"But isn't this what you said was basically what appeared in Phrack?" + +Ms. Williams denied this. + +Zenner now pointed out that the E911 Document as published in Phrack +was only half the size of the original E911 Document (as Prophet +had purloined it). Half of it had been deleted--edited by Neidorf. + +Ms. Williams countered that "Most of the information that is +in the text file is redundant." + +Zenner continued to probe. Exactly what bits of knowledge in the Document +were, in fact, unknown to the public? Locations of E911 computers? +Phone numbers for telco personnel? Ongoing maintenance subcommittees? +Hadn't Neidorf removed much of this? + +Then he pounced. "Are you familiar with Bellcore Technical Reference +Document TR-TSY-000350?" It was, Zenner explained, officially titled +"E911 Public Safety Answering Point Interface Between 1-1AESS Switch +and Customer Premises Equipment." It contained highly detailed +and specific technical information about the E911 System. +It was published by Bellcore and publicly available for about $20. + +He showed the witness a Bellcore catalog which listed thousands +of documents from Bellcore and from all the Baby Bells, BellSouth included. +The catalog, Zenner pointed out, was free. Anyone with a credit card +could call the Bellcore toll-free 800 number and simply order any +of these documents, which would be shipped to any customer without question. +Including, for instance, "BellSouth E911 Service Interfaces to +Customer Premises Equipment at a Public Safety Answering Point." + +Zenner gave the witness a copy of "BellSouth E911 Service Interfaces," +which cost, as he pointed out, $13, straight from the catalog. +"Look at it carefully," he urged Ms. Williams, "and tell me +if it doesn't contain about twice as much detailed information +about the E911 system of BellSouth than appeared anywhere in Phrack." + +"You want me to. . . ." Ms. Williams trailed off. "I don't understand." + +"Take a careful look," Zenner persisted. "Take a look at that document, +and tell me when you're done looking at it if, indeed, it doesn't contain +much more detailed information about the E911 system than appeared in Phrack." + +"Phrack wasn't taken from this," Ms. Williams said. + +"Excuse me?" said Zenner. + +"Phrack wasn't taken from this." + +"I can't hear you," Zenner said. + +"Phrack was not taken from this document. I don't understand +your question to me." + +"I guess you don't," Zenner said. + +At this point, the prosecution's case had been gutshot. +Ms. Williams was distressed. Her confusion was quite genuine. +Phrack had not been taken from any publicly available Bellcore document. +Phrack's E911 Document had been stolen from her own company's computers, +from her own company's text files, that her own colleagues had written, +and revised, with much labor. + +But the "value" of the Document had been blown to smithereens. +It wasn't worth eighty grand. According to Bellcore it was worth +thirteen bucks. And the looming menace that it supposedly posed +had been reduced in instants to a scarecrow. Bellcore itself +was selling material far more detailed and "dangerous," +to anybody with a credit card and a phone. + +Actually, Bellcore was not giving this information to just anybody. +They gave it to ANYBODY WHO ASKED, but not many did ask. +Not many people knew that Bellcore had a free catalog and an 800 number. +John Nagle knew, but certainly the average teenage phreak didn't know. +"Tuc," a friend of Neidorf's and sometime Phrack contributor, knew, +and Tuc had been very helpful to the defense, behind the scenes. +But the Legion of Doom didn't know--otherwise, they would never +have wasted so much time raiding dumpsters. Cook didn't know. +Foley didn't know. Kluepfel didn't know. The right hand +of Bellcore knew not what the left hand was doing. The right +hand was battering hackers without mercy, while the left hand +was distributing Bellcore's intellectual property to anybody +who was interested in telephone technical trivia--apparently, +a pathetic few. + +The digital underground was so amateurish and poorly organized +that they had never discovered this heap of unguarded riches. +The ivory tower of the telcos was so wrapped-up in the fog +of its own technical obscurity that it had left all the +windows open and flung open the doors. No one had even noticed. + +Zenner sank another nail in the coffin. He produced a printed issue +of Telephone Engineer & Management, a prominent industry journal +that comes out twice a month and costs $27 a year. This particular issue +of TE&M, called "Update on 911," featured a galaxy of technical details +on 911 service and a glossary far more extensive than Phrack's. + +The trial rumbled on, somehow, through its own momentum. +Tim Foley testified about his interrogations of Neidorf. +Neidorf's written admission that he had known the E911 Document +was pilfered was officially read into the court record. + +An interesting side issue came up: "Terminus" had once passed Neidorf +a piece of UNIX AT&T software, a log-in sequence, that had been cunningly +altered so that it could trap passwords. The UNIX software itself was +illegally copied AT&T property, and the alterations "Terminus" had made to it, +had transformed it into a device for facilitating computer break-ins. Terminus +himself would eventually plead guilty to theft of this piece of software, +and the Chicago group would send Terminus to prison for it. But it was +of dubious relevance in the Neidorf case. Neidorf hadn't written the program. +He wasn't accused of ever having used it. And Neidorf wasn't being charged +with software theft or owning a password trapper. + +On the next day, Zenner took the offensive. The civil libertarians +now had their own arcane, untried legal weaponry to launch into action-- +the Electronic Communications Privacy Act of 1986, 18 US Code, +Section 2701 et seq. Section 2701 makes it a crime to intentionally +access without authorization a facility in which an electronic communication +service is provided--it is, at heart, an anti-bugging and anti-tapping law, +intended to carry the traditional protections of telephones into other +electronic channels of communication. While providing penalties for amateur +snoops, however, Section 2703 of the ECPA also lays some formal difficulties +on the bugging and tapping activities of police. + +The Secret Service, in the person of Tim Foley, had served Richard Andrews +with a federal grand jury subpoena, in their pursuit of Prophet, +the E911 Document, and the Terminus software ring. But according to +the Electronic Communications Privacy Act, a "provider of remote +computing service" was legally entitled to "prior notice" from +the government if a subpoena was used. Richard Andrews and his +basement UNIX node, Jolnet, had not received any "prior notice." +Tim Foley had purportedly violated the ECPA and committed +an electronic crime! Zenner now sought the judge's permission +to cross-examine Foley on the topic of Foley's own electronic misdeeds. + +Cook argued that Richard Andrews' Jolnet was a privately owned +bulletin board, and not within the purview of ECPA. Judge Bua +granted the motion of the government to prevent cross-examination +on that point, and Zenner's offensive fizzled. This, however, +was the first direct assault on the legality of the actions +of the Computer Fraud and Abuse Task Force itself-- +the first suggestion that they themselves had broken the law, +and might, perhaps, be called to account. + +Zenner, in any case, did not really need the ECPA. +Instead, he grilled Foley on the glaring contradictions in +the supposed value of the E911 Document. He also brought up +the embarrassing fact that the supposedly red-hot E911 Document +had been sitting around for months, in Jolnet, with Kluepfel's knowledge, +while Kluepfel had done nothing about it. + +In the afternoon, the Prophet was brought in to testify +for the prosecution. (The Prophet, it will be recalled, +had also been indicted in the case as partner in a fraud +scheme with Neidorf.) In Atlanta, the Prophet had already +pled guilty to one charge of conspiracy, one charge of wire fraud +and one charge of interstate transportation of stolen property. +The wire fraud charge, and the stolen property charge, +were both directly based on the E911 Document. + +The twenty-year-old Prophet proved a sorry customer, +answering questions politely but in a barely audible mumble, +his voice trailing off at the ends of sentences. +He was constantly urged to speak up. + +Cook, examining Prophet, forced him to admit that +he had once had a "drug problem," abusing amphetamines, +marijuana, cocaine, and LSD. This may have established +to the jury that "hackers" are, or can be, seedy lowlife characters, +but it may have damaged Prophet's credibility somewhat. +Zenner later suggested that drugs might have damaged Prophet's memory. +The interesting fact also surfaced that Prophet had never +physically met Craig Neidorf. He didn't even know +Neidorf's last name--at least, not until the trial. + +Prophet confirmed the basic facts of his hacker career. +He was a member of the Legion of Doom. He had abused codes, +he had broken into switching stations and re-routed calls, +he had hung out on pirate bulletin boards. He had raided +the BellSouth AIMSX computer, copied the E911 Document, +stored it on Jolnet, mailed it to Neidorf. He and Neidorf +had edited it, and Neidorf had known where it came from. + +Zenner, however, had Prophet confirm that Neidorf was not a member +of the Legion of Doom, and had not urged Prophet to break into +BellSouth computers. Neidorf had never urged Prophet to defraud anyone, +or to steal anything. Prophet also admitted that he had never known Neidorf +to break in to any computer. Prophet said that no one in the Legion of Doom +considered Craig Neidorf a "hacker" at all. Neidorf was not a UNIX maven, +and simply lacked the necessary skill and ability to break into computers. +Neidorf just published a magazine. + +On Friday, July 27, 1990, the case against Neidorf collapsed. +Cook moved to dismiss the indictment, citing "information currently +available to us that was not available to us at the inception of the trial." +Judge Bua praised the prosecution for this action, which he described as +"very responsible," then dismissed a juror and declared a mistrial. + +Neidorf was a free man. His defense, however, had cost himself +and his family dearly. Months of his life had been consumed in anguish; +he had seen his closest friends shun him as a federal criminal. +He owed his lawyers over a hundred thousand dollars, despite +a generous payment to the defense by Mitch Kapor. + +Neidorf was not found innocent. The trial was simply dropped. +Nevertheless, on September 9, 1991, Judge Bua granted Neidorf's +motion for the "expungement and sealing" of his indictment record. +The United States Secret Service was ordered to delete and destroy +all fingerprints, photographs, and other records of arrest +or processing relating to Neidorf's indictment, including +their paper documents and their computer records. + +Neidorf went back to school, blazingly determined to become a lawyer. +Having seen the justice system at work, Neidorf lost much of his enthusiasm +for merely technical power. At this writing, Craig Neidorf is working +in Washington as a salaried researcher for the American Civil Liberties Union. + +# + +The outcome of the Neidorf trial changed the EFF +from voices-in-the-wilderness to the media darlings +of the new frontier. + +Legally speaking, the Neidorf case was not a sweeping triumph +for anyone concerned. No constitutional principles had been established. +The issues of "freedom of the press" for electronic publishers remained +in legal limbo. There were public misconceptions about the case. +Many people thought Neidorf had been found innocent and relieved +of all his legal debts by Kapor. The truth was that the government +had simply dropped the case, and Neidorf's family had gone deeply +into hock to support him. + +But the Neidorf case did provide a single, devastating, public sound-bite: +THE FEDS SAID IT WAS WORTH EIGHTY GRAND, AND IT WAS ONLY WORTH THIRTEEN BUCKS. + +This is the Neidorf case's single most memorable element. No serious report +of the case missed this particular element. Even cops could not read this +without a wince and a shake of the head. It left the public credibility +of the crackdown agents in tatters. + +The crackdown, in fact, continued, however. Those two charges +against Prophet, which had been based on the E911 Document, +were quietly forgotten at his sentencing--even though Prophet +had already pled guilty to them. Georgia federal prosecutors +strongly argued for jail time for the Atlanta Three, insisting on +"the need to send a message to the community," "the message that +hackers around the country need to hear." + +There was a great deal in their sentencing memorandum +about the awful things that various other hackers had done +(though the Atlanta Three themselves had not, in fact, +actually committed these crimes). There was also much +speculation about the awful things that the Atlanta Three +MIGHT have done and WERE CAPABLE of doing (even though +they had not, in fact, actually done them). +The prosecution's argument carried the day. +The Atlanta Three were sent to prison: +Urvile and Leftist both got 14 months each, +while Prophet (a second offender) got 21 months. + +The Atlanta Three were also assessed staggering fines as "restitution": +$233,000 each. BellSouth claimed that the defendants had "stolen" +"approximately $233,880 worth" of "proprietary computer access information"-- +specifically, $233,880 worth of computer passwords and connect addresses. +BellSouth's astonishing claim of the extreme value of its own computer +passwords and addresses was accepted at face value by the Georgia court. +Furthermore (as if to emphasize its theoretical nature) this enormous sum +was not divvied up among the Atlanta Three, but each of them had to pay +all of it. + +A striking aspect of the sentence was that the Atlanta Three were +specifically forbidden to use computers, except for work or under supervision. +Depriving hackers of home computers and modems makes some sense if one +considers hackers as "computer addicts," but EFF, filing an amicus brief +in the case, protested that this punishment was unconstitutional-- +it deprived the Atlanta Three of their rights of free association +and free expression through electronic media. + +Terminus, the "ultimate hacker," was finally sent to prison for a year +through the dogged efforts of the Chicago Task Force. His crime, +to which he pled guilty, was the transfer of the UNIX password trapper, +which was officially valued by AT&T at $77,000, a figure which aroused +intense skepticism among those familiar with UNIX "login.c" programs. + +The jailing of Terminus and the Atlanta Legionnaires of Doom, however, +did not cause the EFF any sense of embarrassment or defeat. +On the contrary, the civil libertarians were rapidly gathering strength. + +An early and potent supporter was Senator Patrick Leahy, +Democrat from Vermont, who had been a Senate sponsor +of the Electronic Communications Privacy Act. Even before +the Neidorf trial, Leahy had spoken out in defense of hacker-power +and freedom of the keyboard: "We cannot unduly inhibit the inquisitive +13-year-old who, if left to experiment today, may tomorrow develop +the telecommunications or computer technology to lead the United States +into the 21st century. He represents our future and our best hope +to remain a technologically competitive nation." + +It was a handsome statement, rendered perhaps rather more effective +by the fact that the crackdown raiders DID NOT HAVE any Senators +speaking out for THEM. On the contrary, their highly secretive +actions and tactics, all "sealed search warrants" here and +"confidential ongoing investigations" there, might have won +them a burst of glamorous publicity at first, but were crippling +them in the on-going propaganda war. Gail Thackeray was reduced +to unsupported bluster: "Some of these people who are loudest +on the bandwagon may just slink into the background," +she predicted in Newsweek--when all the facts came out, +and the cops were vindicated. + +But all the facts did not come out. Those facts that did, +were not very flattering. And the cops were not vindicated. +And Gail Thackeray lost her job. By the end of 1991, +William Cook had also left public employment. + +1990 had belonged to the crackdown, but by '91 its agents +were in severe disarray, and the libertarians were on a roll. +People were flocking to the cause. + +A particularly interesting ally had been Mike Godwin of Austin, Texas. +Godwin was an individual almost as difficult to describe as Barlow; +he had been editor of the student newspaper of the University of Texas, +and a computer salesman, and a programmer, and in 1990 was back +in law school, looking for a law degree. + +Godwin was also a bulletin board maven. He was very well-known +in the Austin board community under his handle "Johnny Mnemonic," +which he adopted from a cyberpunk science fiction story by William Gibson. +Godwin was an ardent cyberpunk science fiction fan. As a fellow Austinite +of similar age and similar interests, I myself had known Godwin socially +for many years. When William Gibson and myself had been writing our +collaborative SF novel, The Difference Engine, Godwin had been our +technical advisor in our effort to link our Apple word-processors +from Austin to Vancouver. Gibson and I were so pleased by his generous +expert help that we named a character in the novel "Michael Godwin" +in his honor. + +The handle "Mnemonic" suited Godwin very well. His erudition +and his mastery of trivia were impressive to the point of stupor; +his ardent curiosity seemed insatiable, and his desire to debate +and argue seemed the central drive of his life. Godwin had even +started his own Austin debating society, wryly known as the +"Dull Men's Club." In person, Godwin could be overwhelming; +a flypaper-brained polymath who could not seem to let any idea go. +On bulletin boards, however, Godwin's closely reasoned, +highly grammatical, erudite posts suited the medium well, +and he became a local board celebrity. + +Mike Godwin was the man most responsible for the public national exposure +of the Steve Jackson case. The Izenberg seizure in Austin had received +no press coverage at all. The March 1 raids on Mentor, Bloodaxe, and +Steve Jackson Games had received a brief front-page splash in the +front page of the Austin American-Statesman, but it was confused +and ill-informed: the warrants were sealed, and the Secret Service +wasn't talking. Steve Jackson seemed doomed to obscurity. +Jackson had not been arrested; he was not charged with any crime; +he was not on trial. He had lost some computers in an ongoing +investigation--so what? Jackson tried hard to attract attention +to the true extent of his plight, but he was drawing a blank; +no one in a position to help him seemed able to get a mental grip +on the issues. + +Godwin, however, was uniquely, almost magically, qualified +to carry Jackson's case to the outside world. Godwin was +a board enthusiast, a science fiction fan, a former journalist, +a computer salesman, a lawyer-to-be, and an Austinite. +Through a coincidence yet more amazing, in his last year +of law school Godwin had specialized in federal prosecutions +and criminal procedure. Acting entirely on his own, Godwin made +up a press packet which summarized the issues and provided useful +contacts for reporters. Godwin's behind-the-scenes effort +(which he carried out mostly to prove a point in a local board debate) +broke the story again in the Austin American-Statesman and then in Newsweek. + +Life was never the same for Mike Godwin after that. As he joined the growing +civil liberties debate on the Internet, it was obvious to all parties involved +that here was one guy who, in the midst of complete murk and confusion, +GENUINELY UNDERSTOOD EVERYTHING HE WAS TALKING ABOUT. The disparate elements +of Godwin's dilettantish existence suddenly fell together as neatly as +the facets of a Rubik's cube. + +When the time came to hire a full-time EFF staff attorney, +Godwin was the obvious choice. He took the Texas bar exam, +left Austin, moved to Cambridge, became a full-time, professional, +computer civil libertarian, and was soon touring the nation on behalf +of EFF, delivering well-received addresses on the issues to crowds +as disparate as academics, industrialists, science fiction fans, +and federal cops. + +Michael Godwin is currently the chief legal counsel of +the Electronic Frontier Foundation in Cambridge, Massachusetts. + +# + +Another early and influential participant in the controversy +was Dorothy Denning. Dr. Denning was unique among investigators +of the computer underground in that she did not enter the debate +with any set of politicized motives. She was a professional +cryptographer and computer security expert whose primary interest +in hackers was SCHOLARLY. She had a B.A. and M.A. in mathematics, +and a Ph.D. in computer science from Purdue. She had worked for SRI +International, the California think-tank that was also the home of +computer-security maven Donn Parker, and had authored an influential text +called Cryptography and Data Security. In 1990, Dr. Denning was working for +Digital Equipment Corporation in their Systems Reseach Center. Her husband, +Peter Denning, was also a computer security expert, working for NASA's +Research Institute for Advanced Computer Science. He had edited the +well-received Computers Under Attack: Intruders, Worms and Viruses. + +Dr. Denning took it upon herself to contact the digital underground, +more or less with an anthropological interest. There she discovered +that these computer-intruding hackers, who had been characterized +as unethical, irresponsible, and a serious danger to society, +did in fact have their own subculture and their own rules. +They were not particularly well-considered rules, but they were, +in fact, rules. Basically, they didn't take money and they +didn't break anything. + +Her dispassionate reports on her researches did a great deal +to influence serious-minded computer professionals--the sort +of people who merely rolled their eyes at the cyberspace +rhapsodies of a John Perry Barlow. + +For young hackers of the digital underground, meeting Dorothy Denning +was a genuinely mind-boggling experience. Here was this neatly coiffed, +conservatively dressed, dainty little personage, who reminded most +hackers of their moms or their aunts. And yet she was an IBM systems +programmer with profound expertise in computer architectures +and high-security information flow, who had personal friends +in the FBI and the National Security Agency. + +Dorothy Denning was a shining example of the American mathematical +intelligentsia, a genuinely brilliant person from the central ranks +of the computer-science elite. And here she was, gently questioning +twenty-year-old hairy-eyed phone-phreaks over the deeper ethical +implications of their behavior. + +Confronted by this genuinely nice lady, most hackers sat up very straight +and did their best to keep the anarchy-file stuff down to a faint whiff +of brimstone. Nevertheless, the hackers WERE in fact prepared to seriously +discuss serious issues with Dorothy Denning. They were willing to speak +the unspeakable and defend the indefensible, to blurt out their convictions +that information cannot be owned, that the databases of governments and large +corporations were a threat to the rights and privacy of individuals. + +Denning's articles made it clear to many that "hacking" +was not simple vandalism by some evil clique of psychotics. +"Hacking" was not an aberrant menace that could be charmed away +by ignoring it, or swept out of existence by jailing a few ringleaders. +Instead, "hacking" was symptomatic of a growing, primal struggle over +knowledge and power in the age of information. + +Denning pointed out that the attitude of hackers were at least partially +shared by forward-looking management theorists in the business community: +people like Peter Drucker and Tom Peters. Peter Drucker, in his book +The New Realities, had stated that "control of information by the government +is no longer possible. Indeed, information is now transnational. +Like money, it has no `fatherland.'" + +And management maven Tom Peters had chided large corporations for uptight, +proprietary attitudes in his bestseller, Thriving on Chaos: +"Information hoarding, especially by politically motivated, +power-seeking staffs, had been commonplace throughout American industry, +service and manufacturing alike. It will be an impossible +millstone aroung the neck of tomorrow's organizations." + +Dorothy Denning had shattered the social membrane of the +digital underground. She attended the Neidorf trial, +where she was prepared to testify for the defense as an expert witness. +She was a behind-the-scenes organizer of two of the most important +national meetings of the computer civil libertarians. Though not +a zealot of any description, she brought disparate elements of the +electronic community into a surprising and fruitful collusion. + +Dorothy Denning is currently the Chair of the Computer Science Department +at Georgetown University in Washington, DC. + +# + +There were many stellar figures in the civil libertarian community. +There's no question, however, that its single most influential figure +was Mitchell D. Kapor. Other people might have formal titles, +or governmental positions, have more experience with crime, +or with the law, or with the arcanities of computer security +or constitutional theory. But by 1991 Kapor had transcended +any such narrow role. Kapor had become "Mitch." + +Mitch had become the central civil-libertarian ad-hocrat. +Mitch had stood up first, he had spoken out loudly, directly, +vigorously and angrily, he had put his own reputation, +and his very considerable personal fortune, on the line. +By mid-'91 Kapor was the best-known advocate of his cause +and was known PERSONALLY by almost every single human being in America +with any direct influence on the question of civil liberties in cyberspace. +Mitch had built bridges, crossed voids, changed paradigms, forged metaphors, +made phone-calls and swapped business cards to such spectacular effect +that it had become impossible for anyone to take any action in the +"hacker question" without wondering what Mitch might think-- +and say--and tell his friends. + +The EFF had simply NETWORKED the situation into an entirely new status quo. +And in fact this had been EFF's deliberate strategy from the beginning. +Both Barlow and Kapor loathed bureaucracies and had deliberately +chosen to work almost entirely through the electronic spiderweb of +"valuable personal contacts." + +After a year of EFF, both Barlow and Kapor had every reason +to look back with satisfaction. EFF had established its own Internet node, +"eff.org," with a well-stocked electronic archive of documents on +electronic civil rights, privacy issues, and academic freedom. +EFF was also publishing EFFector, a quarterly printed journal, +as well as EFFector Online, an electronic newsletter with +over 1,200 subscribers. And EFF was thriving on the Well. + +EFF had a national headquarters in Cambridge and a full-time staff. +It had become a membership organization and was attracting +grass-roots support. It had also attracted the support +of some thirty civil-rights lawyers, ready and eager +to do pro bono work in defense of the Constitution in Cyberspace. + +EFF had lobbied successfully in Washington and in Massachusetts +to change state and federal legislation on computer networking. +Kapor in particular had become a veteran expert witness, +and had joined the Computer Science and Telecommunications Board +of the National Academy of Science and Engineering. + +EFF had sponsored meetings such as "Computers, Freedom and Privacy" +and the CPSR Roundtable. It had carried out a press offensive that, +in the words of EFFector, "has affected the climate of opinion about +computer networking and begun to reverse the slide into +`hacker hysteria' that was beginning to grip the nation." + +It had helped Craig Neidorf avoid prison. + +And, last but certainly not least, the Electronic Frontier Foundation +had filed a federal lawsuit in the name of Steve Jackson, +Steve Jackson Games Inc., and three users of the Illuminati +bulletin board system. The defendants were, and are, +the United States Secret Service, William Cook, Tim Foley, +Barbara Golden and Henry Kleupfel. + +The case, which is in pre-trial procedures in an Austin federal court +as of this writing, is a civil action for damages to redress +alleged violations of the First and Fourth Amendments to the +United States Constitution, as well as the Privacy Protection Act +of 1980 (42 USC 2000aa et seq.), and the Electronic Communications +Privacy Act (18 USC 2510 et seq and 2701 et seq). + +EFF had established that it had credibility. It had also established +that it had teeth. + +In the fall of 1991 I travelled to Massachusetts to speak personally +with Mitch Kapor. It was my final interview for this book. + +# + +The city of Boston has always been one of the major intellectual centers +of the American republic. It is a very old city by American standards, +a place of skyscrapers overshadowing seventeenth-century graveyards, +where the high-tech start-up companies of Route 128 co-exist with the +hand-wrought pre-industrial grace of "Old Ironsides," the USS CONSTITUTION. + +The Battle of Bunker Hill, one of the first and bitterest armed clashes +of the American Revolution, was fought in Boston's environs. Today there is +a monumental spire on Bunker Hill, visible throughout much of the city. +The willingness of the republican revolutionaries to take up arms and fire +on their oppressors has left a cultural legacy that two full centuries +have not effaced. Bunker Hill is still a potent center of American political +symbolism, and the Spirit of '76 is still a potent image for those who seek +to mold public opinion. + +Of course, not everyone who wraps himself in the flag is necessarily +a patriot. When I visited the spire in September 1991, it bore a huge, +badly-erased, spray-can grafitto around its bottom reading +"BRITS OUT--IRA PROVOS." Inside this hallowed edifice was +a glass-cased diorama of thousands of tiny toy soldiers, +rebels and redcoats, fighting and dying over the green hill, +the riverside marshes, the rebel trenchworks. Plaques indicated the +movement of troops, the shiftings of strategy. The Bunker Hill Monument +is occupied at its very center by the toy soldiers of a military +war-game simulation. + +The Boston metroplex is a place of great universities, +prominent among the Massachusetts Institute of Technology, +where the term "computer hacker" was first coined. The Hacker Crackdown +of 1990 might be interpreted as a political struggle among American cities: +traditional strongholds of longhair intellectual liberalism, +such as Boston, San Francisco, and Austin, versus the bare-knuckle +industrial pragmatism of Chicago and Phoenix (with Atlanta and New York +wrapped in internal struggle). + +The headquarters of the Electronic Frontier Foundation is on +155 Second Street in Cambridge, a Bostonian suburb north +of the River Charles. Second Street has weedy sidewalks of dented, +sagging brick and elderly cracked asphalt; large street-signs warn +"NO PARKING DURING DECLARED SNOW EMERGENCY." This is an old area +of modest manufacturing industries; the EFF is catecorner from the +Greene Rubber Company. EFF's building is two stories of red brick; +its large wooden windows feature gracefully arched tops and stone sills. + +The glass window beside the Second Street entrance bears three sheets +of neatly laser-printed paper, taped against the glass. They read: +ON Technology. EFF. KEI. + +"ON Technology" is Kapor's software company, which currently specializes +in "groupware" for the Apple Macintosh computer. "Groupware" is intended +to promote efficient social interaction among office-workers linked +by computers. ON Technology's most successful software products to date +are "Meeting Maker" and "Instant Update." + +"KEI" is Kapor Enterprises Inc., Kapor's personal holding company, +the commercial entity that formally controls his extensive investments +in other hardware and software corporations. + +"EFF" is a political action group--of a special sort. + +Inside, someone's bike has been chained to the handrails +of a modest flight of stairs. A wall of modish glass brick +separates this anteroom from the offices. Beyond the brick, +there's an alarm system mounted on the wall, a sleek, complex little +number that resembles a cross between a thermostat and a CD player. +Piled against the wall are box after box of a recent special issue +of Scientific American, "How to Work, Play, and Thrive in Cyberspace," +with extensive coverage of electronic networking techniques +and political issues, including an article by Kapor himself. +These boxes are addressed to Gerard Van der Leun, EFF's +Director of Communications, who will shortly mail those magazines +to every member of the EFF. + +The joint headquarters of EFF, KEI, and ON Technology, +which Kapor currently rents, is a modestly bustling place. +It's very much the same physical size as Steve Jackson's gaming company. +It's certainly a far cry from the gigantic gray steel-sided railway +shipping barn, on the Monsignor O'Brien Highway, that is owned +by Lotus Development Corporation. + +Lotus is, of course, the software giant that Mitchell Kapor founded +in the late 70s. The software program Kapor co-authored, +"Lotus 1-2-3," is still that company's most profitable product. +"Lotus 1-2-3" also bears a singular distinction in the +digital underground: it's probably the most pirated piece +of application software in world history. + +Kapor greets me cordially in his own office, down a hall. +Kapor, whose name is pronounced KAY-por, is in his early forties, +married and the father of two. He has a round face, high forehead, +straight nose, a slightly tousled mop of black hair peppered with gray. +His large brown eyes are wideset, reflective, one might almost say soulful. +He disdains ties, and commonly wears Hawaiian shirts and tropical prints, +not so much garish as simply cheerful and just that little bit anomalous. + +There is just the whiff of hacker brimstone about Mitch Kapor. +He may not have the hard-riding, hell-for-leather, guitar-strumming +charisma of his Wyoming colleague John Perry Barlow, but there's +something about the guy that still stops one short. He has the air +of the Eastern city dude in the bowler hat, the dreamy, +Longfellow-quoting poker shark who only HAPPENS to know +the exact mathematical odds against drawing to an inside straight. +Even among his computer-community colleagues, who are hardly known +for mental sluggishness, Kapor strikes one forcefully as a very +intelligent man. He speaks rapidly, with vigorous gestures, +his Boston accent sometimes slipping to the sharp nasal tang +of his youth in Long Island. + +Kapor, whose Kapor Family Foundation does much of his philanthropic work, +is a strong supporter of Boston's Computer Museum. Kapor's interest +in the history of his industry has brought him some remarkable curios, +such as the "byte" just outside his office door. This "byte"-- +eight digital bits--has been salvaged from the wreck of an +electronic computer of the pre-transistor age. It's a standing gunmetal +rack about the size of a small toaster-oven: with eight slots +of hand-soldered breadboarding featuring thumb-sized vacuum tubes. +If it fell off a table it could easily break your foot, +but it was state-of-the-art computation in the 1940s. +(It would take exactly 157,184 of these primordial toasters +to hold the first part of this book.) + +There's also a coiling, multicolored, scaly dragon that some +inspired techno-punk artist has cobbled up entirely out of transistors, +capacitors, and brightly plastic-coated wiring. + +Inside the office, Kapor excuses himself briefly to do a little +mouse-whizzing housekeeping on his personal Macintosh IIfx. +If its giant screen were an open window, an agile person +could climb through it without much trouble at all. +There's a coffee-cup at Kapor's elbow, a memento of his +recent trip to Eastern Europe, which has a black-and-white +stencilled photo and the legend CAPITALIST FOOLS TOUR. +It's Kapor, Barlow, and two California venture-capitalist luminaries +of their acquaintance, four windblown, grinning Baby Boomer +dudes in leather jackets, boots, denim, travel bags, +standing on airport tarmac somewhere behind the formerly Iron Curtain. +They look as if they're having the absolute time of their lives. + +Kapor is in a reminiscent mood. We talk a bit about his youth-- +high school days as a "math nerd," Saturdays attending Columbia University's +high-school science honors program, where he had his first experience +programming computers. IBM 1620s, in 1965 and '66. "I was very interested," +says Kapor, "and then I went off to college and got distracted by drugs sex +and rock and roll, like anybody with half a brain would have then!" +After college he was a progressive-rock DJ in Hartford, Connecticut, +for a couple of years. + +I ask him if he ever misses his rock and roll days--if he ever wished +he could go back to radio work. + +He shakes his head flatly. "I stopped thinking about going back +to be a DJ the day after Altamont." + +Kapor moved to Boston in 1974 and got a job programming mainframes in COBOL. +He hated it. He quit and became a teacher of transcendental meditation. +(It was Kapor's long flirtation with Eastern mysticism that gave the +world "Lotus.") + +In 1976 Kapor went to Switzerland, where the Transcendental Meditation +movement had rented a gigantic Victorian hotel in St-Moritz. It was +an all-male group--a hundred and twenty of them--determined upon +Enlightenment or Bust. Kapor had given the transcendant his best shot. +He was becoming disenchanted by "the nuttiness in the organization." +"They were teaching people to levitate," he says, staring at the floor. +His voice drops an octave, becomes flat. "THEY DON'T LEVITATE." + +Kapor chose Bust. He went back to the States and acquired a degree +in counselling psychology. He worked a while in a hospital, +couldn't stand that either. "My rep was," he says "a very bright kid +with a lot of potential who hasn't found himself. Almost thirty. +Sort of lost." + +Kapor was unemployed when he bought his first personal computer--an Apple II. +He sold his stereo to raise cash and drove to New Hampshire to avoid the +sales tax. + +"The day after I purchased it," Kapor tells me, "I was hanging out +in a computer store and I saw another guy, a man in his forties, +well-dressed guy, and eavesdropped on his conversation with the salesman. +He didn't know anything about computers. I'd had a year programming. +And I could program in BASIC. I'd taught myself. So I went up to him, +and I actually sold myself to him as a consultant." He pauses. +"I don't know where I got the nerve to do this. It was uncharacteristic. +I just said, `I think I can help you, I've been listening, +this is what you need to do and I think I can do it for you.' +And he took me on! He was my first client! I became a computer +consultant the first day after I bought the Apple II." + +Kapor had found his true vocation. He attracted more clients +for his consultant service, and started an Apple users' group. + +A friend of Kapor's, Eric Rosenfeld, a graduate student at MIT, +had a problem. He was doing a thesis on an arcane form of +financial statistics, but could not wedge himself into the crowded queue +for time on MIT's mainframes. (One might note at this point that if +Mr. Rosenfeld had dishonestly broken into the MIT mainframes, +Kapor himself might have never invented Lotus 1-2-3 and +the PC business might have been set back for years!) +Eric Rosenfeld did have an Apple II, however, +and he thought it might be possible to scale the problem down. +Kapor, as favor, wrote a program for him in BASIC that did the job. + +It then occurred to the two of them, out of the blue, +that it might be possible to SELL this program. +They marketed it themselves, in plastic baggies, +for about a hundred bucks a pop, mail order. +"This was a total cottage industry by a marginal consultant," +Kapor says proudly. "That's how I got started, honest to God." + +Rosenfeld, who later became a very prominent figure on Wall Street, +urged Kapor to go to MIT's business school for an MBA. +Kapor did seven months there, but never got his MBA. +He picked up some useful tools--mainly a firm grasp +of the principles of accounting--and, in his own words, +"learned to talk MBA." Then he dropped out and went to Silicon Valley. + +The inventors of VisiCalc, the Apple computer's premier business program, +had shown an interest in Mitch Kapor. Kapor worked diligently for them +for six months, got tired of California, and went back to Boston +where they had better bookstores. The VisiCalc group had made +the critical error of bringing in "professional management." +"That drove them into the ground," Kapor says. + +"Yeah, you don't hear a lot about VisiCalc these days," I muse. + +Kapor looks surprised. "Well, Lotus. . . we BOUGHT it." + +"Oh. You BOUGHT it?" + +"Yeah." + +"Sort of like the Bell System buying Western Union?" + +Kapor grins. "Yep! Yep! Yeah, exactly!" + +Mitch Kapor was not in full command of the destiny of himself +or his industry. The hottest software commodities of the early 1980s +were COMPUTER GAMES--the Atari seemed destined to enter every teenage home +in America. Kapor got into business software simply because he didn't have +any particular feeling for computer games. But he was supremely fast +on his feet, open to new ideas and inclined to trust his instincts. +And his instincts were good. He chose good people to deal with-- +gifted programmer Jonathan Sachs (the co-author of Lotus 1-2-3). +Financial wizard Eric Rosenfeld, canny Wall Street analyst +and venture capitalist Ben Rosen. Kapor was the founder and CEO of Lotus, +one of the most spectacularly successful business ventures of the +later twentieth century. + +He is now an extremely wealthy man. I ask him if he actually +knows how much money he has. + +"Yeah," he says. "Within a percent or two." + +How much does he actually have, then? + +He shakes his head. "A lot. A lot. Not something I talk about. +Issues of money and class are things that cut pretty close to the bone." + +I don't pry. It's beside the point. One might presume, impolitely, +that Kapor has at least forty million--that's what he got the year +he left Lotus. People who ought to know claim Kapor has about +a hundred and fifty million, give or take a market swing +in his stock holdings. If Kapor had stuck with Lotus, +as his colleague friend and rival Bill Gates has stuck +with his own software start-up, Microsoft, then Kapor +would likely have much the same fortune Gates has-- +somewhere in the neighborhood of three billion, +give or take a few hundred million. Mitch Kapor +has all the money he wants. Money has lost whatever charm +it ever held for him--probably not much in the first place. +When Lotus became too uptight, too bureaucratic, too far +from the true sources of his own satisfaction, Kapor walked. +He simply severed all connections with the company and went out the door. +It stunned everyone--except those who knew him best. + +Kapor has not had to strain his resources to wreak a thorough +transformation in cyberspace politics. In its first year, +EFF's budget was about a quarter of a million dollars. +Kapor is running EFF out of his pocket change. + +Kapor takes pains to tell me that he does not consider himself +a civil libertarian per se. He has spent quite some time +with true-blue civil libertarians lately, and there's a +political-correctness to them that bugs him. They seem +to him to spend entirely too much time in legal nitpicking +and not enough vigorously exercising civil rights in the +everyday real world. + +Kapor is an entrepreneur. Like all hackers, he prefers his involvements +direct, personal, and hands-on. "The fact that EFF has a node on the +Internet is a great thing. We're a publisher. We're a distributor +of information." Among the items the eff.org Internet node carries +is back issues of Phrack. They had an internal debate about that in EFF, +and finally decided to take the plunge. They might carry other +digital underground publications--but if they do, he says, +"we'll certainly carry Donn Parker, and anything Gail Thackeray +wants to put up. We'll turn it into a public library, that has +the whole spectrum of use. Evolve in the direction of people making up +their own minds." He grins. "We'll try to label all the editorials." + +Kapor is determined to tackle the technicalities of the Internet +in the service of the public interest. "The problem with being a node +on the Net today is that you've got to have a captive technical specialist. +We have Chris Davis around, for the care and feeding of the balky beast! +We couldn't do it ourselves!" + +He pauses. "So one direction in which technology has to evolve +is much more standardized units, that a non-technical person +can feel comfortable with. It's the same shift as from minicomputers to PCs. +I can see a future in which any person can have a Node on the Net. +Any person can be a publisher. It's better than the media we now have. +It's possible. We're working actively." + +Kapor is in his element now, fluent, thoroughly in command in his material. +"You go tell a hardware Internet hacker that everyone should have a node +on the Net," he says, "and the first thing they're going to say is, +`IP doesn't scale!'" ("IP" is the interface protocol for the Internet. +As it currently exists, the IP software is simply not capable of +indefinite expansion; it will run out of usable addresses, it will saturate.) +"The answer," Kapor says, "is: evolve the protocol! Get the smart people +together and figure out what to do. Do we add ID? Do we add new protocol? +Don't just say, WE CAN'T DO IT." + +Getting smart people together to figure out what to do is a skill +at which Kapor clearly excels. I counter that people on the Internet +rather enjoy their elite technical status, and don't seem particularly +anxious to democratize the Net. + +Kapor agrees, with a show of scorn. "I tell them that this is the snobbery +of the people on the Mayflower looking down their noses at the people +who came over ON THE SECOND BOAT! Just because they got here a year, +or five years, or ten years before everybody else, that doesn't give +them ownership of cyberspace! By what right?" + +I remark that the telcos are an electronic network, too, +and they seem to guard their specialized knowledge pretty closely. + +Kapor ripostes that the telcos and the Internet are entirely +different animals. "The Internet is an open system, +everything is published, everything gets argued about, +basically by anybody who can get in. Mostly, it's exclusive +and elitist just because it's so difficult. Let's make it easier to use." + +On the other hand, he allows with a swift change of emphasis, +the so-called elitists do have a point as well. "Before people start coming in, +who are new, who want to make suggestions, and criticize the Net as +`all screwed up'. . . . They should at least take the time to understand +the culture on its own terms. It has its own history--show some respect +for it. I'm a conservative, to that extent." + +The Internet is Kapor's paradigm for the future of telecommunications. +The Internet is decentralized, non-hierarchical, almost anarchic. +There are no bosses, no chain of command, no secret data. +If each node obeys the general interface standards, +there's simply no need for any central network authority. + +Wouldn't that spell the doom of AT&T as an institution? I ask. + +That prospect doesn't faze Kapor for a moment. "Their big advantage, +that they have now, is that they have all of the wiring. +But two things are happening. Anyone with right-of-way +is putting down fiber--Southern Pacific Railroad, +people like that--there's enormous `dark fiber' laid in." +("Dark Fiber" is fiber-optic cable, whose enormous capacity +so exceeds the demands of current usage that much of the +fiber still has no light-signals on it--it's still `dark,' +awaiting future use.) + +"The other thing that's happening is the local-loop stuff +is going to go wireless. Everyone from Bellcore to the cable TV +companies to AT&T wants to put in these things called +`personal communication systems.' So you could have local competition-- +you could have multiplicity of people, a bunch of neighborhoods, +sticking stuff up on poles. And a bunch of other people laying in dark fiber. +So what happens to the telephone companies? There's enormous pressure +on them from both sides. + +"The more I look at this, the more I believe that in a post-industrial, +digital world, the idea of regulated monopolies is bad. People will +look back on it and say that in the 19th and 20th centuries +the idea of public utilities was an okay compromise. +You needed one set of wires in the ground. It was too economically +inefficient, otherwise. And that meant one entity running it. +But now, with pieces being wireless--the connections are going +to be via high-level interfaces, not via wires. I mean, ULTIMATELY +there are going to be wires--but the wires are just a commodity. +Fiber, wireless. You no longer NEED a utility." + +Water utilities? Gas utilities? + +Of course we still need those, he agrees. "But when what you're moving +is information, instead of physical substances, then you can play by +a different set of rules. We're evolving those rules now! +Hopefully you can have a much more decentralized system, +and one in which there's more competition in the marketplace. + +"The role of government will be to make sure that nobody cheats. +The proverbial `level playing field.' A policy that prevents monopolization. +It should result in better service, lower prices, more choices, +and local empowerment." He smiles. "I'm very big on local empowerment." + +Kapor is a man with a vision. It's a very novel vision which he +and his allies are working out in considerable detail and with great energy. +Dark, cynical, morbid cyberpunk that I am, I cannot avoid considering +some of the darker implications of "decentralized, nonhierarchical, +locally empowered" networking. + +I remark that some pundits have suggested that electronic networking--faxes, +phones, small-scale photocopiers--played a strong role in dissolving +the power of centralized communism and causing the collapse of the Warsaw Pact. + +Socialism is totally discredited, says Kapor, fresh back from +the Eastern Bloc. The idea that faxes did it, all by themselves, +is rather wishful thinking. + +Has it occurred to him that electronic networking might corrode +America's industrial and political infrastructure to the point +where the whole thing becomes untenable, unworkable--and the old order +just collapses headlong, like in Eastern Europe? + +"No," Kapor says flatly. "I think that's extraordinarily unlikely. +In part, because ten or fifteen years ago, I had similar hopes +about personal computers--which utterly failed to materialize." +He grins wryly, then his eyes narrow. "I'm VERY opposed to techno-utopias. +Every time I see one, I either run away, or try to kill it." + +It dawns on me then that Mitch Kapor is not trying to +make the world safe for democracy. He certainly is not +trying to make it safe for anarchists or utopians-- +least of all for computer intruders or electronic rip-off artists. +What he really hopes to do is make the world safe for +future Mitch Kapors. This world of decentralized, small-scale nodes, +with instant global access for the best and brightest, +would be a perfect milieu for the shoestring attic capitalism +that made Mitch Kapor what he is today. + +Kapor is a very bright man. He has a rare combination +of visionary intensity with a strong practical streak. +The Board of the EFF: John Barlow, Jerry Berman of the ACLU, +Stewart Brand, John Gilmore, Steve Wozniak, and Esther Dyson, +the doyenne of East-West computer entrepreneurism--share his gift, +his vision, and his formidable networking talents. +They are people of the 1960s, winnowed-out by its turbulence +and rewarded with wealth and influence. They are some of the best +and the brightest that the electronic community has to offer. +But can they do it, in the real world? Or are they only dreaming? +They are so few. And there is so much against them. + +I leave Kapor and his networking employees struggling cheerfully +with the promising intricacies of their newly installed Macintosh +System 7 software. The next day is Saturday. EFF is closed. +I pay a few visits to points of interest downtown. + +One of them is the birthplace of the telephone. + +It's marked by a bronze plaque in a plinth of black-and-white speckled granite. It sits in the +plaza of the John F. Kennedy Federal Building, the very place where Kapor was +once fingerprinted by the FBI. + +The plaque has a bas-relief picture of Bell's original telephone. +"BIRTHPLACE OF THE TELEPHONE," it reads. "Here, on June 2, 1875, +Alexander Graham Bell and Thomas A. Watson first transmitted sound over wires. + +"This successful experiment was completed in a fifth floor garret +at what was then 109 Court Street and marked the beginning of +world-wide telephone service." + +109 Court Street is long gone. Within sight of Bell's plaque, +across a street, is one of the central offices of NYNEX, +the local Bell RBOC, on 6 Bowdoin Square. + +I cross the street and circle the telco building, slowly, +hands in my jacket pockets. It's a bright, windy, New England +autumn day. The central office is a handsome 1940s-era megalith +in late Art Deco, eight stories high. + +Parked outside the back is a power-generation truck. +The generator strikes me as rather anomalous. Don't they +already have their own generators in this eight-story monster? +Then the suspicion strikes me that NYNEX must have heard +of the September 17 AT&T power-outage which crashed New York City. +Belt-and-suspenders, this generator. Very telco. + +Over the glass doors of the front entrance is a handsome bronze +bas-relief of Art Deco vines, sunflowers, and birds, entwining +the Bell logo and the legend NEW ENGLAND TELEPHONE AND TELEGRAPH COMPANY +--an entity which no longer officially exists. + +The doors are locked securely. I peer through the shadowed glass. +Inside is an official poster reading: + + +"New England Telephone a NYNEX Company + +ATTENTION + +"All persons while on New England Telephone +Company premises are required to visibly wear their +identification cards (C.C.P. Section 2, Page 1). + +"Visitors, vendors, contractors, and all others are +required to visibly wear a daily pass. + +"Thank you. + +Kevin C. Stanton. +Building Security Coordinator." + + +Outside, around the corner, is a pull-down ribbed metal security door, +a locked delivery entrance. Some passing stranger has grafitti-tagged +this door, with a single word in red spray-painted cursive: + +Fury + +# + +My book on the Hacker Crackdown is almost over now. +I have deliberately saved the best for last. + +In February 1991, I attended the CPSR Public Policy Roundtable, +in Washington, DC. CPSR, Computer Professionals for Social Responsibility, +was a sister organization of EFF, or perhaps its aunt, being older +and perhaps somewhat wiser in the ways of the world of politics. + +Computer Professionals for Social Responsibility began in 1981 +in Palo Alto, as an informal discussion group of Californian +computer scientists and technicians, united by nothing more +than an electronic mailing list. This typical high-tech +ad-hocracy received the dignity of its own acronym in 1982, +and was formally incorporated in 1983. + +CPSR lobbied government and public alike with an educational +outreach effort, sternly warning against any foolish +and unthinking trust in complex computer systems. +CPSR insisted that mere computers should never be +considered a magic panacea for humanity's social, +ethical or political problems. CPSR members were especially +troubled about the stability, safety, and dependability +of military computer systems, and very especially troubled +by those systems controlling nuclear arsenals. CPSR was +best-known for its persistent and well-publicized attacks on the +scientific credibility of the Strategic Defense Initiative ("Star Wars"). + +In 1990, CPSR was the nation's veteran cyber-political activist group, +with over two thousand members in twenty- one local chapters across the US. +It was especially active in Boston, Silicon Valley, and Washington DC, +where its Washington office sponsored the Public Policy Roundtable. + +The Roundtable, however, had been funded by EFF, which had passed CPSR +an extensive grant for operations. This was the first large-scale, +official meeting of what was to become the electronic civil +libertarian community. + +Sixty people attended, myself included--in this instance, not so much +as a journalist as a cyberpunk author. Many of the luminaries +of the field took part: Kapor and Godwin as a matter of course. +Richard Civille and Marc Rotenberg of CPSR. Jerry Berman of the ACLU. +John Quarterman, author of The Matrix. Steven Levy, author of Hackers. +George Perry and Sandy Weiss of Prodigy Services, there to network +about the civil-liberties troubles their young commercial +network was experiencing. Dr. Dorothy Denning. Cliff Figallo, +manager of the Well. Steve Jackson was there, having finally +found his ideal target audience, and so was Craig Neidorf, +"Knight Lightning" himself, with his attorney, Sheldon Zenner. +Katie Hafner, science journalist, and co-author of Cyberpunk: +Outlaws and Hackers on the Computer Frontier. Dave Farber, +ARPAnet pioneer and fabled Internet guru. Janlori Goldman +of the ACLU's Project on Privacy and Technology. John Nagle +of Autodesk and the Well. Don Goldberg of the House Judiciary Committee. +Tom Guidoboni, the defense attorney in the Internet Worm case. +Lance Hoffman, computer-science professor at The George Washington +University. Eli Noam of Columbia. And a host of others no less distinguished. + +Senator Patrick Leahy delivered the keynote address, +expressing his determination to keep ahead of the curve +on the issue of electronic free speech. The address was +well-received, and the sense of excitement was palpable. +Every panel discussion was interesting--some were entirely +compelling. People networked with an almost frantic interest. + +I myself had a most interesting and cordial lunch discussion with +Noel and Jeanne Gayler, Admiral Gayler being a former director +of the National Security Agency. As this was the first known encounter +between an actual no-kidding cyberpunk and a chief executive of +America's largest and best-financed electronic espionage apparat, +there was naturally a bit of eyebrow-raising on both sides. + +Unfortunately, our discussion was off-the-record. In fact +all the discussions at the CPSR were officially off-the-record, +the idea being to do some serious networking in an atmosphere +of complete frankness, rather than to stage a media circus. + +In any case, CPSR Roundtable, though interesting and intensely valuable, +was as nothing compared to the truly mind-boggling event that transpired +a mere month later. + +# + +"Computers, Freedom and Privacy." Four hundred people from +every conceivable corner of America's electronic community. +As a science fiction writer, I have been to some weird gigs in my day, +but this thing is truly BEYOND THE PALE. Even "Cyberthon," +Point Foundation's "Woodstock of Cyberspace" where Bay Area +psychedelia collided headlong with the emergent world +of computerized virtual reality, was like a Kiwanis Club gig +compared to this astonishing do. + +The "electronic community" had reached an apogee. +Almost every principal in this book is in attendance. +Civil Libertarians. Computer Cops. The Digital Underground. +Even a few discreet telco people. Colorcoded dots +for lapel tags are distributed. Free Expression issues. +Law Enforcement. Computer Security. Privacy. Journalists. +Lawyers. Educators. Librarians. Programmers. +Stylish punk-black dots for the hackers and phone phreaks. +Almost everyone here seems to wear eight or nine dots, +to have six or seven professional hats. + +It is a community. Something like Lebanon perhaps, +but a digital nation. People who had feuded all year +in the national press, people who entertained the deepest +suspicions of one another's motives and ethics, are now +in each others' laps. "Computers, Freedom and Privacy" +had every reason in the world to turn ugly, and yet except +for small irruptions of puzzling nonsense from the +convention's token lunatic, a surprising bonhomie reigned. +CFP was like a wedding-party in which two lovers, +unstable bride and charlatan groom, tie the knot +in a clearly disastrous matrimony. + +It is clear to both families--even to neighbors and random guests-- +that this is not a workable relationship, and yet the young couple's +desperate attraction can brook no further delay. They simply cannot +help themselves. Crockery will fly, shrieks from their newlywed home +will wake the city block, divorce waits in the wings like a vulture +over the Kalahari, and yet this is a wedding, and there is going +to be a child from it. Tragedies end in death; comedies in marriage. +The Hacker Crackdown is ending in marriage. And there will be a child. + +From the beginning, anomalies reign. John Perry Barlow, +cyberspace ranger, is here. His color photo in +The New York Times Magazine, Barlow scowling +in a grim Wyoming snowscape, with long black coat, +dark hat, a Macintosh SE30 propped on a fencepost +and an awesome frontier rifle tucked under one arm, +will be the single most striking visual image +of the Hacker Crackdown. And he is CFP's guest of honor-- +along with Gail Thackeray of the FCIC! What on earth do +they expect these dual guests to do with each other? Waltz? + +Barlow delivers the first address. Uncharacteristically, +he is hoarse--the sheer volume of roadwork has worn him down. +He speaks briefly, congenially, in a plea for conciliation, +and takes his leave to a storm of applause. + +Then Gail Thackeray takes the stage. She's visibly nervous. +She's been on the Well a lot lately. Reading those Barlow posts. +Following Barlow is a challenge to anyone. In honor of the famous +lyricist for the Grateful Dead, she announces reedily, she is going to read-- +A POEM. A poem she has composed herself. + +It's an awful poem, doggerel in the rollicking meter of Robert W. Service's +The Cremation of Sam McGee, but it is in fact, a poem. It's the Ballad +of the Electronic Frontier! A poem about the Hacker Crackdown and the +sheer unlikelihood of CFP. It's full of in-jokes. The score or so cops +in the audience, who are sitting together in a nervous claque, +are absolutely cracking-up. Gail's poem is the funniest goddamn thing +they've ever heard. The hackers and civil-libs, who had this woman figured +for Ilsa She-Wolf of the SS, are staring with their jaws hanging loosely. +Never in the wildest reaches of their imagination had they figured +Gail Thackeray was capable of such a totally off-the-wall move. +You can see them punching their mental CONTROL-RESET buttons. +Jesus! This woman's a hacker weirdo! She's JUST LIKE US! +God, this changes everything! + +Al Bayse, computer technician for the FBI, had been the only cop +at the CPSR Roundtable, dragged there with his arm bent by +Dorothy Denning. He was guarded and tightlipped at CPSR Roundtable; +a "lion thrown to the Christians." + +At CFP, backed by a claque of cops, Bayse suddenly waxes eloquent +and even droll, describing the FBI's "NCIC 2000", a gigantic digital catalog +of criminal records, as if he has suddenly become some weird hybrid +of George Orwell and George Gobel. Tentatively, he makes an arcane +joke about statistical analysis. At least a third of the crowd laughs aloud. + +"They didn't laugh at that at my last speech," Bayse observes. +He had been addressing cops--STRAIGHT cops, not computer people. +It had been a worthy meeting, useful one supposes, but nothing like THIS. +There has never been ANYTHING like this. Without any prodding, +without any preparation, people in the audience simply begin to ask questions. +Longhairs, freaky people, mathematicians. Bayse is answering, politely, +frankly, fully, like a man walking on air. The ballroom's atmosphere +crackles with surreality. A female lawyer behind me breaks into a sweat +and a hot waft of surprisingly potent and musky perfume flows off +her pulse-points. + +People are giddy with laughter. People are interested, +fascinated, their eyes so wide and dark that they seem eroticized. +Unlikely daisy-chains form in the halls, around the bar, on the escalators: +cops with hackers, civil rights with FBI, Secret Service with phone phreaks. + +Gail Thackeray is at her crispest in a white wool sweater with a +tiny Secret Service logo. "I found Phiber Optik at the payphones, +and when he saw my sweater, he turned into a PILLAR OF SALT!" she chortles. + +Phiber discusses his case at much length with his arresting officer, +Don Delaney of the New York State Police. After an hour's chat, +the two of them look ready to begin singing "Auld Lang Syne." +Phiber finally finds the courage to get his worst complaint off his chest. +It isn't so much the arrest. It was the CHARGE. Pirating service +off 900 numbers. I'm a PROGRAMMER, Phiber insists. This lame charge +is going to hurt my reputation. It would have been cool to be busted +for something happening, like Section 1030 computer intrusion. +Maybe some kind of crime that's scarcely been invented yet. +Not lousy phone fraud. Phooey. + +Delaney seems regretful. He had a mountain of possible criminal charges +against Phiber Optik. The kid's gonna plead guilty anyway. He's a +first timer, they always plead. Coulda charged the kid with most anything, +and gotten the same result in the end. Delaney seems genuinely sorry +not to have gratified Phiber in this harmless fashion. Too late now. +Phiber's pled already. All water under the bridge. Whaddya gonna do? + +Delaney's got a good grasp on the hacker mentality. +He held a press conference after he busted a bunch of +Masters of Deception kids. Some journo had asked him: +"Would you describe these people as GENIUSES?" +Delaney's deadpan answer, perfect: "No, I would describe +these people as DEFENDANTS." Delaney busts a kid for +hacking codes with repeated random dialling. Tells the +press that NYNEX can track this stuff in no time flat nowadays, +and a kid has to be STUPID to do something so easy to catch. +Dead on again: hackers don't mind being thought of as Genghis Khan +by the straights, but if there's anything that really gets 'em +where they live, it's being called DUMB. + +Won't be as much fun for Phiber next time around. +As a second offender he's gonna see prison. +Hackers break the law. They're not geniuses, either. +They're gonna be defendants. And yet, Delaney muses over +a drink in the hotel bar, he has found it impossible to treat +them as common criminals. Delaney knows criminals. These kids, +by comparison, are clueless--there is just no crook vibe off of them, +they don't smell right, they're just not BAD. + +Delaney has seen a lot of action. He did Vietnam. +He's been shot at, he has shot people. He's a homicide +cop from New York. He has the appearance of a man who +has not only seen the shit hit the fan but has seen it splattered +across whole city blocks and left to ferment for years. +This guy has been around. + +He listens to Steve Jackson tell his story. The dreamy +game strategist has been dealt a bad hand. He has played +it for all he is worth. Under his nerdish SF-fan exterior +is a core of iron. Friends of his say Steve Jackson believes +in the rules, believes in fair play. He will never compromise +his principles, never give up. "Steve," Delaney says to +Steve Jackson, "they had some balls, whoever busted you. +You're all right!" Jackson, stunned, falls silent and +actually blushes with pleasure. + +Neidorf has grown up a lot in the past year. The kid is +a quick study, you gotta give him that. Dressed by his mom, +the fashion manager for a national clothing chain, +Missouri college techie-frat Craig Neidorf out-dappers +everyone at this gig but the toniest East Coast lawyers. +The iron jaws of prison clanged shut without him and now +law school beckons for Neidorf. He looks like a larval Congressman. + +Not a "hacker," our Mr. Neidorf. He's not interested +in computer science. Why should he be? He's not +interested in writing C code the rest of his life, +and besides, he's seen where the chips fall. +To the world of computer science he and Phrack +were just a curiosity. But to the world of law. . . . +The kid has learned where the bodies are buried. +He carries his notebook of press clippings wherever he goes. + +Phiber Optik makes fun of Neidorf for a Midwestern geek, +for believing that "Acid Phreak" does acid and listens to acid rock. +Hell no. Acid's never done ACID! Acid's into ACID HOUSE MUSIC. +Jesus. The very idea of doing LSD. Our PARENTS did LSD, ya clown. + +Thackeray suddenly turns upon Craig Neidorf the full lighthouse +glare of her attention and begins a determined half-hour attempt +to WIN THE BOY OVER. The Joan of Arc of Computer Crime is +GIVING CAREER ADVICE TO KNIGHT LIGHTNING! "Your experience +would be very valuable--a real asset," she tells him with +unmistakeable sixty-thousand-watt sincerity. Neidorf is fascinated. +He listens with unfeigned attention. He's nodding and saying yes ma'am. +Yes, Craig, you too can forget all about money and enter the glamorous +and horribly underpaid world of PROSECUTING COMPUTER CRIME! +You can put your former friends in prison--ooops. . . . + +You cannot go on dueling at modem's length indefinitely. +You cannot beat one another senseless with rolled-up press-clippings. +Sooner or later you have to come directly to grips. +And yet the very act of assembling here has changed +the entire situation drastically. John Quarterman, +author of The Matrix, explains the Internet at his symposium. +It is the largest news network in the world, it is growing +by leaps and bounds, and yet you cannot measure Internet because +you cannot stop it in place. It cannot stop, because there +is no one anywhere in the world with the authority to stop Internet. +It changes, yes, it grows, it embeds itself across the post-industrial, +postmodern world and it generates community wherever it +touches, and it is doing this all by itself. + +Phiber is different. A very fin de siecle kid, Phiber Optik. +Barlow says he looks like an Edwardian dandy. He does rather. +Shaven neck, the sides of his skull cropped hip-hop close, +unruly tangle of black hair on top that looks pomaded, +he stays up till four a.m. and misses all the sessions, +then hangs out in payphone booths with his acoustic coupler +gutsily CRACKING SYSTEMS RIGHT IN THE MIDST OF THE HEAVIEST +LAW ENFORCEMENT DUDES IN THE U.S., or at least PRETENDING to. . . . +Unlike "Frank Drake." Drake, who wrote Dorothy Denning out +of nowhere, and asked for an interview for his cheapo +cyberpunk fanzine, and then started grilling her on her ethics. +She was squirmin', too. . . . Drake, scarecrow-tall with his +floppy blond mohawk, rotting tennis shoes and black leather jacket +lettered ILLUMINATI in red, gives off an unmistakeable air +of the bohemian literatus. Drake is the kind of guy +who reads British industrial design magazines and appreciates +William Gibson because the quality of the prose is so tasty. +Drake could never touch a phone or a keyboard again, +and he'd still have the nose-ring and the blurry photocopied +fanzines and the sampled industrial music. He's a radical punk +with a desktop-publishing rig and an Internet address. +Standing next to Drake, the diminutive Phiber looks like he's +been physically coagulated out of phone-lines. Born to phreak. + +Dorothy Denning approaches Phiber suddenly. The two of them +are about the same height and body-build. Denning's blue eyes +flash behind the round window-frames of her glasses. +"Why did you say I was `quaint?'" she asks Phiber, quaintly. + +It's a perfect description but Phiber is nonplussed. . . +"Well, I uh, you know. . . ." + +"I also think you're quaint, Dorothy," I say, novelist to the rescue, +the journo gift of gab. . . . She is neat and dapper and yet there's +an arcane quality to her, something like a Pilgrim Maiden behind +leaded glass; if she were six inches high Dorothy Denning would look +great inside a china cabinet. . .The Cryptographeress. . . +The Cryptographrix. . .whatever. . . . Weirdly, Peter Denning looks +just like his wife, you could pick this gentleman out of a thousand guys +as the soulmate of Dorothy Denning. Wearing tailored slacks, +a spotless fuzzy varsity sweater, and a neatly knotted academician's tie. . . . +This fineboned, exquisitely polite, utterly civilized and hyperintelligent +couple seem to have emerged from some cleaner and finer parallel universe, +where humanity exists to do the Brain Teasers column in Scientific American. +Why does this Nice Lady hang out with these unsavory characters? + +Because the time has come for it, that's why. +Because she's the best there is at what she does. + +Donn Parker is here, the Great Bald Eagle of Computer Crime. . . . +With his bald dome, great height, and enormous Lincoln-like hands, +the great visionary pioneer of the field plows through the lesser mortals +like an icebreaker. . . . His eyes are fixed on the future with the +rigidity of a bronze statue. . . . Eventually, he tells his audience, +all business crime will be computer crime, because businesses will do +everything through computers. "Computer crime" as a category will vanish. + +In the meantime, passing fads will flourish and fail and evaporate. . . . +Parker's commanding, resonant voice is sphinxlike, everything is viewed +from some eldritch valley of deep historical abstraction. . . . +Yes, they've come and they've gone, these passing flaps in the world +of digital computation. . . . The radio-frequency emanation scandal. . . +KGB and MI5 and CIA do it every day, it's easy, but nobody else ever has. . . . +The salami-slice fraud, mostly mythical. . . . "Crimoids," he calls them. . . . +Computer viruses are the current crimoid champ, a lot less dangerous than +most people let on, but the novelty is fading and there's a crimoid vacuum at +the moment, the press is visibly hungering for something more outrageous. . . . +The Great Man shares with us a few speculations on the coming crimoids. . . . +Desktop Forgery! Wow. . . . Computers stolen just for the sake of the +information within them--data-napping! Happened in Britain a while ago, +could be the coming thing. . . . Phantom nodes in the Internet! + +Parker handles his overhead projector sheets with an ecclesiastical air. . . . +He wears a grey double-breasted suit, a light blue shirt, and a +very quiet tie of understated maroon and blue paisley. . . . +Aphorisms emerge from him with slow, leaden emphasis. . . . +There is no such thing as an adequately secure computer +when one faces a sufficiently powerful adversary. . . . +Deterrence is the most socially useful aspect of security. . . . +People are the primary weakness in all information systems. . . . +The entire baseline of computer security must be shifted upward. . . . +Don't ever violate your security by publicly describing +your security measures. . . . + +People in the audience are beginning to squirm, and yet +there is something about the elemental purity of this guy's +philosophy that compels uneasy respect. . . . Parker sounds +like the only sane guy left in the lifeboat, sometimes. +The guy who can prove rigorously, from deep moral principles, +that Harvey there, the one with the broken leg and the checkered past, +is the one who has to be, err. . .that is, Mr. Harvey is best placed +to make the necessary sacrifice for the security and indeed +the very survival of the rest of this lifeboat's crew. . . . +Computer security, Parker informs us mournfully, is a +nasty topic, and we wish we didn't have to have it. . . . +The security expert, armed with method and logic, must think--imagine-- +everything that the adversary might do before the adversary might +actually do it. It is as if the criminal's dark brain were an +extensive subprogram within the shining cranium of Donn Parker. +He is a Holmes whose Moriarty does not quite yet exist +and so must be perfectly simulated. + +CFP is a stellar gathering, with the giddiness of a wedding. +It is a happy time, a happy ending, they know their world +is changing forever tonight, and they're proud to have been there +to see it happen, to talk, to think, to help. + +And yet as night falls, a certain elegiac quality manifests itself, +as the crowd gathers beneath the chandeliers with their wineglasses +and dessert plates. Something is ending here, gone forever, +and it takes a while to pinpoint it. + +It is the End of the Amateurs. + + + + + + + + + +End of the Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling + +*** END OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** + +***** This file should be named 101.txt or 101.zip ***** +This and all associated files of various formats will be found in: + http://www.gutenberg.org/1/0/101/ + + + +Updated editions will replace the previous one--the old editions will be +renamed. + +Creating the works from public domain print editions means that no one +owns a United States copyright in these works, so the Foundation (and +you!) can copy and distribute it in the United States without permission +and without paying copyright royalties. Special rules, set forth in the +General Terms of Use part of this license, apply to copying and +distributing Project Gutenberg-tm electronic works to protect the +PROJECT GUTENBERG-tm concept and trademark. Project Gutenberg is a +registered trademark, and may not be used if you charge for the eBooks, +unless you receive specific permission. If you do not charge anything +for copies of this eBook, complying with the rules is very easy. You may +use this eBook for nearly any purpose such as creation of derivative +works, reports, performances and research. They may be modified and +printed and given away--you may do practically ANYTHING with public +domain eBooks. Redistribution is subject to the trademark license, +especially commercial redistribution. + + + +*** START: FULL LICENSE *** + +THE FULL PROJECT GUTENBERG LICENSE +PLEASE READ THIS BEFORE YOU DISTRIBUTE OR USE THIS WORK + +To protect the Project Gutenberg-tm mission of promoting the free +distribution of electronic works, by using or distributing this work +(or any other work associated in any way with the phrase "Project +Gutenberg"), you agree to comply with all the terms of the Full Project +Gutenberg-tm License (available with this file or online at +http://www.gutenberg.org/license). + + +Section 1. General Terms of Use and Redistributing Project Gutenberg-tm +electronic works + +1.A. By reading or using any part of this Project Gutenberg-tm +electronic work, you indicate that you have read, understand, agree to +and accept all the terms of this license and intellectual property +(trademark/copyright) agreement. If you do not agree to abide by all +the terms of this agreement, you must cease using and return or destroy +all copies of Project Gutenberg-tm electronic works in your possession. +If you paid a fee for obtaining a copy of or access to a Project +Gutenberg-tm electronic work and you do not agree to be bound by the +terms of this agreement, you may obtain a refund from the person or +entity to whom you paid the fee as set forth in paragraph 1.E.8. + +1.B. "Project Gutenberg" is a registered trademark. It may only be +used on or associated in any way with an electronic work by people who +agree to be bound by the terms of this agreement. There are a few +things that you can do with most Project Gutenberg-tm electronic works +even without complying with the full terms of this agreement. See +paragraph 1.C below. There are a lot of things you can do with Project +Gutenberg-tm electronic works if you follow the terms of this agreement +and help preserve free future access to Project Gutenberg-tm electronic +works. See paragraph 1.E below. + +1.C. The Project Gutenberg Literary Archive Foundation ("the Foundation" +or PGLAF), owns a compilation copyright in the collection of Project +Gutenberg-tm electronic works. Nearly all the individual works in the +collection are in the public domain in the United States. If an +individual work is in the public domain in the United States and you are +located in the United States, we do not claim a right to prevent you from +copying, distributing, performing, displaying or creating derivative +works based on the work as long as all references to Project Gutenberg +are removed. Of course, we hope that you will support the Project +Gutenberg-tm mission of promoting free access to electronic works by +freely sharing Project Gutenberg-tm works in compliance with the terms of +this agreement for keeping the Project Gutenberg-tm name associated with +the work. You can easily comply with the terms of this agreement by +keeping this work in the same format with its attached full Project +Gutenberg-tm License when you share it without charge with others. +This particular work is one of the few copyrighted individual works +included with the permission of the copyright holder. Information on +the copyright owner for this particular work and the terms of use +imposed by the copyright holder on this work are set forth at the +beginning of this work. + +1.D. The copyright laws of the place where you are located also govern +what you can do with this work. Copyright laws in most countries are in +a constant state of change. If you are outside the United States, check +the laws of your country in addition to the terms of this agreement +before downloading, copying, displaying, performing, distributing or +creating derivative works based on this work or any other Project +Gutenberg-tm work. The Foundation makes no representations concerning +the copyright status of any work in any country outside the United +States. + +1.E. Unless you have removed all references to Project Gutenberg: + +1.E.1. The following sentence, with active links to, or other immediate +access to, the full Project Gutenberg-tm License must appear prominently +whenever any copy of a Project Gutenberg-tm work (any work on which the +phrase "Project Gutenberg" appears, or with which the phrase "Project +Gutenberg" is associated) is accessed, displayed, performed, viewed, +copied or distributed: + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + +1.E.2. If an individual Project Gutenberg-tm electronic work is derived +from the public domain (does not contain a notice indicating that it is +posted with permission of the copyright holder), the work can be copied +and distributed to anyone in the United States without paying any fees +or charges. If you are redistributing or providing access to a work +with the phrase "Project Gutenberg" associated with or appearing on the +work, you must comply either with the requirements of paragraphs 1.E.1 +through 1.E.7 or obtain permission for the use of the work and the +Project Gutenberg-tm trademark as set forth in paragraphs 1.E.8 or +1.E.9. + +1.E.3. If an individual Project Gutenberg-tm electronic work is posted +with the permission of the copyright holder, your use and distribution +must comply with both paragraphs 1.E.1 through 1.E.7 and any additional +terms imposed by the copyright holder. Additional terms will be linked +to the Project Gutenberg-tm License for all works posted with the +permission of the copyright holder found at the beginning of this work. + +1.E.4. Do not unlink or detach or remove the full Project Gutenberg-tm +License terms from this work, or any files containing a part of this +work or any other work associated with Project Gutenberg-tm. + +1.E.5. Do not copy, display, perform, distribute or redistribute this +electronic work, or any part of this electronic work, without +prominently displaying the sentence set forth in paragraph 1.E.1 with +active links or immediate access to the full terms of the Project +Gutenberg-tm License. + +1.E.6. You may convert to and distribute this work in any binary, +compressed, marked up, nonproprietary or proprietary form, including any +word processing or hypertext form. However, if you provide access to or +distribute copies of a Project Gutenberg-tm work in a format other than +"Plain Vanilla ASCII" or other format used in the official version +posted on the official Project Gutenberg-tm web site (www.gutenberg.org), +you must, at no additional cost, fee or expense to the user, provide a +copy, a means of exporting a copy, or a means of obtaining a copy upon +request, of the work in its original "Plain Vanilla ASCII" or other +form. Any alternate format must include the full Project Gutenberg-tm +License as specified in paragraph 1.E.1. + +1.E.7. Do not charge a fee for access to, viewing, displaying, +performing, copying or distributing any Project Gutenberg-tm works +unless you comply with paragraph 1.E.8 or 1.E.9. + +1.E.8. You may charge a reasonable fee for copies of or providing +access to or distributing Project Gutenberg-tm electronic works provided +that + +- You pay a royalty fee of 20% of the gross profits you derive from + the use of Project Gutenberg-tm works calculated using the method + you already use to calculate your applicable taxes. The fee is + owed to the owner of the Project Gutenberg-tm trademark, but he + has agreed to donate royalties under this paragraph to the + Project Gutenberg Literary Archive Foundation. Royalty payments + must be paid within 60 days following each date on which you + prepare (or are legally required to prepare) your periodic tax + returns. Royalty payments should be clearly marked as such and + sent to the Project Gutenberg Literary Archive Foundation at the + address specified in Section 4, "Information about donations to + the Project Gutenberg Literary Archive Foundation." + +- You provide a full refund of any money paid by a user who notifies + you in writing (or by e-mail) within 30 days of receipt that s/he + does not agree to the terms of the full Project Gutenberg-tm + License. You must require such a user to return or + destroy all copies of the works possessed in a physical medium + and discontinue all use of and all access to other copies of + Project Gutenberg-tm works. + +- You provide, in accordance with paragraph 1.F.3, a full refund of any + money paid for a work or a replacement copy, if a defect in the + electronic work is discovered and reported to you within 90 days + of receipt of the work. + +- You comply with all other terms of this agreement for free + distribution of Project Gutenberg-tm works. + +1.E.9. If you wish to charge a fee or distribute a Project Gutenberg-tm +electronic work or group of works on different terms than are set +forth in this agreement, you must obtain permission in writing from +both the Project Gutenberg Literary Archive Foundation and Michael +Hart, the owner of the Project Gutenberg-tm trademark. Contact the +Foundation as set forth in Section 3 below. + +1.F. + +1.F.1. Project Gutenberg volunteers and employees expend considerable +effort to identify, do copyright research on, transcribe and proofread +public domain works in creating the Project Gutenberg-tm +collection. Despite these efforts, Project Gutenberg-tm electronic +works, and the medium on which they may be stored, may contain +"Defects," such as, but not limited to, incomplete, inaccurate or +corrupt data, transcription errors, a copyright or other intellectual +property infringement, a defective or damaged disk or other medium, a +computer virus, or computer codes that damage or cannot be read by +your equipment. + +1.F.2. LIMITED WARRANTY, DISCLAIMER OF DAMAGES - Except for the "Right +of Replacement or Refund" described in paragraph 1.F.3, the Project +Gutenberg Literary Archive Foundation, the owner of the Project +Gutenberg-tm trademark, and any other party distributing a Project +Gutenberg-tm electronic work under this agreement, disclaim all +liability to you for damages, costs and expenses, including legal +fees. YOU AGREE THAT YOU HAVE NO REMEDIES FOR NEGLIGENCE, STRICT +LIABILITY, BREACH OF WARRANTY OR BREACH OF CONTRACT EXCEPT THOSE +PROVIDED IN PARAGRAPH 1.F.3. YOU AGREE THAT THE FOUNDATION, THE +TRADEMARK OWNER, AND ANY DISTRIBUTOR UNDER THIS AGREEMENT WILL NOT BE +LIABLE TO YOU FOR ACTUAL, DIRECT, INDIRECT, CONSEQUENTIAL, PUNITIVE OR +INCIDENTAL DAMAGES EVEN IF YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH +DAMAGE. + +1.F.3. LIMITED RIGHT OF REPLACEMENT OR REFUND - If you discover a +defect in this electronic work within 90 days of receiving it, you can +receive a refund of the money (if any) you paid for it by sending a +written explanation to the person you received the work from. If you +received the work on a physical medium, you must return the medium with +your written explanation. The person or entity that provided you with +the defective work may elect to provide a replacement copy in lieu of a +refund. If you received the work electronically, the person or entity +providing it to you may choose to give you a second opportunity to +receive the work electronically in lieu of a refund. If the second copy +is also defective, you may demand a refund in writing without further +opportunities to fix the problem. + +1.F.4. Except for the limited right of replacement or refund set forth +in paragraph 1.F.3, this work is provided to you 'AS-IS,' WITH NO OTHER +WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +WARRANTIES OF MERCHANTIBILITY OR FITNESS FOR ANY PURPOSE. + +1.F.5. Some states do not allow disclaimers of certain implied +warranties or the exclusion or limitation of certain types of damages. +If any disclaimer or limitation set forth in this agreement violates the +law of the state applicable to this agreement, the agreement shall be +interpreted to make the maximum disclaimer or limitation permitted by +the applicable state law. The invalidity or unenforceability of any +provision of this agreement shall not void the remaining provisions. + +1.F.6. INDEMNITY - You agree to indemnify and hold the Foundation, the +trademark owner, any agent or employee of the Foundation, anyone +providing copies of Project Gutenberg-tm electronic works in accordance +with this agreement, and any volunteers associated with the production, +promotion and distribution of Project Gutenberg-tm electronic works, +harmless from all liability, costs and expenses, including legal fees, +that arise directly or indirectly from any of the following which you do +or cause to occur: (a) distribution of this or any Project Gutenberg-tm +work, (b) alteration, modification, or additions or deletions to any +Project Gutenberg-tm work, and (c) any Defect you cause. + + +Section 2. Information about the Mission of Project Gutenberg-tm + +Project Gutenberg-tm is synonymous with the free distribution of +electronic works in formats readable by the widest variety of computers +including obsolete, old, middle-aged and new computers. It exists +because of the efforts of hundreds of volunteers and donations from +people in all walks of life. + +Volunteers and financial support to provide volunteers with the +assistance they need are critical to reaching Project Gutenberg-tm's +goals and ensuring that the Project Gutenberg-tm collection will +remain freely available for generations to come. In 2001, the Project +Gutenberg Literary Archive Foundation was created to provide a secure +and permanent future for Project Gutenberg-tm and future generations. +To learn more about the Project Gutenberg Literary Archive Foundation +and how your efforts and donations can help, see Sections 3 and 4 +and the Foundation web page at http://www.pglaf.org. + + +Section 3. Information about the Project Gutenberg Literary Archive +Foundation + +The Project Gutenberg Literary Archive Foundation is a non profit +501(c)(3) educational corporation organized under the laws of the +state of Mississippi and granted tax exempt status by the Internal +Revenue Service. The Foundation's EIN or federal tax identification +number is 64-6221541. Its 501(c)(3) letter is posted at +http://pglaf.org/fundraising. Contributions to the Project Gutenberg +Literary Archive Foundation are tax deductible to the full extent +permitted by U.S. federal laws and your state's laws. + +The Foundation's principal office is located at 4557 Melan Dr. S. +Fairbanks, AK, 99712., but its volunteers and employees are scattered +throughout numerous locations. Its business office is located at +809 North 1500 West, Salt Lake City, UT 84116, (801) 596-1887, email +business@pglaf.org. Email contact links and up to date contact +information can be found at the Foundation's web site and official +page at http://pglaf.org + +For additional contact information: + Dr. Gregory B. Newby + Chief Executive and Director + gbnewby@pglaf.org + +Section 4. Information about Donations to the Project Gutenberg +Literary Archive Foundation + +Project Gutenberg-tm depends upon and cannot survive without wide +spread public support and donations to carry out its mission of +increasing the number of public domain and licensed works that can be +freely distributed in machine readable form accessible by the widest +array of equipment including outdated equipment. Many small donations +($1 to $5,000) are particularly important to maintaining tax exempt +status with the IRS. + +The Foundation is committed to complying with the laws regulating +charities and charitable donations in all 50 states of the United +States. Compliance requirements are not uniform and it takes a +considerable effort, much paperwork and many fees to meet and keep up +with these requirements. We do not solicit donations in locations +where we have not received written confirmation of compliance. To +SEND DONATIONS or determine the status of compliance for any +particular state visit http://pglaf.org + +While we cannot and do not solicit contributions from states where we +have not met the solicitation requirements, we know of no prohibition +against accepting unsolicited donations from donors in such states who +approach us with offers to donate. + +International donations are gratefully accepted, but we cannot make +any statements concerning tax treatment of donations received from +outside the United States. U.S. laws alone swamp our small staff. + +Please check the Project Gutenberg Web pages for current donation +methods and addresses. Donations are accepted in a number of other +ways including checks, online payments and credit card donations. +To donate, please visit: http://pglaf.org/donate + + +Section 5. General Information About Project Gutenberg-tm electronic +works. + +Professor Michael S. Hart is the originator of the Project Gutenberg-tm +concept of a library of electronic works that could be freely shared +with anyone. For thirty years, he produced and distributed Project +Gutenberg-tm eBooks with only a loose network of volunteer support. + +Project Gutenberg-tm eBooks are often created from several printed +editions, all of which are confirmed as Public Domain in the U.S. +unless a copyright notice is included. Thus, we do not necessarily +keep eBooks in compliance with any particular paper edition. + +Each eBook is in a subdirectory of the same number as the eBook's +eBook number, often in several formats including plain vanilla ASCII, +compressed (zipped), HTML and others. + +Corrected EDITIONS of our eBooks replace the old file and take over +the old filename and etext number. The replaced older file is renamed. +VERSIONS based on separate sources are treated as new eBooks receiving +new filenames and etext numbers. + +Most people start at our Web site which has the main PG search facility: + +http://www.gutenberg.org + +This Web site includes information about Project Gutenberg-tm, +including how to make donations to the Project Gutenberg Literary +Archive Foundation, how to help produce our new eBooks, and how to +subscribe to our email newsletter to hear about new eBooks. + +EBooks posted prior to November 2003, with eBook numbers BELOW #10000, +are filed in directories based on their release date. If you want to +download any of these eBooks directly, rather than using the regular +search system you may utilize the following addresses and just +download by the etext year. + +http://www.ibiblio.org/gutenberg/etext06 + + (Or /etext 05, 04, 03, 02, 01, 00, 99, + 98, 97, 96, 95, 94, 93, 92, 92, 91 or 90) + +EBooks posted since November 2003, with etext numbers OVER #10000, are +filed in a different way. The year of a release date is no longer part +of the directory path. The path is based on the etext number (which is +identical to the filename). The path to the file is made up of single +digits corresponding to all but the last digit in the filename. For +example an eBook of filename 10234 would be found at: + +http://www.gutenberg.org/1/0/2/3/10234 + +or filename 24689 would be found at: +http://www.gutenberg.org/2/4/6/8/24689 + +An alternative method of locating eBooks: +http://www.gutenberg.org/GUTINDEX.ALL + +*** END: FULL LICENSE *** diff --git a/tests/sqlcipher/test_async.py b/tests/sqlcipher/test_async.py new file mode 100644 index 00000000..5c220cc4 --- /dev/null +++ b/tests/sqlcipher/test_async.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# test_async.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import os +import hashlib + +from twisted.internet import defer + +from test_soledad.util import BaseSoledadTest +from leap.soledad.client._db import adbapi +from leap.soledad.client._db import sqlcipher + + +class ASyncSQLCipherRetryTestCase(BaseSoledadTest): + + """ + Test asynchronous SQLCipher operation. + """ + + NUM_DOCS = 5000 + + def setUp(self): + BaseSoledadTest.setUp(self) + self._dbpool = self._get_dbpool() + + def tearDown(self): + self._dbpool.close() + BaseSoledadTest.tearDown(self) + + def _get_dbpool(self): + tmpdb = os.path.join(self.tempdir, "test.soledad") + opts = sqlcipher.SQLCipherOptions(tmpdb, "secret", create=True) + return adbapi.getConnectionPool(opts) + + def _get_sample(self): + if not getattr(self, "_sample", None): + dirname = os.path.dirname(os.path.realpath(__file__)) + sample_file = os.path.join(dirname, "hacker_crackdown.txt") + with open(sample_file) as f: + self._sample = f.readlines() + return self._sample + + def test_concurrent_puts_fail_with_few_retries_and_small_timeout(self): + """ + Test if concurrent updates to the database with small timeout and + small number of retries fail with "database is locked" error. + + Many concurrent write attempts to the same sqlcipher database may fail + when the timeout is small and there are no retries. This test will + pass if any of the attempts to write the database fail. + + This test is much dependent on the environment and its result intends + to contrast with the test for the workaround for the "database is + locked" problem, which is addressed by the "test_concurrent_puts" test + below. + + If this test ever fails, it means that either (1) the platform where + you are running is it very powerful and you should try with an even + lower timeout value, or (2) the bug has been solved by a better + implementation of the underlying database pool, and thus this test + should be removed from the test suite. + """ + + old_timeout = adbapi.SQLCIPHER_CONNECTION_TIMEOUT + old_max_retries = adbapi.SQLCIPHER_MAX_RETRIES + + adbapi.SQLCIPHER_CONNECTION_TIMEOUT = 1 + adbapi.SQLCIPHER_MAX_RETRIES = 1 + + def _create_doc(doc): + return self._dbpool.runU1DBQuery("create_doc", doc) + + def _insert_docs(): + deferreds = [] + for i in range(self.NUM_DOCS): + payload = self._get_sample()[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = _create_doc(doc) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def _errback(e): + if e.value[0].getErrorMessage() == "database is locked": + adbapi.SQLCIPHER_CONNECTION_TIMEOUT = old_timeout + adbapi.SQLCIPHER_MAX_RETRIES = old_max_retries + return defer.succeed("") + raise Exception + + d = _insert_docs() + d.addCallback(lambda _: self._dbpool.runU1DBQuery("get_all_docs")) + d.addErrback(_errback) + return d + + def test_concurrent_puts(self): + """ + Test that many concurrent puts succeed. + + Currently, there's a known problem with the concurrent database pool + which is that many concurrent attempts to write to the database may + fail when the lock timeout is small and when there are no (or few) + retries. We currently workaround this problem by increasing the + timeout and the number of retries. + + Should this test ever fail, it probably means that the timeout and/or + number of retries should be increased for the platform you're running + the test. If the underlying database pool is ever fixed, then the test + above will fail and we should remove this comment from here. + """ + + def _create_doc(doc): + return self._dbpool.runU1DBQuery("create_doc", doc) + + def _insert_docs(): + deferreds = [] + for i in range(self.NUM_DOCS): + payload = self._get_sample()[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = _create_doc(doc) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def _count_docs(results): + _, docs = results + if self.NUM_DOCS == len(docs): + return defer.succeed("") + raise Exception + + d = _insert_docs() + d.addCallback(lambda _: self._dbpool.runU1DBQuery("get_all_docs")) + d.addCallback(_count_docs) + return d diff --git a/tests/sqlcipher/test_backend.py b/tests/sqlcipher/test_backend.py new file mode 100644 index 00000000..68f8f9f2 --- /dev/null +++ b/tests/sqlcipher/test_backend.py @@ -0,0 +1,677 @@ +# -*- coding: utf-8 -*- +# test_sqlcipher.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test sqlcipher backend internals. +""" +import os +import pytest +import time +import threading +import sys + +# l2db stuff. +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import query_parser + +# soledad stuff. +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client._db.sqlite import SQLitePartialExpandDatabase +from leap.soledad.client._db.sqlcipher import SQLCipherDatabase +from leap.soledad.client._db.sqlcipher import SQLCipherOptions +from leap.soledad.client._db.sqlcipher import DatabaseIsNotEncrypted + +# u1db tests stuff. +from test_soledad import u1db_tests as tests +from test_soledad.u1db_tests import test_backends +from test_soledad.u1db_tests import test_open +from test_soledad.util import SQLCIPHER_SCENARIOS +from test_soledad.util import PASSWORD +from test_soledad.util import BaseSoledadTest + +from testscenarios import TestWithScenarios + +if sys.version_info[0] < 3: + from pysqlcipher import dbapi2 +else: + from pysqlcipher3 import dbapi2 + + +def sqlcipher_open(path, passphrase, create=True, document_factory=None): + return SQLCipherDatabase( + SQLCipherOptions(path, passphrase, create=create)) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_common_backend`. +# ----------------------------------------------------------------------------- + +class TestSQLCipherBackendImpl(tests.TestCase): + + def test__allocate_doc_id(self): + db = sqlcipher_open(':memory:', PASSWORD) + doc_id1 = db._allocate_doc_id() + self.assertTrue(doc_id1.startswith('D-')) + self.assertEqual(34, len(doc_id1)) + int(doc_id1[len('D-'):], 16) + self.assertNotEqual(doc_id1, db._allocate_doc_id()) + db.close() + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_backends`. +# ----------------------------------------------------------------------------- + +class SQLCipherTests(TestWithScenarios, test_backends.AllDatabaseTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherDatabaseTests(TestWithScenarios, + test_backends.LocalDatabaseTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherValidateGenNTransIdTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateGenNTransIdTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherValidateSourceGenTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateSourceGenTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherWithConflictsTests( + TestWithScenarios, + test_backends.LocalDatabaseWithConflictsTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherIndexTests( + TestWithScenarios, test_backends.DatabaseIndexTests): + scenarios = SQLCIPHER_SCENARIOS + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sqlite_backend`. +# ----------------------------------------------------------------------------- + +@pytest.mark.usefixtures('method_tmpdir') +class TestSQLCipherDatabase(tests.TestCase): + """ + Tests from u1db.tests.test_sqlite_backend.TestSQLiteDatabase. + """ + + def test_atomic_initialize(self): + # This test was modified to ensure that db2.close() is called within + # the thread that created the database. + dbname = os.path.join(self.tempdir, 'atomic.db') + + t2 = None # will be a thread + + class SQLCipherDatabaseTesting(SQLCipherDatabase): + _index_storage_value = "testing" + + def __init__(self, dbname, ntry): + self._try = ntry + self._is_initialized_invocations = 0 + SQLCipherDatabase.__init__( + self, + SQLCipherOptions(dbname, PASSWORD)) + + def _is_initialized(self, c): + res = \ + SQLCipherDatabase._is_initialized(self, c) + if self._try == 1: + self._is_initialized_invocations += 1 + if self._is_initialized_invocations == 2: + t2.start() + # hard to do better and have a generic test + time.sleep(0.05) + return res + + class SecondTry(threading.Thread): + + outcome2 = [] + + def run(self): + try: + db2 = SQLCipherDatabaseTesting(dbname, 2) + except Exception as e: + SecondTry.outcome2.append(e) + else: + SecondTry.outcome2.append(db2) + + t2 = SecondTry() + db1 = SQLCipherDatabaseTesting(dbname, 1) + t2.join() + + self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting) + self.assertTrue(db1._is_initialized(db1._get_sqlite_handle().cursor())) + db1.close() + + +@pytest.mark.usefixtures('method_tmpdir') +class TestSQLCipherPartialExpandDatabase(tests.TestCase): + """ + Tests from u1db.tests.test_sqlite_backend.TestSQLitePartialExpandDatabase. + """ + + # The following tests had to be cloned from u1db because they all + # instantiate the backend directly, so we need to change that in order to + # our backend be instantiated in place. + + def setUp(self): + self.db = sqlcipher_open(':memory:', PASSWORD) + + def tearDown(self): + self.db.close() + + def test_default_replica_uid(self): + self.assertIsNot(None, self.db._replica_uid) + self.assertEqual(32, len(self.db._replica_uid)) + int(self.db._replica_uid, 16) + + def test__parse_index(self): + g = self.db._parse_index_definition('fieldname') + self.assertIsInstance(g, query_parser.ExtractField) + self.assertEqual(['fieldname'], g.field) + + def test__update_indexes(self): + g = self.db._parse_index_definition('fieldname') + c = self.db._get_sqlite_handle().cursor() + self.db._update_indexes('doc-id', {'fieldname': 'val'}, + [('fieldname', g)], c) + c.execute('SELECT doc_id, field_name, value FROM document_fields') + self.assertEqual([('doc-id', 'fieldname', 'val')], + c.fetchall()) + + def test_create_database(self): + raw_db = self.db._get_sqlite_handle() + self.assertNotEqual(None, raw_db) + + def test__set_replica_uid(self): + # Start from scratch, so that replica_uid isn't set. + self.assertIsNot(None, self.db._real_replica_uid) + self.assertIsNot(None, self.db._replica_uid) + self.db._set_replica_uid('foo') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT value FROM u1db_config WHERE name='replica_uid'") + self.assertEqual(('foo',), c.fetchone()) + self.assertEqual('foo', self.db._real_replica_uid) + self.assertEqual('foo', self.db._replica_uid) + self.db._close_sqlite_handle() + self.assertEqual('foo', self.db._replica_uid) + + def test__open_database(self): + # SQLCipherDatabase has no _open_database() method, so we just pass + # (and test for the same funcionality on test_open_database_existing() + # below). + pass + + def test__open_database_with_factory(self): + # SQLCipherDatabase has no _open_database() method. + pass + + def test__open_database_non_existent(self): + path = self.tempdir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, + path, PASSWORD, create=False) + + def test__open_database_during_init(self): + # The purpose of this test is to ensure that _open_database() parallel + # db initialization behaviour is correct. As SQLCipherDatabase does + # not have an _open_database() method, we just do not implement this + # test. + pass + + def test__open_database_invalid(self): + # This test was modified to ensure that an empty database file will + # raise a DatabaseIsNotEncrypted exception instead of a + # dbapi2.OperationalError exception. + path1 = self.tempdir + '/invalid1.db' + with open(path1, 'wb') as f: + f.write("") + self.assertRaises(DatabaseIsNotEncrypted, + sqlcipher_open, path1, + PASSWORD) + with open(path1, 'wb') as f: + f.write("invalid") + self.assertRaises(dbapi2.DatabaseError, + sqlcipher_open, path1, + PASSWORD) + + def test_open_database_existing(self): + # In the context of SQLCipherDatabase, where no _open_database() + # method exists and thus there's no call to _which_index_storage(), + # this test tests for the same functionality as + # test_open_database_create() below. So, we just pass. + pass + + def test_open_database_with_factory(self): + # SQLCipherDatabase's constructor has no factory parameter. + pass + + def test_open_database_create(self): + # SQLCipherDatabas has no open_database() method, so we just test for + # the actual database constructor effects. + path = self.tempdir + '/new.sqlite' + db1 = sqlcipher_open(path, PASSWORD, create=True) + db2 = sqlcipher_open(path, PASSWORD, create=False) + self.assertIsInstance(db2, SQLCipherDatabase) + db1.close() + db2.close() + + def test_create_database_initializes_schema(self): + # This test had to be cloned because our implementation of SQLCipher + # backend is referenced with an index_storage_value that includes the + # word "encrypted". See u1db's sqlite_backend and our + # sqlcipher_backend for reference. + raw_db = self.db._get_sqlite_handle() + c = raw_db.cursor() + c.execute("SELECT * FROM u1db_config") + config = dict([(r[0], r[1]) for r in c.fetchall()]) + replica_uid = self.db._replica_uid + self.assertEqual({'sql_schema': '0', 'replica_uid': replica_uid, + 'index_storage': 'expand referenced encrypted'}, + config) + + def test_store_syncable(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + # assert that docs are syncable by default + self.assertEqual(True, doc.syncable) + # assert that we can store syncable = False + doc.syncable = False + self.db.put_doc(doc) + self.assertEqual(False, self.db.get_doc(doc.doc_id).syncable) + # assert that we can store syncable = True + doc.syncable = True + self.db.put_doc(doc) + self.assertEqual(True, self.db.get_doc(doc.doc_id).syncable) + + def test__close_sqlite_handle(self): + raw_db = self.db._get_sqlite_handle() + self.db._close_sqlite_handle() + self.assertRaises(dbapi2.ProgrammingError, + raw_db.cursor) + + def test__get_generation(self): + self.assertEqual(0, self.db._get_generation()) + + def test__get_generation_info(self): + self.assertEqual((0, ''), self.db._get_generation_info()) + + def test_create_index(self): + self.db.create_index('test-idx', "key") + self.assertEqual([('test-idx', ["key"])], self.db.list_indexes()) + + def test_create_index_multiple_fields(self): + self.db.create_index('test-idx', "key", "key2") + self.assertEqual([('test-idx', ["key", "key2"])], + self.db.list_indexes()) + + def test__get_index_definition(self): + self.db.create_index('test-idx', "key", "key2") + # TODO: How would you test that an index is getting used for an SQL + # request? + self.assertEqual(["key", "key2"], + self.db._get_index_definition('test-idx')) + + def test_list_index_mixed(self): + # Make sure that we properly order the output + c = self.db._get_sqlite_handle().cursor() + # We intentionally insert the data in weird ordering, to make sure the + # query still gets it back correctly. + c.executemany("INSERT INTO index_definitions VALUES (?, ?, ?)", + [('idx-1', 0, 'key10'), + ('idx-2', 2, 'key22'), + ('idx-1', 1, 'key11'), + ('idx-2', 0, 'key20'), + ('idx-2', 1, 'key21')]) + self.assertEqual([('idx-1', ['key10', 'key11']), + ('idx-2', ['key20', 'key21', 'key22'])], + self.db.list_indexes()) + + def test_no_indexes_no_document_fields(self): + self.db.create_doc_from_json( + '{"key1": "val1", "key2": "val2"}') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([], c.fetchall()) + + def test_create_extracts_fields(self): + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + doc2 = self.db.create_doc_from_json('{"key1": "valx", "key2": "valy"}') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([], c.fetchall()) + self.db.create_index('test', 'key1', 'key2') + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual(sorted( + [(doc1.doc_id, "key1", "val1"), + (doc1.doc_id, "key2", "val2"), + (doc2.doc_id, "key1", "valx"), + (doc2.doc_id, "key2", "valy"), ]), sorted(c.fetchall())) + + def test_put_updates_fields(self): + self.db.create_index('test', 'key1', 'key2') + doc1 = self.db.create_doc_from_json( + '{"key1": "val1", "key2": "val2"}') + doc1.content = {"key1": "val1", "key2": "valy"} + self.db.put_doc(doc1) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, "key1", "val1"), + (doc1.doc_id, "key2", "valy"), ], c.fetchall()) + + def test_put_updates_nested_fields(self): + self.db.create_index('test', 'key', 'sub.doc') + doc1 = self.db.create_doc_from_json(tests.nested_doc) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, "key", "value"), + (doc1.doc_id, "sub.doc", "underneath"), ], + c.fetchall()) + + def test__ensure_schema_rollback(self): + path = self.tempdir + '/rollback.db' + + class SQLitePartialExpandDbTesting(SQLCipherDatabase): + + def _set_replica_uid_in_transaction(self, uid): + super(SQLitePartialExpandDbTesting, + self)._set_replica_uid_in_transaction(uid) + if fail: + raise Exception() + + db = SQLitePartialExpandDbTesting.__new__(SQLitePartialExpandDbTesting) + db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed + fail = True + self.assertRaises(Exception, db._ensure_schema) + fail = False + db._initialize(db._db_handle.cursor()) + + def test_open_database_non_existent(self): + path = self.tempdir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, path, "123", + create=False) + + def test_delete_database_existent(self): + path = self.tempdir + '/new.sqlite' + db = sqlcipher_open(path, "123", create=True) + db.close() + SQLCipherDatabase.delete_database(path) + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, path, "123", + create=False) + + def test_delete_database_nonexistent(self): + path = self.tempdir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + SQLCipherDatabase.delete_database, path) + + def test__get_indexed_fields(self): + self.db.create_index('idx1', 'a', 'b') + self.assertEqual(set(['a', 'b']), self.db._get_indexed_fields()) + self.db.create_index('idx2', 'b', 'c') + self.assertEqual(set(['a', 'b', 'c']), self.db._get_indexed_fields()) + + def test_indexed_fields_expanded(self): + self.db.create_index('idx1', 'key1') + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + self.assertEqual(set(['key1']), self.db._get_indexed_fields()) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) + + def test_create_index_updates_fields(self): + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + self.db.create_index('idx1', 'key1') + self.assertEqual(set(['key1']), self.db._get_indexed_fields()) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) + + def assertFormatQueryEquals(self, exp_statement, exp_args, definition, + values): + statement, args = self.db._format_query(definition, values) + self.assertEqual(exp_statement, statement) + self.assertEqual(exp_args, args) + + def test__format_query(self): + self.assertFormatQueryEquals( + "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " + "document d, document_fields d0 LEFT OUTER JOIN conflicts c ON " + "c.doc_id = d.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name " + "= ? AND d0.value = ? GROUP BY d.doc_id, d.doc_rev, d.content " + "ORDER BY d0.value;", ["key1", "a"], + ["key1"], ["a"]) + + def test__format_query2(self): + self.assertFormatQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value = ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value = ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ["key1", "a", "key2", "b", "key3", "c"], + ["key1", "key2", "key3"], ["a", "b", "c"]) + + def test__format_query_wildcard(self): + self.assertFormatQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value GLOB ? AND d.doc_id = d2.doc_id AND d2.field_name = ? ' + 'AND d2.value NOT NULL GROUP BY d.doc_id, d.doc_rev, d.content ' + 'ORDER BY d0.value, d1.value, d2.value;', + ["key1", "a", "key2", "b*", "key3"], ["key1", "key2", "key3"], + ["a", "b*", "*"]) + + def assertFormatRangeQueryEquals(self, exp_statement, exp_args, definition, + start_value, end_value): + statement, args = self.db._format_range_query( + definition, start_value, end_value) + self.assertEqual(exp_statement, statement) + self.assertEqual(exp_args, args) + + def test__format_range_query(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value >= ? AND d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c', 'key1', 'p', 'key2', 'q', + 'key3', 'r'], + ["key1", "key2", "key3"], ["a", "b", "c"], ["p", "q", "r"]) + + def test__format_range_query_no_start(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c'], + ["key1", "key2", "key3"], None, ["a", "b", "c"]) + + def test__format_range_query_no_end(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value >= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c'], + ["key1", "key2", "key3"], ["a", "b", "c"], None) + + def test__format_range_query_wildcard(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value NOT NULL AND d.doc_id = d0.doc_id AND d0.field_name = ? ' + 'AND d0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? ' + 'AND (d1.value < ? OR d1.value GLOB ?) AND d.doc_id = d2.doc_id ' + 'AND d2.field_name = ? AND d2.value NOT NULL GROUP BY d.doc_id, ' + 'd.doc_rev, d.content ORDER BY d0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'key1', 'p', 'key2', 'q', 'q*', + 'key3'], + ["key1", "key2", "key3"], ["a", "b*", "*"], ["p", "q*", "*"]) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_open`. +# ----------------------------------------------------------------------------- + + +class SQLCipherOpen(test_open.TestU1DBOpen): + + def test_open_no_create(self): + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, self.db_path, + PASSWORD, + create=False) + self.assertFalse(os.path.exists(self.db_path)) + + def test_open_create(self): + db = sqlcipher_open(self.db_path, PASSWORD, create=True) + self.addCleanup(db.close) + self.assertTrue(os.path.exists(self.db_path)) + self.assertIsInstance(db, SQLCipherDatabase) + + def test_open_with_factory(self): + db = sqlcipher_open(self.db_path, PASSWORD, create=True, + document_factory=SoledadDocument) + self.addCleanup(db.close) + doc = db.create_doc({}) + self.assertTrue(isinstance(doc, SoledadDocument)) + + def test_open_existing(self): + db = sqlcipher_open(self.db_path, PASSWORD) + self.addCleanup(db.close) + doc = db.create_doc_from_json(tests.simple_doc) + # Even though create=True, we shouldn't wipe the db + db2 = sqlcipher_open(self.db_path, PASSWORD, create=True) + self.addCleanup(db2.close) + doc2 = db2.get_doc(doc.doc_id) + self.assertEqual(doc, doc2) + + def test_open_existing_no_create(self): + db = sqlcipher_open(self.db_path, PASSWORD) + self.addCleanup(db.close) + db2 = sqlcipher_open(self.db_path, PASSWORD, create=False) + self.addCleanup(db2.close) + self.assertIsInstance(db2, SQLCipherDatabase) + + +# ----------------------------------------------------------------------------- +# Tests for actual encryption of the database +# ----------------------------------------------------------------------------- + +class SQLCipherEncryptionTests(BaseSoledadTest): + + """ + Tests to guarantee SQLCipher is indeed encrypting data when storing. + """ + + def _delete_dbfiles(self): + for dbfile in [self.DB_FILE]: + if os.path.exists(dbfile): + os.unlink(dbfile) + + def setUp(self): + BaseSoledadTest.setUp(self) + self.DB_FILE = os.path.join(self.tempdir, 'test.db') + self._delete_dbfiles() + + def tearDown(self): + self._delete_dbfiles() + BaseSoledadTest.tearDown(self) + + def test_try_to_open_encrypted_db_with_sqlite_backend(self): + """ + SQLite backend should not succeed to open SQLCipher databases. + """ + db = sqlcipher_open(self.DB_FILE, PASSWORD) + doc = db.create_doc_from_json(tests.simple_doc) + db.close() + try: + # trying to open an encrypted database with the regular u1db + # backend should raise a DatabaseError exception. + SQLitePartialExpandDatabase(self.DB_FILE, + document_factory=SoledadDocument) + raise DatabaseIsNotEncrypted() + except dbapi2.DatabaseError: + # at this point we know that the regular U1DB sqlcipher backend + # did not succeed on opening the database, so it was indeed + # encrypted. + db = sqlcipher_open(self.DB_FILE, PASSWORD) + doc = db.get_doc(doc.doc_id) + self.assertEqual(tests.simple_doc, doc.get_json(), + 'decrypted content mismatch') + db.close() + + def test_try_to_open_raw_db_with_sqlcipher_backend(self): + """ + SQLCipher backend should not succeed to open unencrypted databases. + """ + db = SQLitePartialExpandDatabase(self.DB_FILE, + document_factory=SoledadDocument) + db.create_doc_from_json(tests.simple_doc) + db.close() + try: + # trying to open the a non-encrypted database with sqlcipher + # backend should raise a DatabaseIsNotEncrypted exception. + db = sqlcipher_open(self.DB_FILE, PASSWORD) + db.close() + raise dbapi2.DatabaseError( + "SQLCipher backend should not be able to open non-encrypted " + "dbs.") + except DatabaseIsNotEncrypted: + pass diff --git a/tests/sqlcipher/test_soledad b/tests/sqlcipher/test_soledad new file mode 120000 index 00000000..c1a35d32 --- /dev/null +++ b/tests/sqlcipher/test_soledad @@ -0,0 +1 @@ +../test_soledad \ No newline at end of file diff --git a/tests/sync/__init__.py b/tests/sync/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/sync/test_soledad b/tests/sync/test_soledad new file mode 120000 index 00000000..c1a35d32 --- /dev/null +++ b/tests/sync/test_soledad @@ -0,0 +1 @@ +../test_soledad \ No newline at end of file diff --git a/tests/sync/test_sqlcipher_sync.py b/tests/sync/test_sqlcipher_sync.py new file mode 100644 index 00000000..26f63a40 --- /dev/null +++ b/tests/sync/test_sqlcipher_sync.py @@ -0,0 +1,719 @@ +# -*- coding: utf-8 -*- +# test_sqlcipher.py +# Copyright (C) 2013-2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test sqlcipher backend sync. +""" +import os + +from uuid import uuid4 + +from testscenarios import TestWithScenarios + +from leap.soledad.common.l2db import sync +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db import errors + +from leap.soledad.client.http_target import SoledadHTTPSyncTarget + +from test_soledad import u1db_tests as tests +from test_soledad.util import SQLCIPHER_SCENARIOS +from test_soledad.util import make_soledad_app +from test_soledad.util import soledad_sync_target +from test_soledad.util import BaseSoledadTest + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sync`. +# ----------------------------------------------------------------------------- + +def sync_via_synchronizer_and_soledad(test, db_source, db_target, + trace_hook=None, + trace_hook_shallow=None): + if trace_hook: + test.skipTest("full trace hook unsupported over http") + path = test._http_at[db_target] + target = SoledadHTTPSyncTarget.connect( + test.getURL(path), test._soledad._crypto) + target.set_token_credentials('user-uuid', 'auth-token') + if trace_hook_shallow: + target._set_trace_hook_shallow(trace_hook_shallow) + return sync.Synchronizer(db_source, target).sync() + + +def sync_via_synchronizer(test, db_source, db_target, + trace_hook=None, + trace_hook_shallow=None): + target = db_target.get_sync_target() + trace_hook = trace_hook or trace_hook_shallow + if trace_hook: + target._set_trace_hook(trace_hook) + return sync.Synchronizer(db_source, target).sync() + + +sync_scenarios = [] +for name, scenario in SQLCIPHER_SCENARIOS: + scenario['do_sync'] = sync_via_synchronizer + sync_scenarios.append((name, scenario)) + + +class SQLCipherDatabaseSyncTests( + TestWithScenarios, + tests.DatabaseBaseTests, + BaseSoledadTest): + + """ + Test for succesfull sync between SQLCipher and LeapBackend. + + Some of the tests in this class had to be adapted because the remote + backend always receive encrypted content, and so it can not rely on + document's content comparison to try to autoresolve conflicts. + """ + + scenarios = sync_scenarios + + def setUp(self): + self._use_tracking = {} + super(tests.DatabaseBaseTests, self).setUp() + + def create_database(self, replica_uid, sync_role=None): + if replica_uid == 'test' and sync_role is None: + # created up the chain by base class but unused + return None + db = self.create_database_for_role(replica_uid, sync_role) + if sync_role: + self._use_tracking[db] = (replica_uid, sync_role) + self.addCleanup(db.close) + return db + + def create_database_for_role(self, replica_uid, sync_role): + # hook point for reuse + return tests.DatabaseBaseTests.create_database(self, replica_uid) + + def sync(self, db_from, db_to, trace_hook=None, + trace_hook_shallow=None): + from_name, from_sync_role = self._use_tracking[db_from] + to_name, to_sync_role = self._use_tracking[db_to] + if from_sync_role not in ('source', 'both'): + raise Exception("%s marked for %s use but used as source" % + (from_name, from_sync_role)) + if to_sync_role not in ('target', 'both'): + raise Exception("%s marked for %s use but used as target" % + (to_name, to_sync_role)) + return self.do_sync(self, db_from, db_to, trace_hook, + trace_hook_shallow) + + def assertLastExchangeLog(self, db, expected): + log = getattr(db, '_last_exchange_log', None) + if log is None: + return + self.assertEqual(expected, log) + + def copy_database(self, db, sync_role=None): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + db_copy = tests.DatabaseBaseTests.copy_database(self, db) + name, orig_sync_role = self._use_tracking[db] + self._use_tracking[db_copy] = (name + '(copy)', sync_role or + orig_sync_role) + return db_copy + + def test_sync_tracks_db_generation_of_other(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertEqual( + (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [], 'last_known_gen': 0}, + 'return': + {'docs': [], 'last_gen': 0}}) + + def test_sync_autoresolves(self): + """ + Test for sync autoresolve remote. + + This test was adapted because the remote database receives encrypted + content and so it can't compare documents contents to autoresolve. + """ + # The remote database can't autoresolve conflicts based on magic + # content convergence, so we modify this test to leave the possibility + # of the remode document ending up in conflicted state. + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev1 = doc1.rev + doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev2 = doc2.rev + self.sync(self.db1, self.db2) + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + # if remote content is in conflicted state, then document revisions + # will be different. + # self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) + v = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) + + def test_sync_autoresolves_moar(self): + """ + Test for sync autoresolve local. + + This test was adapted to decrypt remote content before assert. + """ + # here we test that when a database that has a conflicted document is + # the source of a sync, and the target database has a revision of the + # conflicted document that is newer than the source database's, and + # that target's database's document's content is the same as the + # source's document's conflict's, the source's document's conflict gets + # autoresolved, and the source's document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # `-------> + # a3b2 a1b2 (autoresolved) + # `-------> + # a3b2 a3b2 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + # because the conflict is on the source, sync it another time + self.sync(self.db1, self.db2) + # make sure db2 now has the exact same thing + doc1 = self.db1.get_doc('doc') + self.assertGetEncryptedDoc( + self.db2, + doc1.doc_id, doc1.rev, doc1.get_json(), False) + + def test_sync_autoresolves_moar_backwards(self): + # here we would test that when a database that has a conflicted + # document is the target of a sync, and the source database has a + # revision of the conflicted document that is newer than the target + # database's, and that source's database's document's content is the + # same as the target's document's conflict's, the target's document's + # conflict gets autoresolved, and the document's revision bumped. + # + # Despite that, in Soledad we suppose that the server never syncs, so + # it never has conflicted documents. Also, if it had, convergence + # would not be possible by checking document's contents because they + # would be encrypted in server. + # + # Therefore we suppress this test. + pass + + def test_sync_autoresolves_moar_backwards_three(self): + # here we would test that when a database that has a conflicted + # document is the target of a sync, and the source database has a + # revision of the conflicted document that is newer than the target + # database's, and that source's database's document's content is the + # same as the target's document's conflict's, the target's document's + # conflict gets autoresolved, and the document's revision bumped. + # + # We use the same reasoning from the last test to suppress this one. + pass + + def test_sync_pulling_doesnt_update_other_if_changed(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(tests.simple_doc) + # After the local side has sent its list of docs, before we start + # receiving the "targets" response, we update the local database with a + # new record. + # When we finish synchronizing, we can notice that something locally + # was updated, and we cannot tell c2 our new updated generation + + def before_get_docs(state): + if state != 'before get_docs': + return + self.db1.create_doc_from_json(tests.simple_doc) + + self.assertEqual(0, self.sync(self.db1, self.db2, + trace_hook=before_get_docs)) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [], 'last_known_gen': 0}, + 'return': + {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + # c2 should not have gotten a '_record_sync_info' call, because the + # local database had been updated more than just by the messages + # returned from c2. + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + + def test_sync_doesnt_update_other_if_nothing_pulled(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc) + + def no_record_sync_info(state): + if state != 'record_sync_info': + return + self.fail('SyncTarget.record_sync_info was called') + self.assertEqual(1, self.sync(self.db1, self.db2, + trace_hook_shallow=no_record_sync_info)) + self.assertEqual( + 1, + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) + + def test_sync_ignores_convergence(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(tests.simple_doc) + self.db3 = self.create_database('test3', 'target') + self.assertEqual(1, self.sync(self.db1, self.db3)) + self.assertEqual(0, self.sync(self.db2, self.db3)) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_ignores_superseded(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(tests.simple_doc) + doc_rev1 = doc.rev + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.sync(self.db2, self.db3) + new_content = '{"key": "altval"}' + doc.set_json(new_content) + self.db1.put_doc(doc) + doc_rev2 = doc.rev + self.sync(self.db2, self.db1) + self.assertLastExchangeLog(self.db1, + {'receive': + {'docs': [(doc.doc_id, doc_rev1)], + 'source_uid': 'test2', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': + {'docs': [(doc.doc_id, doc_rev2)], + 'last_gen': 2}}) + self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) + + def test_sync_sees_remote_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc1.doc_id + doc1_rev = doc1.rev + self.db1.create_index('test-idx', 'key') + new_doc = '{"key": "altval"}' + doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) + doc2_rev = doc2.rev + self.assertTransactionLog([doc1.doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc_id, doc1_rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': + {'docs': [(doc_id, doc2_rev)], + 'last_gen': 1}}) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) + self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) + from_idx = self.db1.get_from_index('test-idx', 'altval')[0] + self.assertEqual(doc2.doc_id, from_idx.doc_id) + self.assertEqual(doc2.rev, from_idx.rev) + self.assertTrue(from_idx.has_conflicts) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_sees_remote_delete_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc1.doc_id + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) + new_doc = '{"key": "altval"}' + doc1.set_json(new_doc) + self.db1.put_doc(doc1) + self.db2.delete_doc(doc2) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc_id, doc1.rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [(doc_id, doc2.rev)], + 'last_gen': 2}}) + self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) + self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, doc2.rev, None, False) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_local_race_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc.doc_id + doc1_rev = doc.rev + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + content1 = '{"key": "localval"}' + content2 = '{"key": "altval"}' + doc.set_json(content2) + self.db2.put_doc(doc) + doc2_rev2 = doc.rev + triggered = [] + + def after_whatschanged(state): + if state != 'after whats_changed': + return + triggered.append(True) + doc = self.make_document(doc_id, doc1_rev, content1) + self.db1.put_doc(doc) + + self.sync(self.db1, self.db2, trace_hook=after_whatschanged) + self.assertEqual([True], triggered) + self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) + from_idx = self.db1.get_from_index('test-idx', 'altval')[0] + self.assertEqual(doc.doc_id, from_idx.doc_id) + self.assertEqual(doc.rev, from_idx.rev) + self.assertTrue(from_idx.has_conflicts) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + self.assertEqual([], self.db1.get_from_index('test-idx', 'localval')) + + def test_sync_propagates_deletes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc1.doc_id + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + self.db2.create_index('test-idx', 'key') + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.db1.delete_doc(doc1) + deleted_rev = doc1.rev + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db1, doc_id, deleted_rev, None, False) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, deleted_rev, None, False) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + self.assertEqual([], self.db2.get_from_index('test-idx', 'value')) + self.sync(self.db2, self.db3) + self.assertLastExchangeLog(self.db3, + {'receive': + {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test2', + 'source_gen': 2, + 'last_known_gen': 0}, + 'return': + {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db3, doc_id, deleted_rev, None, False) + + def test_sync_propagates_deletes_2(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') + self.sync(self.db1, self.db2) + doc1_2 = self.db2.get_doc('the-doc') + self.db2.delete_doc(doc1_2) + self.sync(self.db1, self.db2) + self.assertGetDocIncludeDeleted( + self.db1, 'the-doc', doc1_2.rev, None, False) + + def test_sync_detects_identical_replica_uid(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test1', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.assertRaises( + errors.InvalidReplicaUID, self.sync, self.db1, self.db2) + + def test_optional_sync_preserve_json(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + cont1 = '{ "a": 2 }' + cont2 = '{ "b":3}' + self.db1.create_doc_from_json(cont1, doc_id="1") + self.db2.create_doc_from_json(cont2, doc_id="2") + self.sync(self.db1, self.db2) + self.assertEqual(cont1, self.db2.get_doc("1").get_json()) + self.assertEqual(cont2, self.db1.get_doc("2").get_json()) + + def test_sync_propagates_resolution(self): + """ + Test if synchronization propagates resolution. + + This test was adapted to decrypt remote content before assert. + """ + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + db3 = self.create_database('test3', 'both') + self.sync(self.db2, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db2._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.sync(db3, self.db1) + # update on 2 + doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') + self.db2.put_doc(doc2) + self.sync(self.db2, db3) + self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) + # update on 1 + doc1.set_json('{"a": 3}') + self.db1.put_doc(doc1) + # conflicts + self.sync(self.db2, self.db1) + self.sync(db3, self.db1) + self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) + self.assertTrue(db3.get_doc('the-doc').has_conflicts) + # resolve + conflicts = self.db2.get_doc_conflicts('the-doc') + doc4 = self.make_document('the-doc', None, '{"a": 4}') + revs = [doc.rev for doc in conflicts] + self.db2.resolve_doc(doc4, revs) + doc2 = self.db2.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc2.get_json()) + self.assertFalse(doc2.has_conflicts) + self.sync(self.db2, db3) + doc3 = db3.get_doc('the-doc') + + self.assertEqual(doc4.get_json(), doc3.get_json()) + self.assertFalse(doc3.has_conflicts) + self.db1.close() + self.db2.close() + db3.close() + + def test_sync_puts_changes(self): + """ + Test if sync puts changes in remote replica. + + This test was adapted to decrypt remote content before assert. + """ + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(tests.simple_doc) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertGetEncryptedDoc( + self.db2, doc.doc_id, doc.rev, tests.simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_pulls_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(tests.simple_doc) + self.db1.create_index('test-idx', 'key') + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertGetDoc(self.db1, doc.doc_id, doc.rev, + tests.simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [], 'last_known_gen': 0}, + 'return': + {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertEqual([doc], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_supersedes_conflicts(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.create_database('test3', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') + self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') + self.sync(self.db3, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.sync(self.db3, self.db2) + self.assertEqual( + self.db2._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + doc1.set_json('{"a": 2}') + self.db1.put_doc(doc1) + self.sync(self.db3, self.db1) + # original doc1 should have been removed from conflicts + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + + def test_sync_stops_after_get_sync_info(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc) + self.sync(self.db1, self.db2) + + def put_hook(state): + self.fail("Tracehook triggered for %s" % (state,)) + + self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) + + def test_sync_detects_rollback_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) + + def test_sync_detects_rollback_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) + + def test_sync_detects_diverged_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db3, self.db2) + + def test_sync_detects_diverged_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db2) + self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db1, self.db3) + + def test_sync_detects_rollback_and_divergence_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db1_copy, self.db2) + + def test_sync_detects_rollback_and_divergence_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db1, self.db2_copy) + + +def make_local_db_and_soledad_target( + test, path='test', + source_replica_uid=uuid4().hex): + test.startTwistedServer() + replica_uid = os.path.basename(path) + db = test.request_state._create_database(replica_uid) + st = soledad_sync_target( + test, db._dbname, + source_replica_uid=source_replica_uid) + return db, st + + +target_scenarios = [ + ('leap', { + 'create_db_and_target': make_local_db_and_soledad_target, + 'make_app_with_state': make_soledad_app, + 'do_sync': sync_via_synchronizer_and_soledad}), +] diff --git a/tests/sync/test_sync.py b/tests/sync/test_sync.py new file mode 100644 index 00000000..fb9a0245 --- /dev/null +++ b/tests/sync/test_sync.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- +# test_sync.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import json +import pytest +import threading +import time + +from six.moves.urllib.parse import urljoin +from mock import Mock +from twisted.internet import defer + +from testscenarios import TestWithScenarios + +from leap.soledad.common import couch +from leap.soledad.client import sync + +from test_soledad import u1db_tests as tests +from test_soledad.u1db_tests import TestCaseWithServer +from test_soledad.u1db_tests import simple_doc +from test_soledad.util import make_token_soledad_app +from test_soledad.util import make_soledad_document_for_test +from test_soledad.util import soledad_sync_target +from test_soledad.util import BaseSoledadTest +from test_soledad.util import SoledadWithCouchServerMixin +from test_soledad.util import CouchDBTestCase + + +class InterruptableSyncTestCase( + BaseSoledadTest, CouchDBTestCase, TestCaseWithServer): + + """ + Tests for encrypted sync using Soledad server backed by a couch database. + """ + + @staticmethod + def make_app_with_state(state): + return make_token_soledad_app(state) + + make_document_for_test = make_soledad_document_for_test + + sync_target = soledad_sync_target + + def make_app(self): + self.request_state = couch.CouchServerState(self.couch_url) + return self.make_app_with_state(self.request_state) + + def setUp(self): + TestCaseWithServer.setUp(self) + CouchDBTestCase.setUp(self) + + def tearDown(self): + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) + + def test_interruptable_sync(self): + """ + Test if Soledad can sync many smallfiles. + """ + + self.skipTest("Sync is currently not interruptable.") + + class _SyncInterruptor(threading.Thread): + + """ + A thread meant to interrupt the sync process. + """ + + def __init__(self, soledad, couchdb): + self._soledad = soledad + self._couchdb = couchdb + threading.Thread.__init__(self) + + def run(self): + while db._get_generation() < 2: + # print "WAITING %d" % db._get_generation() + time.sleep(0.1) + self._soledad.stop_sync() + time.sleep(1) + + number_of_docs = 10 + self.startServer() + + # instantiate soledad and create a document + sol = self._soledad_instance( + user='user-uuid', server_url=self.getURL()) + + # ensure remote db exists before syncing + db = couch.CouchDatabase.open_database( + urljoin(self.couch_url, 'user-user-uuid'), + create=True) + + # create interruptor thread + t = _SyncInterruptor(sol, db) + t.start() + + d = sol.get_all_docs() + d.addCallback(lambda results: self.assertEqual([], results[1])) + + def _create_docs(results): + # create many small files + deferreds = [] + for i in range(0, number_of_docs): + deferreds.append(sol.create_doc(json.loads(simple_doc))) + return defer.DeferredList(deferreds) + + # sync with server + d.addCallback(_create_docs) + d.addCallback(lambda _: sol.get_all_docs()) + d.addCallback( + lambda results: self.assertEqual(number_of_docs, len(results[1]))) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: t.join()) + d.addCallback(lambda _: db.get_all_docs()) + d.addCallback( + lambda results: self.assertNotEqual( + number_of_docs, len(results[1]))) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: db.get_all_docs()) + d.addCallback( + lambda results: self.assertEqual(number_of_docs, len(results[1]))) + + def _tear_down(results): + db.delete_database() + db.close() + sol.close() + + d.addCallback(_tear_down) + return d + + +@pytest.mark.needs_couch +class TestSoledadDbSync( + TestWithScenarios, + SoledadWithCouchServerMixin, + tests.TestCaseWithServer): + + """ + Test db.sync remote sync shortcut + """ + + scenarios = [ + ('py-token-http', { + 'make_app_with_state': make_token_soledad_app, + 'make_database_for_test': tests.make_memory_database_for_test, + 'token': True + }), + ] + + oauth = False + token = False + + def setUp(self): + """ + Need to explicitely invoke inicialization on all bases. + """ + SoledadWithCouchServerMixin.setUp(self) + self.startTwistedServer() + self.db = self.make_database_for_test(self, 'test1') + self.db2 = self.request_state._create_database(replica_uid='test') + + def tearDown(self): + """ + Need to explicitely invoke destruction on all bases. + """ + SoledadWithCouchServerMixin.tearDown(self) + # tests.TestCaseWithServer.tearDown(self) + + def do_sync(self): + """ + Perform sync using SoledadSynchronizer, SoledadSyncTarget + and Token auth. + """ + target = soledad_sync_target( + self, self.db2._dbname, + source_replica_uid=self._soledad._dbpool.replica_uid) + return sync.SoledadSynchronizer( + self.db, + target).sync() + + @defer.inlineCallbacks + def test_db_sync(self): + """ + Test sync. + + Adapted to check for encrypted content. + """ + + doc1 = self.db.create_doc_from_json(tests.simple_doc) + doc2 = self.db2.create_doc_from_json(tests.nested_doc) + + local_gen_before_sync = yield self.do_sync() + gen, _, changes = self.db.whats_changed(local_gen_before_sync) + self.assertEqual(1, len(changes)) + self.assertEqual(doc2.doc_id, changes[0][0]) + self.assertEqual(1, gen - local_gen_before_sync) + self.assertGetEncryptedDoc( + self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False) + + # TODO: add u1db.tests.test_sync.TestRemoteSyncIntegration + + +class TestSoledadSynchronizer(BaseSoledadTest): + + def setUp(self): + BaseSoledadTest.setUp(self) + self.db = Mock() + self.target = Mock() + self.synchronizer = sync.SoledadSynchronizer( + self.db, + self.target) + + def test_docs_by_gen_includes_deleted(self): + changes = [('id', 'gen', 'trans')] + docs_by_gen = self.synchronizer._docs_by_gen_from_changes(changes) + f, args, kwargs = docs_by_gen[0][0] + self.assertIn('include_deleted', kwargs) + self.assertTrue(kwargs['include_deleted']) diff --git a/tests/sync/test_sync_mutex.py b/tests/sync/test_sync_mutex.py new file mode 100644 index 00000000..fdd2aacd --- /dev/null +++ b/tests/sync/test_sync_mutex.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +# test_sync_mutex.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Test that synchronization is a critical section and, as such, there might not +be two concurrent synchronization processes at the same time. +""" + + +import pytest +import time +import uuid + +from six.moves.urllib.parse import urljoin + +from twisted.internet import defer + +from leap.soledad.client.sync import SoledadSynchronizer + +from leap.soledad.common.couch.state import CouchServerState +from leap.soledad.common.couch import CouchDatabase +from test_soledad.u1db_tests import TestCaseWithServer + +from test_soledad.util import CouchDBTestCase +from test_soledad.util import BaseSoledadTest +from test_soledad.util import make_token_soledad_app +from test_soledad.util import make_soledad_document_for_test +from test_soledad.util import soledad_sync_target + + +# monkey-patch the soledad synchronizer so it stores start and finish times + +_old_sync = SoledadSynchronizer.sync + + +def _timed_sync(self): + t = time.time() + + sync_id = uuid.uuid4() + + if not getattr(self.source, 'sync_times', False): + self.source.sync_times = {} + + self.source.sync_times[sync_id] = {'start': t} + + def _store_finish_time(passthrough): + t = time.time() + self.source.sync_times[sync_id]['end'] = t + return passthrough + + d = _old_sync(self) + d.addBoth(_store_finish_time) + return d + + +SoledadSynchronizer.sync = _timed_sync + +# -- end of monkey-patching + + +@pytest.mark.needs_couch +class TestSyncMutex( + BaseSoledadTest, CouchDBTestCase, TestCaseWithServer): + + @staticmethod + def make_app_with_state(state): + return make_token_soledad_app(state) + + make_document_for_test = make_soledad_document_for_test + + sync_target = soledad_sync_target + + def make_app(self): + self.request_state = CouchServerState(self.couch_url) + return self.make_app_with_state(self.request_state) + + def setUp(self): + TestCaseWithServer.setUp(self) + CouchDBTestCase.setUp(self) + self.user = ('user-%s' % uuid.uuid4().hex) + + def tearDown(self): + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) + + def test_two_concurrent_syncs_do_not_overlap_no_docs(self): + self.startServer() + + # ensure remote db exists before syncing + db = CouchDatabase.open_database( + urljoin(self.couch_url, 'user-' + self.user), + create=True) + + sol = self._soledad_instance( + user=self.user, server_url=self.getURL()) + + d1 = sol.sync() + d2 = sol.sync() + + def _assert_syncs_do_not_overlap(thearg): + # recover sync times + sync_times = [] + for key in sol._dbsyncer.sync_times: + sync_times.append(sol._dbsyncer.sync_times[key]) + sync_times.sort(key=lambda s: s['start']) + + self.assertTrue( + (sync_times[0]['start'] < sync_times[0]['end'] and + sync_times[0]['end'] < sync_times[1]['start'] and + sync_times[1]['start'] < sync_times[1]['end'])) + + db.delete_database() + db.close() + sol.close() + + d = defer.gatherResults([d1, d2]) + d.addBoth(_assert_syncs_do_not_overlap) + return d diff --git a/tests/sync/test_sync_target.py b/tests/sync/test_sync_target.py new file mode 100644 index 00000000..712f0d3f --- /dev/null +++ b/tests/sync/test_sync_target.py @@ -0,0 +1,968 @@ +# -*- coding: utf-8 -*- +# test_sync_target.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: sync target +""" +import os +import time +import json +import pytest +import random +import string +import shutil + +from six import StringIO as cStringIO +from uuid import uuid4 + +from testscenarios import TestWithScenarios +from twisted.internet import defer + +from leap.soledad.client import http_target as target +from leap.soledad.client.http_target.fetch_protocol import DocStreamReceiver +from leap.soledad.client._db.sqlcipher import SQLCipherU1DBSync +from leap.soledad.client._db.sqlcipher import SQLCipherOptions +from leap.soledad.client._db.sqlcipher import SQLCipherDatabase +from leap.soledad.client import _crypto + +from leap.soledad.common import l2db + +from leap.soledad.common.document import SoledadDocument +from test_soledad import u1db_tests as tests +from test_soledad.util import make_sqlcipher_database_for_test +from test_soledad.util import make_soledad_app +from test_soledad.util import make_token_soledad_app +from test_soledad.util import make_soledad_document_for_test +from test_soledad.util import soledad_sync_target +from twisted.trial import unittest +from test_soledad.util import SoledadWithCouchServerMixin +from test_soledad.util import ADDRESS +from test_soledad.util import SQLCIPHER_SCENARIOS + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_remote_sync_target`. +# ----------------------------------------------------------------------------- + +class TestSoledadParseReceivedDocResponse(unittest.TestCase): + + """ + Some tests had to be copied to this class so we can instantiate our own + target. + """ + + def parse(self, stream): + parser = DocStreamReceiver(None, defer.Deferred(), + lambda *_: defer.succeed(42)) + parser.dataReceived(stream) + parser.finish() + + def test_extra_comma(self): + doc = SoledadDocument('i', rev='r') + doc.content = {'a': 'b'} + + encrypted_docstr = _crypto.SoledadCrypto('safe').encrypt_doc(doc) + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.parse("[\r\n{},\r\n]") + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.parse( + ('[\r\n{},\r\n{"id": "i", "rev": "r", ' + + '"gen": 3, "trans_id": "T-sid"},\r\n' + + '%s,\r\n]') % encrypted_docstr) + + def test_wrong_start(self): + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.parse("{}\r\n]") + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.parse("\r\n{}\r\n]") + + def test_wrong_end(self): + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.parse("[\r\n{}") + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.parse("[\r\n") + + def test_missing_comma(self): + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.parse( + '[\r\n{}\r\n{"id": "i", "rev": "r", ' + '"content": "c", "gen": 3}\r\n]') + + def test_no_entries(self): + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.parse("[\r\n]") + + def test_error_in_stream(self): + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.parse( + '[\r\n{"new_generation": 0},' + '\r\n{"error": "unavailable"}\r\n') + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.parse( + '[\r\n{"error": "unavailable"}\r\n') + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.parse('[\r\n{"error": "?"}\r\n') + +# +# functions for TestRemoteSyncTargets +# + + +def make_local_db_and_soledad_target( + test, path='test', + source_replica_uid=uuid4().hex): + test.startTwistedServer() + replica_uid = os.path.basename(path) + db = test.request_state._create_database(replica_uid) + st = soledad_sync_target( + test, db._dbname, + source_replica_uid=source_replica_uid) + return db, st + + +def make_local_db_and_token_soledad_target( + test, + source_replica_uid=uuid4().hex): + db, st = make_local_db_and_soledad_target( + test, path='test', + source_replica_uid=source_replica_uid) + st.set_token_credentials('user-uuid', 'auth-token') + return db, st + + +@pytest.mark.needs_couch +class TestSoledadSyncTarget( + TestWithScenarios, + SoledadWithCouchServerMixin, + tests.TestCaseWithServer): + + scenarios = [ + ('token_soledad', + {'make_app_with_state': make_token_soledad_app, + 'make_document_for_test': make_soledad_document_for_test, + 'create_db_and_target': make_local_db_and_token_soledad_target, + 'make_database_for_test': make_sqlcipher_database_for_test, + 'sync_target': soledad_sync_target}), + ] + + def getSyncTarget(self, path=None, source_replica_uid=uuid4().hex): + if self.port is None: + self.startTwistedServer() + if path is None: + path = self.db2._dbname + target = self.sync_target( + self, path, + source_replica_uid=source_replica_uid) + return target + + def setUp(self): + TestWithScenarios.setUp(self) + SoledadWithCouchServerMixin.setUp(self) + self.startTwistedServer() + self.db1 = make_sqlcipher_database_for_test(self, 'test1') + self.db2 = self.request_state._create_database('test') + + def tearDown(self): + # db2, _ = self.request_state.ensure_database('test2') + self.delete_db(self.db2._dbname) + self.db1.close() + SoledadWithCouchServerMixin.tearDown(self) + TestWithScenarios.tearDown(self) + + @defer.inlineCallbacks + def test_sync_exchange_send(self): + """ + Test for sync exchanging send of document. + + This test was adapted to decrypt remote content before assert. + """ + db = self.db2 + remote_target = self.getSyncTarget() + other_docs = [] + + def receive_doc(doc, gen, trans_id): + other_docs.append((doc.doc_id, doc.rev, doc.get_json())) + + doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + get_doc = (lambda _: doc, (1,), {}) + new_gen, trans_id = yield remote_target.sync_exchange( + [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=receive_doc) + self.assertEqual(1, new_gen) + self.assertGetEncryptedDoc( + db, 'doc-here', 'replica:1', '{"value": "here"}', False) + + @defer.inlineCallbacks + def test_sync_exchange_send_failure_and_retry_scenario(self): + """ + Test for sync exchange failure and retry. + + This test was adapted to decrypt remote content before assert. + """ + + def blackhole_getstderr(inst): + return cStringIO.StringIO() + + db = self.db2 + _put_doc_if_newer = db._put_doc_if_newer + trigger_ids = ['doc-here2'] + + def bomb_put_doc_if_newer(self, doc, save_conflict, + replica_uid=None, replica_gen=None, + replica_trans_id=None, number_of_docs=None, + doc_idx=None, sync_id=None): + if doc.doc_id in trigger_ids: + raise l2db.errors.U1DBError + return _put_doc_if_newer(doc, save_conflict=save_conflict, + replica_uid=replica_uid, + replica_gen=replica_gen, + replica_trans_id=replica_trans_id, + number_of_docs=number_of_docs, + doc_idx=doc_idx, sync_id=sync_id) + from leap.soledad.common.backend import SoledadBackend + self.patch( + SoledadBackend, '_put_doc_if_newer', bomb_put_doc_if_newer) + remote_target = self.getSyncTarget( + source_replica_uid='replica') + other_changes = [] + + def receive_doc(doc, gen, trans_id): + other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + doc2 = self.make_document('doc-here2', 'replica:1', + '{"value": "here2"}') + get_doc1 = (lambda _: doc1, (1,), {}) + get_doc2 = (lambda _: doc2, (2,), {}) + + with self.assertRaises(l2db.errors.U1DBError): + yield remote_target.sync_exchange( + [(get_doc1, 10, 'T-sid'), (get_doc2, 11, 'T-sud')], + 'replica', + last_known_generation=0, + last_known_trans_id=None, + insert_doc_cb=receive_doc) + + self.assertGetEncryptedDoc( + db, 'doc-here', 'replica:1', '{"value": "here"}', + False) + self.assertEqual( + (10, 'T-sid'), db._get_replica_gen_and_trans_id('replica')) + self.assertEqual([], other_changes) + # retry + trigger_ids = [] + new_gen, trans_id = yield remote_target.sync_exchange( + [(get_doc2, 11, 'T-sud')], 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=receive_doc) + self.assertGetEncryptedDoc( + db, 'doc-here2', 'replica:1', '{"value": "here2"}', + False) + self.assertEqual( + (11, 'T-sud'), db._get_replica_gen_and_trans_id('replica')) + self.assertEqual(2, new_gen) + self.assertEqual( + ('doc-here', 'replica:1', '{"value": "here"}', 1), + other_changes[0][:-1]) + + @defer.inlineCallbacks + def test_sync_exchange_send_ensure_callback(self): + """ + Test for sync exchange failure and retry. + + This test was adapted to decrypt remote content before assert. + """ + remote_target = self.getSyncTarget() + other_docs = [] + replica_uid_box = [] + + def receive_doc(doc, gen, trans_id): + other_docs.append((doc.doc_id, doc.rev, doc.get_json())) + + def ensure_cb(replica_uid): + replica_uid_box.append(replica_uid) + + doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + get_doc = (lambda _: doc, (1,), {}) + new_gen, trans_id = yield remote_target.sync_exchange( + [(get_doc, 10, 'T-sid')], 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=receive_doc, + ensure_callback=ensure_cb) + self.assertEqual(1, new_gen) + db = self.db2 + self.assertEqual(1, len(replica_uid_box)) + self.assertEqual(db._replica_uid, replica_uid_box[0]) + self.assertGetEncryptedDoc( + db, 'doc-here', 'replica:1', '{"value": "here"}', False) + + @defer.inlineCallbacks + def test_sync_exchange_send_events(self): + """ + Test for sync exchange's SOLEDAD_SYNC_SEND_STATUS event. + """ + remote_target = self.getSyncTarget() + uuid = remote_target.uuid + events = [] + + def mocked_events(*args): + events.append((args)) + self.patch( + target.send, '_emit_send_status', mocked_events) + + doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + doc2 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + doc3 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + get_doc = (lambda _: doc, (1,), {}) + get_doc2 = (lambda _: doc2, (1,), {}) + get_doc3 = (lambda _: doc3, (1,), {}) + docs = [(get_doc, 10, 'T-sid'), + (get_doc2, 11, 'T-sid2'), (get_doc3, 12, 'T-sid3')] + new_gen, trans_id = yield remote_target.sync_exchange( + docs, 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=lambda _: 1, + ensure_callback=lambda _: 1) + self.assertEqual(1, new_gen) + self.assertEqual(4, len(events)) + self.assertEquals([(uuid, 0, 3), (uuid, 1, 3), (uuid, 2, 3), + (uuid, 3, 3)], events) + + def test_sync_exchange_in_stream_error(self): + self.skipTest("bypass this test because our sync_exchange process " + "does not return u1db error 503 \"unavailable\" for " + "now") + + @defer.inlineCallbacks + def test_get_sync_info(self): + db = self.db2 + db._set_replica_gen_and_trans_id('other-id', 1, 'T-transid') + remote_target = self.getSyncTarget( + source_replica_uid='other-id') + sync_info = yield remote_target.get_sync_info('other-id') + self.assertEqual( + ('test', 0, '', 1, 'T-transid'), + sync_info) + + @defer.inlineCallbacks + def test_record_sync_info(self): + remote_target = self.getSyncTarget( + source_replica_uid='other-id') + yield remote_target.record_sync_info('other-id', 2, 'T-transid') + self.assertEqual((2, 'T-transid'), + self.db2._get_replica_gen_and_trans_id('other-id')) + + @defer.inlineCallbacks + def test_sync_exchange_receive(self): + db = self.db2 + doc = db.create_doc_from_json('{"value": "there"}') + remote_target = self.getSyncTarget() + other_changes = [] + + def receive_doc(doc, gen, trans_id): + other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + new_gen, trans_id = yield remote_target.sync_exchange( + [], 'replica', last_known_generation=0, last_known_trans_id=None, + insert_doc_cb=receive_doc) + self.assertEqual(1, new_gen) + self.assertEqual( + (doc.doc_id, doc.rev, '{"value": "there"}', 1), + other_changes[0][:-1]) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sync`. +# ----------------------------------------------------------------------------- + +target_scenarios = [ + ('mem,token_soledad', + {'create_db_and_target': make_local_db_and_token_soledad_target, + 'make_app_with_state': make_soledad_app, + 'make_database_for_test': tests.make_memory_database_for_test, + 'copy_database_for_test': tests.copy_memory_database_for_test, + 'make_document_for_test': tests.make_document_for_test}) +] + + +@pytest.mark.needs_couch +class SoledadDatabaseSyncTargetTests( + TestWithScenarios, + SoledadWithCouchServerMixin, + tests.DatabaseBaseTests, + tests.TestCaseWithServer): + """ + Adaptation of u1db.tests.test_sync.DatabaseSyncTargetTests. + """ + + # TODO: implement _set_trace_hook(_shallow) in SoledadHTTPSyncTarget so + # skipped tests can be succesfully executed. + + scenarios = target_scenarios + + whitebox = False + + def setUp(self): + tests.TestCaseWithServer.setUp(self) + self.other_changes = [] + SoledadWithCouchServerMixin.setUp(self) + self.db, self.st = make_local_db_and_soledad_target(self) + + def tearDown(self): + self.db.close() + tests.TestCaseWithServer.tearDown(self) + SoledadWithCouchServerMixin.tearDown(self) + + def set_trace_hook(self, callback, shallow=False): + setter = (self.st._set_trace_hook if not shallow else + self.st._set_trace_hook_shallow) + try: + setter(callback) + except NotImplementedError: + self.skipTest("%s does not implement _set_trace_hook" + % (self.st.__class__.__name__,)) + + @defer.inlineCallbacks + def test_sync_exchange(self): + """ + Test sync exchange. + + This test was adapted to decrypt remote content before assert. + """ + docs_by_gen = [ + ((self.make_document, + ('doc-id', 'replica:1', tests.simple_doc,), {}), + 10, 'T-sid')] + new_gen, trans_id = yield self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertGetEncryptedDoc( + self.db, 'doc-id', 'replica:1', tests.simple_doc, False) + self.assertTransactionLog(['doc-id'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, last_trans_id)) + sync_info = yield self.st.get_sync_info('replica') + self.assertEqual(10, sync_info[3]) + + @defer.inlineCallbacks + def test_sync_exchange_push_many(self): + """ + Test sync exchange. + + This test was adapted to decrypt remote content before assert. + """ + docs_by_gen = [ + ((self.make_document, + ('doc-id', 'replica:1', tests.simple_doc), {}), 10, 'T-1'), + ((self.make_document, + ('doc-id2', 'replica:1', tests.nested_doc), {}), 11, 'T-2')] + new_gen, trans_id = yield self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertGetEncryptedDoc( + self.db, 'doc-id', 'replica:1', tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db, 'doc-id2', 'replica:1', tests.nested_doc, False) + self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + sync_info = yield self.st.get_sync_info('replica') + self.assertEqual(11, sync_info[3]) + + @defer.inlineCallbacks + def test_sync_exchange_returns_many_new_docs(self): + """ + Test sync exchange. + + This test was adapted to avoid JSON serialization comparison as local + and remote representations might differ. It looks directly at the + doc's contents instead. + """ + doc = self.db.create_doc_from_json(tests.simple_doc) + doc2 = self.db.create_doc_from_json(tests.nested_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + new_gen, _ = yield self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + self.assertEqual(2, new_gen) + self.assertEqual( + [(doc.doc_id, doc.rev, 1), + (doc2.doc_id, doc2.rev, 2)], + [c[:-3] + c[-2:-1] for c in self.other_changes]) + self.assertEqual( + json.loads(tests.simple_doc), + json.loads(self.other_changes[0][2])) + self.assertEqual( + json.loads(tests.nested_doc), + json.loads(self.other_changes[1][2])) + if self.whitebox: + self.assertEqual( + self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': + [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) + + def receive_doc(self, doc, gen, trans_id): + self.other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + def test_get_sync_target(self): + self.assertIsNot(None, self.st) + + @defer.inlineCallbacks + def test_get_sync_info(self): + sync_info = yield self.st.get_sync_info('other') + self.assertEqual( + ('test', 0, '', 0, ''), sync_info) + + @defer.inlineCallbacks + def test_create_doc_updates_sync_info(self): + sync_info = yield self.st.get_sync_info('other') + self.assertEqual( + ('test', 0, '', 0, ''), sync_info) + self.db.create_doc_from_json(tests.simple_doc) + sync_info = yield self.st.get_sync_info('other') + self.assertEqual(1, sync_info[1]) + + @defer.inlineCallbacks + def test_record_sync_info(self): + yield self.st.record_sync_info('replica', 10, 'T-transid') + sync_info = yield self.st.get_sync_info('replica') + self.assertEqual( + ('test', 0, '', 10, 'T-transid'), sync_info) + + @defer.inlineCallbacks + def test_sync_exchange_deleted(self): + doc = self.db.create_doc_from_json('{}') + edit_rev = 'replica:1|' + doc.rev + docs_by_gen = [ + ((self.make_document, (doc.doc_id, edit_rev, None), {}), + 10, 'T-sid')] + new_gen, trans_id = yield self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, edit_rev, None, False) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + sync_info = yield self.st.get_sync_info('replica') + self.assertEqual(10, sync_info[3]) + + @defer.inlineCallbacks + def test_sync_exchange_refuses_conflicts(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + ((self.make_document, (doc.doc_id, 'replica:1', new_doc), {}), 10, + 'T-sid')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, tests.simple_doc, 1), + self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + @defer.inlineCallbacks + def test_sync_exchange_ignores_convergence(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + gen, txid = self.db._get_generation_info() + docs_by_gen = [ + ((self.make_document, (doc.doc_id, doc.rev, tests.simple_doc), {}), + 10, 'T-sid')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=gen, + last_known_trans_id=txid, insert_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual(([], 1), (self.other_changes, new_gen)) + + @defer.inlineCallbacks + def test_sync_exchange_returns_new_docs(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_gen, _ = yield self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, tests.simple_doc, 1), + self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + @defer.inlineCallbacks + def test_sync_exchange_returns_deleted_docs(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + self.db.delete_doc(doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + new_gen, _ = yield self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual(2, new_gen) + self.assertEqual( + (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) + + @defer.inlineCallbacks + def test_sync_exchange_getting_newer_docs(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, + 'T-sid')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + @defer.inlineCallbacks + def test_sync_exchange_with_concurrent_updates_of_synced_doc(self): + expected = [] + + def before_whatschanged_cb(state): + if state != 'before whats_changed': + return + cont = '{"key": "cuncurrent"}' + conc_rev = self.db.put_doc( + self.make_document(doc.doc_id, 'test:1|z:2', cont)) + expected.append((doc.doc_id, conc_rev, cont, 3)) + + self.set_trace_hook(before_whatschanged_cb) + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, + 'T-sid')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertEqual(expected, [c[:-1] for c in self.other_changes]) + self.assertEqual(3, new_gen) + + @defer.inlineCallbacks + def test_sync_exchange_with_concurrent_updates(self): + + def after_whatschanged_cb(state): + if state != 'after whats_changed': + return + self.db.create_doc_from_json('{"new": "doc"}') + + self.set_trace_hook(after_whatschanged_cb) + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + ((self.make_document, (doc.doc_id, 'test:1|z:2', new_doc), {}), 10, + 'T-sid')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + @defer.inlineCallbacks + def test_sync_exchange_converged_handling(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + docs_by_gen = [ + ((self.make_document, ('new', 'other:1', '{}'), {}), 4, 'T-foo'), + ((self.make_document, (doc.doc_id, doc.rev, doc.get_json()), {}), + 5, 'T-bar')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + @defer.inlineCallbacks + def test_sync_exchange_detect_incomplete_exchange(self): + def before_get_docs_explode(state): + if state != 'before get_docs': + return + raise l2db.errors.U1DBError("fail") + self.set_trace_hook(before_get_docs_explode) + # suppress traceback printing in the wsgiref server + # self.patch(simple_server.ServerHandler, + # 'log_exception', lambda h, exc_info: None) + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertRaises( + (l2db.errors.U1DBError, l2db.errors.BrokenSyncStream), + self.st.sync_exchange, [], 'other-replica', + last_known_generation=0, last_known_trans_id=None, + insert_doc_cb=self.receive_doc) + + @defer.inlineCallbacks + def test_sync_exchange_doc_ids(self): + sync_exchange_doc_ids = getattr(self.st, 'sync_exchange_doc_ids', None) + if sync_exchange_doc_ids is None: + self.skipTest("sync_exchange_doc_ids not implemented") + db2 = self.create_database('test2') + doc = db2.create_doc_from_json(tests.simple_doc) + new_gen, trans_id = yield sync_exchange_doc_ids( + db2, [(doc.doc_id, 10, 'T-sid')], 0, None, + insert_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, + tests.simple_doc, False) + self.assertTransactionLog([doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(10, self.st.get_sync_info(db2._replica_uid)[3]) + + @defer.inlineCallbacks + def test__set_trace_hook(self): + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb) + yield self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + yield self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ], + called) + + @defer.inlineCallbacks + def test__set_trace_hook_shallow(self): + if (self.st._set_trace_hook_shallow == self.st._set_trace_hook or + self.st._set_trace_hook_shallow.im_func == + target.SoledadHTTPSyncTarget._set_trace_hook_shallow.im_func): + # shallow same as full + expected = ['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ] + else: + expected = ['sync_exchange', 'record_sync_info'] + + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb, shallow=True) + yield self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + yield self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(expected, called) + + +WAIT_STEP = 1 +MAX_WAIT = 10 +DBPASS = "pass" + + +class SyncTimeoutError(Exception): + + """ + Dummy exception to notify timeout during sync. + """ + pass + + +@pytest.mark.needs_couch +class TestSoledadDbSync( + TestWithScenarios, + SoledadWithCouchServerMixin, + tests.TestCaseWithServer): + + """Test db.sync remote sync shortcut""" + + scenarios = [ + ('py-token-http', { + 'create_db_and_target': make_local_db_and_token_soledad_target, + 'make_app_with_state': make_token_soledad_app, + 'make_database_for_test': make_sqlcipher_database_for_test, + 'token': True + }), + ] + + oauth = False + token = False + + def setUp(self): + """ + Need to explicitely invoke inicialization on all bases. + """ + SoledadWithCouchServerMixin.setUp(self) + self.server = self.server_thread = None + self.startTwistedServer() + self.syncer = None + + # config info + self.db1_file = os.path.join(self.tempdir, "db1.u1db") + os.unlink(self.db1_file) + self.db_pass = DBPASS + self.email = ADDRESS + + # get a random prefix for each test, so we do not mess with + # concurrency during initialization and shutting down of + # each local db. + self.rand_prefix = ''.join( + map(lambda x: random.choice(string.ascii_letters), range(6))) + + # open test dbs: db1 will be the local sqlcipher db (which + # instantiates a syncdb). We use the self._soledad instance that was + # already created on some setUp method. + import binascii + tohex = binascii.b2a_hex + key = tohex(self._soledad.secrets.local_key) + dbpath = self._soledad._local_db_path + + self.opts = SQLCipherOptions( + dbpath, key, is_raw_key=True, create=False) + self.db1 = SQLCipherDatabase(self.opts) + + self.db2 = self.request_state._create_database(replica_uid='test') + + def tearDown(self): + """ + Need to explicitely invoke destruction on all bases. + """ + dbsyncer = getattr(self, 'dbsyncer', None) + if dbsyncer: + dbsyncer.close() + self.db1.close() + self.db2.close() + self._soledad.close() + + # XXX should not access "private" attrs + shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) + SoledadWithCouchServerMixin.tearDown(self) + + def do_sync(self, target_name): + """ + Perform sync using SoledadSynchronizer, SoledadSyncTarget + and Token auth. + """ + if self.token: + creds = {'token': { + 'uuid': 'user-uuid', + 'token': 'auth-token', + }} + target_url = self.getURL(self.db2._dbname) + + # get a u1db syncer + crypto = self._soledad._crypto + replica_uid = self.db1._replica_uid + dbsyncer = SQLCipherU1DBSync( + self.opts, + crypto, + replica_uid, + None) + self.dbsyncer = dbsyncer + return dbsyncer.sync(target_url, + creds=creds) + else: + return self._do_sync(self, target_name) + + def _do_sync(self, target_name): + if self.oauth: + path = '~/' + target_name + extra = dict(creds={'oauth': { + 'consumer_key': tests.consumer1.key, + 'consumer_secret': tests.consumer1.secret, + 'token_key': tests.token1.key, + 'token_secret': tests.token1.secret, + }}) + else: + path = target_name + extra = {} + target_url = self.getURL(path) + return self.db.sync(target_url, **extra) + + def wait_for_sync(self): + """ + Wait for sync to finish. + """ + wait = 0 + syncer = self.syncer + if syncer is not None: + while syncer.syncing: + time.sleep(WAIT_STEP) + wait += WAIT_STEP + if wait >= MAX_WAIT: + raise SyncTimeoutError + + def test_db_sync(self): + """ + Test sync. + + Adapted to check for encrypted content. + """ + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc2 = self.db2.create_doc_from_json(tests.nested_doc) + d = self.do_sync('test') + + def _assert_successful_sync(results): + import time + # need to give time to the encryption to proceed + # TODO should implement a defer list to subscribe to the + # all-decrypted event + time.sleep(2) + local_gen_before_sync = results + self.wait_for_sync() + + gen, _, changes = self.db1.whats_changed(local_gen_before_sync) + self.assertEqual(1, len(changes)) + + self.assertEqual(doc2.doc_id, changes[0][0]) + self.assertEqual(1, gen - local_gen_before_sync) + + self.assertGetEncryptedDoc( + self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) + + d.addCallback(_assert_successful_sync) + return d + + +@pytest.mark.needs_couch +class SQLCipherSyncTargetTests(SoledadDatabaseSyncTargetTests): + + # TODO: implement _set_trace_hook(_shallow) in SoledadHTTPSyncTarget so + # skipped tests can be succesfully executed. + + scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS, + target_scenarios)) + + whitebox = False diff --git a/tests/test_soledad/__init__.py b/tests/test_soledad/__init__.py new file mode 100644 index 00000000..c07c8b0e --- /dev/null +++ b/tests/test_soledad/__init__.py @@ -0,0 +1,5 @@ +from test_soledad import util + +__all__ = [ + 'util', +] diff --git a/tests/test_soledad/fixture_soledad.conf b/tests/test_soledad/fixture_soledad.conf new file mode 100644 index 00000000..80e7a4d4 --- /dev/null +++ b/tests/test_soledad/fixture_soledad.conf @@ -0,0 +1,12 @@ +[soledad-server] +couch_url = http://soledad:passwd@localhost:5984 +create_cmd = sudo -u soledad-admin /usr/bin/soledad-create-userdb +admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc +services_tokens_file = /etc/soledad/services.tokens +batching = 0 + +[database-security] +members = user1, user2 +members_roles = role1, role2 +admins = user3, user4 +admins_roles = role3, role3 diff --git a/tests/test_soledad/u1db_tests/README b/tests/test_soledad/u1db_tests/README new file mode 100644 index 00000000..546dfdc9 --- /dev/null +++ b/tests/test_soledad/u1db_tests/README @@ -0,0 +1,23 @@ +General info +------------ + +Test files in this directory are derived from u1db-0.1.4 tests. The main +difference is that: + + (1) they include the test infrastructure packed with soledad; and + (2) they do not include c_backend_wrapper testing. + +Dependencies +------------ + +u1db tests depend on the following python packages: + + unittest2 + mercurial + hgtools + testtools + discover + testscenarios + paste + routes + cython diff --git a/tests/test_soledad/u1db_tests/__init__.py b/tests/test_soledad/u1db_tests/__init__.py new file mode 100644 index 00000000..2a4415a6 --- /dev/null +++ b/tests/test_soledad/u1db_tests/__init__.py @@ -0,0 +1,420 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . +""" +Test infrastructure for U1DB +""" + +import copy +import shutil +import socket +import tempfile +import threading +import json +import sys + +from six import StringIO +from wsgiref import simple_server + +import testscenarios +from twisted.trial import unittest +from twisted.web.server import Site +from twisted.web.wsgi import WSGIResource +from twisted.internet import reactor + +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import Document +from leap.soledad.common.l2db.backends import inmemory +from leap.soledad.common.l2db.remote import server_state +from leap.soledad.common.l2db.remote import http_app +from leap.soledad.common.l2db.remote import http_target + +from leap.soledad.client._db import sqlite + +if sys.version_info[0] < 3: + from pysqlcipher import dbapi2 +else: + from pysqlcipher3 import dbapi2 + + +class TestCase(unittest.TestCase): + + def createTempDir(self, prefix='u1db-tmp-'): + """Create a temporary directory to do some work in. + + This directory will be scheduled for cleanup when the test ends. + """ + tempdir = tempfile.mkdtemp(prefix=prefix) + self.addCleanup(shutil.rmtree, tempdir) + return tempdir + + def make_document(self, doc_id, doc_rev, content, has_conflicts=False): + return self.make_document_for_test( + self, doc_id, doc_rev, content, has_conflicts) + + def make_document_for_test(self, test, doc_id, doc_rev, content, + has_conflicts): + return make_document_for_test( + test, doc_id, doc_rev, content, has_conflicts) + + def assertGetDoc(self, db, doc_id, doc_rev, content, has_conflicts): + """Assert that the document in the database looks correct.""" + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + self.assertEqual(exp_doc, db.get_doc(doc_id)) + + def assertGetDocIncludeDeleted(self, db, doc_id, doc_rev, content, + has_conflicts): + """Assert that the document in the database looks correct.""" + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + self.assertEqual(exp_doc, db.get_doc(doc_id, include_deleted=True)) + + def assertGetDocConflicts(self, db, doc_id, conflicts): + """Assert what conflicts are stored for a given doc_id. + + :param conflicts: A list of (doc_rev, content) pairs. + The first item must match the first item returned from the + database, however the rest can be returned in any order. + """ + if conflicts: + conflicts = [(rev, + (json.loads(cont) if isinstance(cont, basestring) + else cont)) for (rev, cont) in conflicts] + conflicts = conflicts[:1] + sorted(conflicts[1:]) + actual = db.get_doc_conflicts(doc_id) + if actual: + actual = [ + (doc.rev, (json.loads(doc.get_json()) + if doc.get_json() is not None else None)) + for doc in actual] + actual = actual[:1] + sorted(actual[1:]) + self.assertEqual(conflicts, actual) + + +def multiply_scenarios(a_scenarios, b_scenarios): + """Create the cross-product of scenarios.""" + + all_scenarios = [] + for a_name, a_attrs in a_scenarios: + for b_name, b_attrs in b_scenarios: + name = '%s,%s' % (a_name, b_name) + attrs = dict(a_attrs) + attrs.update(b_attrs) + all_scenarios.append((name, attrs)) + return all_scenarios + + +simple_doc = '{"key": "value"}' +nested_doc = '{"key": "value", "sub": {"doc": "underneath"}}' + + +def make_memory_database_for_test(test, replica_uid): + return inmemory.InMemoryDatabase(replica_uid) + + +def copy_memory_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = inmemory.InMemoryDatabase(db._replica_uid) + new_db._transaction_log = db._transaction_log[:] + new_db._docs = copy.deepcopy(db._docs) + new_db._conflicts = copy.deepcopy(db._conflicts) + new_db._indexes = copy.deepcopy(db._indexes) + new_db._factory = db._factory + return new_db + + +def make_sqlite_partial_expanded_for_test(test, replica_uid): + db = sqlite.SQLitePartialExpandDatabase(':memory:') + db._set_replica_uid(replica_uid) + return db + + +def copy_sqlite_partial_expanded_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = sqlite.SQLitePartialExpandDatabase(':memory:') + tmpfile = StringIO() + for line in db._db_handle.iterdump(): + if 'sqlite_sequence' not in line: # work around bug in iterdump + tmpfile.write('%s\n' % line) + tmpfile.seek(0) + new_db._db_handle = dbapi2.connect(':memory:') + new_db._db_handle.cursor().executescript(tmpfile.read()) + new_db._db_handle.commit() + new_db._set_replica_uid(db._replica_uid) + new_db._factory = db._factory + return new_db + + +def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): + return Document(doc_id, rev, content, has_conflicts=has_conflicts) + + +LOCAL_DATABASES_SCENARIOS = [ + ('mem', {'make_database_for_test': make_memory_database_for_test, + 'copy_database_for_test': copy_memory_database_for_test, + 'make_document_for_test': make_document_for_test}), + ('sql', {'make_database_for_test': + make_sqlite_partial_expanded_for_test, + 'copy_database_for_test': + copy_sqlite_partial_expanded_for_test, + 'make_document_for_test': make_document_for_test}), +] + + +class DatabaseBaseTests(TestCase): + + # set to True assertTransactionLog + # is happy with all trans ids = '' + accept_fixed_trans_id = False + + scenarios = LOCAL_DATABASES_SCENARIOS + + def make_database_for_test(self, replica_uid): + return make_memory_database_for_test(self, replica_uid) + + def create_database(self, *args): + return self.make_database_for_test(self, *args) + + def copy_database(self, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + return self.copy_database_for_test(self, db) + + def setUp(self): + super(DatabaseBaseTests, self).setUp() + self.db = self.create_database('test') + + def tearDown(self): + if hasattr(self, 'db') and self.db is not None: + self.db.close() + super(DatabaseBaseTests, self).tearDown() + + def assertTransactionLog(self, doc_ids, db): + """Assert that the given docs are in the transaction log.""" + log = db._get_transaction_log() + just_ids = [] + seen_transactions = set() + for doc_id, transaction_id in log: + just_ids.append(doc_id) + self.assertIsNot(None, transaction_id, + "Transaction id should not be None") + if transaction_id == '' and self.accept_fixed_trans_id: + continue + self.assertNotEqual('', transaction_id, + "Transaction id should be a unique string") + self.assertTrue(transaction_id.startswith('T-')) + self.assertNotIn(transaction_id, seen_transactions) + seen_transactions.add(transaction_id) + self.assertEqual(doc_ids, just_ids) + + def getLastTransId(self, db): + """Return the transaction id for the last database update.""" + return self.db._get_transaction_log()[-1][-1] + + +class ServerStateForTests(server_state.ServerState): + + """Used in the test suite, so we don't have to touch disk, etc.""" + + def __init__(self): + super(ServerStateForTests, self).__init__() + self._dbs = {} + + def open_database(self, path): + try: + return self._dbs[path] + except KeyError: + raise errors.DatabaseDoesNotExist + + def check_database(self, path): + # cares only about the possible exception + self.open_database(path) + + def ensure_database(self, path): + try: + db = self.open_database(path) + except errors.DatabaseDoesNotExist: + db = self._create_database(path) + return db, db._replica_uid + + def _copy_database(self, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + new_db = copy_memory_database_for_test(None, db) + path = db._replica_uid + while path in self._dbs: + path += 'copy' + self._dbs[path] = new_db + return new_db + + def _create_database(self, path): + db = inmemory.InMemoryDatabase(path) + self._dbs[path] = db + return db + + def delete_database(self, path): + del self._dbs[path] + + +class ResponderForTests(object): + + """Responder for tests.""" + _started = False + sent_response = False + status = None + + def start_response(self, status='success', **kwargs): + self._started = True + self.status = status + self.kwargs = kwargs + + def send_response(self, status='success', **kwargs): + self.start_response(status, **kwargs) + self.finish_response() + + def finish_response(self): + self.sent_response = True + + +class TestCaseWithServer(TestCase): + + @staticmethod + def server_def(): + # hook point + # should return (ServerClass, "shutdown method name", "url_scheme") + class _RequestHandler(simple_server.WSGIRequestHandler): + + def log_request(*args): + pass # suppress + + def make_server(host_port, application): + assert application, "forgot to override make_app(_with_state)?" + srv = simple_server.WSGIServer(host_port, _RequestHandler) + # patch the value in if it's None + if getattr(application, 'base_url', 1) is None: + application.base_url = "http://%s:%s" % srv.server_address + srv.set_app(application) + return srv + + return make_server, "shutdown", "http" + + @staticmethod + def make_app_with_state(state): + # hook point + return None + + def make_app(self): + # potential hook point + self.request_state = ServerStateForTests() + return self.make_app_with_state(self.request_state) + + def setUp(self): + super(TestCaseWithServer, self).setUp() + self.server = self.server_thread = self.port = None + + def tearDown(self): + if self.server is not None: + self.server.shutdown() + self.server_thread.join() + self.server.server_close() + if self.port: + self.port.stopListening() + super(TestCaseWithServer, self).tearDown() + + @property + def url_scheme(self): + return 'http' + + def startTwistedServer(self): + application = self.make_app() + resource = WSGIResource(reactor, reactor.getThreadPool(), application) + site = Site(resource) + self.port = reactor.listenTCP(0, site, interface='127.0.0.1') + host = self.port.getHost() + self.server_address = (host.host, host.port) + self.addCleanup(self.port.stopListening) + + def startServer(self): + server_def = self.server_def() + server_class, shutdown_meth, _ = server_def + application = self.make_app() + self.server = server_class(('127.0.0.1', 0), application) + self.server_thread = threading.Thread(target=self.server.serve_forever, + kwargs=dict(poll_interval=0.01)) + self.server_thread.start() + self.addCleanup(self.server_thread.join) + self.addCleanup(getattr(self.server, shutdown_meth)) + self.server_address = self.server.server_address + + def getURL(self, path=None): + host, port = self.server_address + if path is None: + path = '' + return '%s://%s:%s/%s' % (self.url_scheme, host, port, path) + + +def socket_pair(): + """Return a pair of TCP sockets connected to each other. + + Unlike socket.socketpair, this should work on Windows. + """ + sock_pair = getattr(socket, 'socket_pair', None) + if sock_pair: + return sock_pair(socket.AF_INET, socket.SOCK_STREAM) + listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listen_sock.bind(('127.0.0.1', 0)) + listen_sock.listen(1) + client_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client_sock.connect(listen_sock.getsockname()) + server_sock, addr = listen_sock.accept() + listen_sock.close() + return server_sock, client_sock + + +def load_with_scenarios(loader, standard_tests, pattern): + """Load the tests in a given module. + + This just applies testscenarios.generate_scenarios to all the tests that + are present. We do it at load time rather than at run time, because it + plays nicer with various tools. + """ + suite = loader.suiteClass() + suite.addTests(testscenarios.generate_scenarios(standard_tests)) + return suite + + +# from u1db.tests.test_remote_sync_target + +def make_http_app(state): + return http_app.HTTPApp(state) + + +def http_sync_target(test, path): + return http_target.HTTPSyncTarget(test.getURL(path)) diff --git a/tests/test_soledad/u1db_tests/test_backends.py b/tests/test_soledad/u1db_tests/test_backends.py new file mode 100644 index 00000000..10dcdff9 --- /dev/null +++ b/tests/test_soledad/u1db_tests/test_backends.py @@ -0,0 +1,1888 @@ +# Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of leap.soledad.common +# +# leap.soledad.common is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +""" +The backend class for L2DB. This deals with hiding storage details. +""" + +import json + +from leap.soledad.common.l2db import DocumentBase +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db.remote import http_database + +from test_soledad import u1db_tests as tests + +from unittest import skip + +simple_doc = tests.simple_doc +nested_doc = tests.nested_doc + + +def make_http_database_for_test(test, replica_uid, path='test', *args): + test.startServer() + test.request_state._create_database(replica_uid) + return http_database.HTTPDatabase(test.getURL(path)) + + +def copy_http_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + return test.request_state._copy_database(db) + + +class TestAlternativeDocument(DocumentBase): + + """A (not very) alternative implementation of Document.""" + + +@skip("Skiping tests imported from U1DB.") +class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + [ + ('http', {'make_database_for_test': make_http_database_for_test, + 'copy_database_for_test': copy_http_database_for_test, + 'make_document_for_test': tests.make_document_for_test, + 'make_app_with_state': tests.make_http_app}), + ] + + def test_close(self): + self.db.close() + + def test_create_doc_allocating_doc_id(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertNotEqual(None, doc.doc_id) + self.assertNotEqual(None, doc.rev) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_create_doc_different_ids_same_db(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertNotEqual(doc1.doc_id, doc2.doc_id) + + def test_create_doc_with_id(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my-id') + self.assertEqual('my-id', doc.doc_id) + self.assertNotEqual(None, doc.rev) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_create_doc_existing_id(self): + doc = self.db.create_doc_from_json(simple_doc) + new_content = '{"something": "else"}' + self.assertRaises( + errors.RevisionConflict, self.db.create_doc_from_json, + new_content, doc.doc_id) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_put_doc_creating_initial(self): + doc = self.make_document('my_doc_id', None, simple_doc) + new_rev = self.db.put_doc(doc) + self.assertIsNot(None, new_rev) + self.assertGetDoc(self.db, 'my_doc_id', new_rev, simple_doc, False) + + def test_put_doc_space_in_id(self): + doc = self.make_document('my doc id', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_update(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + orig_rev = doc.rev + doc.set_json('{"updated": "stuff"}') + new_rev = self.db.put_doc(doc) + self.assertNotEqual(new_rev, orig_rev) + self.assertGetDoc(self.db, 'my_doc_id', new_rev, + '{"updated": "stuff"}', False) + self.assertEqual(doc.rev, new_rev) + + def test_put_non_ascii_key(self): + content = json.dumps({u'key\xe5': u'val'}) + doc = self.db.create_doc_from_json(content, doc_id='my_doc') + self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) + + def test_put_non_ascii_value(self): + content = json.dumps({'key': u'\xe5'}) + doc = self.db.create_doc_from_json(content, doc_id='my_doc') + self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) + + def test_put_doc_refuses_no_id(self): + doc = self.make_document(None, None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + doc = self.make_document("", None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_refuses_slashes(self): + doc = self.make_document('a/b', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + doc = self.make_document(r'\b', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_url_quoting_is_fine(self): + doc_id = "%2F%2Ffoo%2Fbar" + doc = self.make_document(doc_id, None, simple_doc) + new_rev = self.db.put_doc(doc) + self.assertGetDoc(self.db, doc_id, new_rev, simple_doc, False) + + def test_put_doc_refuses_non_existing_old_rev(self): + doc = self.make_document('doc-id', 'test:4', simple_doc) + self.assertRaises(errors.RevisionConflict, self.db.put_doc, doc) + + def test_put_doc_refuses_non_ascii_doc_id(self): + doc = self.make_document('d\xc3\xa5c-id', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_fails_with_bad_old_rev(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + old_rev = doc.rev + bad_doc = self.make_document(doc.doc_id, 'other:1', + '{"something": "else"}') + self.assertRaises(errors.RevisionConflict, self.db.put_doc, bad_doc) + self.assertGetDoc(self.db, 'my_doc_id', old_rev, simple_doc, False) + + def test_create_succeeds_after_delete(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) + deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) + new_doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.assertGetDoc(self.db, 'my_doc_id', new_doc.rev, simple_doc, False) + new_vc = vectorclock.VectorClockRev(new_doc.rev) + self.assertTrue( + new_vc.is_newer(deleted_vc), + "%s does not supersede %s" % (new_doc.rev, deleted_doc.rev)) + + def test_put_succeeds_after_delete(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) + deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) + doc2 = self.make_document('my_doc_id', None, simple_doc) + self.db.put_doc(doc2) + self.assertGetDoc(self.db, 'my_doc_id', doc2.rev, simple_doc, False) + new_vc = vectorclock.VectorClockRev(doc2.rev) + self.assertTrue( + new_vc.is_newer(deleted_vc), + "%s does not supersede %s" % (doc2.rev, deleted_doc.rev)) + + def test_get_doc_after_put(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.assertGetDoc(self.db, 'my_doc_id', doc.rev, simple_doc, False) + + def test_get_doc_nonexisting(self): + self.assertIs(None, self.db.get_doc('non-existing')) + + def test_get_doc_deleted(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + self.assertIs(None, self.db.get_doc('my_doc_id')) + + def test_get_doc_include_deleted(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + + def test_get_docs(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual([doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + + def test_get_docs_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc1) + self.assertEqual([doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + + def test_get_docs_include_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc1) + self.assertEqual( + [doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id], + include_deleted=True))) + + def test_get_docs_request_ordered(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual([doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + self.assertEqual([doc2, doc1], + list(self.db.get_docs([doc2.doc_id, doc1.doc_id]))) + + def test_get_docs_empty_list(self): + self.assertEqual([], list(self.db.get_docs([]))) + + def test_handles_nested_content(self): + doc = self.db.create_doc_from_json(nested_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) + + def test_handles_doc_with_null(self): + doc = self.db.create_doc_from_json('{"key": null}') + self.assertGetDoc(self.db, doc.doc_id, doc.rev, '{"key": null}', False) + + def test_delete_doc(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + orig_rev = doc.rev + self.db.delete_doc(doc) + self.assertNotEqual(orig_rev, doc.rev) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + self.assertIs(None, self.db.get_doc(doc.doc_id)) + + def test_delete_doc_non_existent(self): + doc = self.make_document('non-existing', 'other:1', simple_doc) + self.assertRaises(errors.DocumentDoesNotExist, self.db.delete_doc, doc) + + def test_delete_doc_already_deleted(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertRaises(errors.DocumentAlreadyDeleted, + self.db.delete_doc, doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + + def test_delete_doc_bad_rev(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + doc2 = self.make_document(doc1.doc_id, 'other:1', simple_doc) + self.assertRaises(errors.RevisionConflict, self.db.delete_doc, doc2) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + + def test_delete_doc_sets_content_to_None(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertIs(None, doc.get_json()) + + def test_delete_doc_rev_supersedes(self): + doc = self.db.create_doc_from_json(simple_doc) + doc.set_json(nested_doc) + self.db.put_doc(doc) + doc.set_json('{"fishy": "content"}') + self.db.put_doc(doc) + old_rev = doc.rev + self.db.delete_doc(doc) + cur_vc = vectorclock.VectorClockRev(old_rev) + deleted_vc = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(deleted_vc.is_newer(cur_vc), + "%s does not supersede %s" % (doc.rev, old_rev)) + + def test_delete_then_put(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + doc.set_json(nested_doc) + self.db.put_doc(doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) + + +@skip("Skiping tests imported from U1DB.") +class DocumentSizeTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_put_doc_refuses_oversized_documents(self): + self.db.set_document_size_limit(1) + doc = self.make_document('doc-id', None, simple_doc) + self.assertRaises(errors.DocumentTooBig, self.db.put_doc, doc) + + def test_create_doc_refuses_oversized_documents(self): + self.db.set_document_size_limit(1) + self.assertRaises( + errors.DocumentTooBig, self.db.create_doc_from_json, simple_doc, + doc_id='my_doc_id') + + def test_set_document_size_limit_zero(self): + self.db.set_document_size_limit(0) + self.assertEqual(0, self.db.document_size_limit) + + def test_set_document_size_limit(self): + self.db.set_document_size_limit(1000000) + self.assertEqual(1000000, self.db.document_size_limit) + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def setUp(self): + tests.DatabaseBaseTests.setUp(self) + + def test_create_doc_different_ids_diff_db(self): + doc1 = self.db.create_doc_from_json(simple_doc) + db2 = self.create_database('other-uid') + doc2 = db2.create_doc_from_json(simple_doc) + self.assertNotEqual(doc1.doc_id, doc2.doc_id) + db2.close() + + def test_put_doc_refuses_slashes_picky(self): + doc = self.make_document('/a', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_get_all_docs_empty(self): + self.assertEqual([], list(self.db.get_all_docs()[1])) + + def test_get_all_docs(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual( + sorted([doc1, doc2]), sorted(list(self.db.get_all_docs()[1]))) + + def test_get_all_docs_exclude_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc2) + self.assertEqual([doc1], list(self.db.get_all_docs()[1])) + + def test_get_all_docs_include_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc2) + self.assertEqual( + sorted([doc1, doc2]), + sorted(list(self.db.get_all_docs(include_deleted=True)[1]))) + + def test_get_all_docs_generation(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json(nested_doc) + self.assertEqual(2, self.db.get_all_docs()[0]) + + def test_simple_put_doc_if_newer(self): + doc = self.make_document('my-doc-id', 'test:1', simple_doc) + state_at_gen = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('inserted', 1), state_at_gen) + self.assertGetDoc(self.db, 'my-doc-id', 'test:1', simple_doc, False) + + def test_simple_put_doc_if_newer_deleted(self): + self.db.create_doc_from_json('{}', doc_id='my-doc-id') + doc = self.make_document('my-doc-id', 'test:2', None) + state_at_gen = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('inserted', 2), state_at_gen) + self.assertGetDocIncludeDeleted( + self.db, 'my-doc-id', 'test:2', None, False) + + def test_put_doc_if_newer_already_superseded(self): + orig_doc = '{"new": "doc"}' + doc1 = self.db.create_doc_from_json(orig_doc) + doc1_rev1 = doc1.rev + doc1.set_json(simple_doc) + self.db.put_doc(doc1) + doc1_rev2 = doc1.rev + # Nothing is inserted, because the document is already superseded + doc = self.make_document(doc1.doc_id, doc1_rev1, orig_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('superseded', state) + self.assertGetDoc(self.db, doc1.doc_id, doc1_rev2, simple_doc, False) + + def test_put_doc_if_newer_autoresolve(self): + doc1 = self.db.create_doc_from_json(simple_doc) + rev = doc1.rev + doc = self.make_document(doc1.doc_id, "whatever:1", doc1.get_json()) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('superseded', state) + doc2 = self.db.get_doc(doc1.doc_id) + v2 = vectorclock.VectorClockRev(doc2.rev) + self.assertTrue(v2.is_newer(vectorclock.VectorClockRev("whatever:1"))) + self.assertTrue(v2.is_newer(vectorclock.VectorClockRev(rev))) + # strictly newer locally + self.assertTrue(rev not in doc2.rev) + + def test_put_doc_if_newer_already_converged(self): + orig_doc = '{"new": "doc"}' + doc1 = self.db.create_doc_from_json(orig_doc) + state_at_gen = self.db._put_doc_if_newer( + doc1, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('converged', 1), state_at_gen) + + def test_put_doc_if_newer_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + # Nothing is inserted, the document id is returned as would-conflict + alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + state, _ = self.db._put_doc_if_newer( + alt_doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('conflicted', state) + # The database wasn't altered + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + + def test_put_doc_if_newer_newer_generation(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.make_document('doc_id', 'other:2', simple_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='other', replica_gen=2, + replica_trans_id='T-irrelevant') + self.assertEqual('inserted', state) + + def test_put_doc_if_newer_same_generation_same_txid(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.db.create_doc_from_json(simple_doc) + self.make_document(doc.doc_id, 'other:1', simple_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='other', replica_gen=1, + replica_trans_id='T-sid') + self.assertEqual('converged', state) + + def test_put_doc_if_newer_wrong_transaction_id(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.make_document('doc_id', 'other:1', simple_doc) + self.assertRaises( + errors.InvalidTransactionId, + self.db._put_doc_if_newer, doc, save_conflict=False, + replica_uid='other', replica_gen=1, replica_trans_id='T-sad') + + def test_put_doc_if_newer_old_generation_older_doc(self): + orig_doc = '{"new": "doc"}' + doc = self.db.create_doc_from_json(orig_doc) + doc_rev1 = doc.rev + doc.set_json(simple_doc) + self.db.put_doc(doc) + self.db._set_replica_gen_and_trans_id('other', 3, 'T-sid') + older_doc = self.make_document(doc.doc_id, doc_rev1, simple_doc) + state, _ = self.db._put_doc_if_newer( + older_doc, save_conflict=False, replica_uid='other', replica_gen=8, + replica_trans_id='T-irrelevant') + self.assertEqual('superseded', state) + + def test_put_doc_if_newer_old_generation_newer_doc(self): + self.db._set_replica_gen_and_trans_id('other', 5, 'T-sid') + doc = self.make_document('doc_id', 'other:1', simple_doc) + self.assertRaises( + errors.InvalidGeneration, + self.db._put_doc_if_newer, doc, save_conflict=False, + replica_uid='other', replica_gen=1, replica_trans_id='T-sad') + + def test_put_doc_if_newer_replica_uid(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', + nested_doc) + self.assertEqual('inserted', + self.db._put_doc_if_newer( + doc2, + save_conflict=False, + replica_uid='other', + replica_gen=2, + replica_trans_id='T-id2')[0]) + self.assertEqual((2, 'T-id2'), self.db._get_replica_gen_and_trans_id( + 'other')) + # Compare to the old rev, should be superseded + doc2 = self.make_document(doc1.doc_id, doc1.rev, nested_doc) + self.assertEqual('superseded', + self.db._put_doc_if_newer( + doc2, + save_conflict=False, + replica_uid='other', + replica_gen=3, + replica_trans_id='T-id3')[0]) + self.assertEqual( + (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) + # A conflict that isn't saved still records the sync gen, because we + # don't need to see it again + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|fourth:1', + '{}') + self.assertEqual('conflicted', + self.db._put_doc_if_newer( + doc2, + save_conflict=False, + replica_uid='other', + replica_gen=4, + replica_trans_id='T-id4')[0]) + self.assertEqual( + (4, 'T-id4'), self.db._get_replica_gen_and_trans_id('other')) + + def test__get_replica_gen_and_trans_id(self): + self.assertEqual( + (0, ''), self.db._get_replica_gen_and_trans_id('other-db')) + self.db._set_replica_gen_and_trans_id('other-db', 2, 'T-transaction') + self.assertEqual( + (2, 'T-transaction'), + self.db._get_replica_gen_and_trans_id('other-db')) + + def test_put_updates_transaction_log(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + doc.set_json('{"something": "else"}') + self.db.put_doc(doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed()) + + def test_delete_updates_transaction_log(self): + doc = self.db.create_doc_from_json(simple_doc) + db_gen, _, _ = self.db.whats_changed() + self.db.delete_doc(doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed(db_gen)) + + def test_whats_changed_initial_database(self): + self.assertEqual((0, '', []), self.db.whats_changed()) + + def test_whats_changed_returns_one_id_for_multiple_changes(self): + doc = self.db.create_doc_from_json(simple_doc) + doc.set_json('{"new": "contents"}') + self.db.put_doc(doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed()) + self.assertEqual((2, last_trans_id, []), self.db.whats_changed(2)) + + def test_whats_changed_returns_last_edits_ascending(self): + doc = self.db.create_doc_from_json(simple_doc) + doc1 = self.db.create_doc_from_json(simple_doc) + doc.set_json('{"new": "contents"}') + self.db.delete_doc(doc1) + delete_trans_id = self.getLastTransId(self.db) + self.db.put_doc(doc) + put_trans_id = self.getLastTransId(self.db) + self.assertEqual((4, put_trans_id, + [(doc1.doc_id, 3, delete_trans_id), + (doc.doc_id, 4, put_trans_id)]), + self.db.whats_changed()) + + def test_whats_changed_doesnt_include_old_gen(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((3, last_trans_id, [(doc2.doc_id, 3, last_trans_id)]), + self.db.whats_changed(2)) + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_validate_gen_and_trans_id(self): + self.db.create_doc_from_json(simple_doc) + gen, trans_id = self.db._get_generation_info() + self.db.validate_gen_and_trans_id(gen, trans_id) + + def test_validate_gen_and_trans_id_invalid_txid(self): + self.db.create_doc_from_json(simple_doc) + gen, _ = self.db._get_generation_info() + self.assertRaises( + errors.InvalidTransactionId, + self.db.validate_gen_and_trans_id, gen, 'wrong') + + def test_validate_gen_and_trans_id_invalid_gen(self): + self.db.create_doc_from_json(simple_doc) + gen, trans_id = self.db._get_generation_info() + self.assertRaises( + errors.InvalidGeneration, + self.db.validate_gen_and_trans_id, gen + 1, trans_id) + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_validate_source_gen_and_trans_id_same(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.db._validate_source('other', 1, 'T-sid') + + def test_validate_source_gen_newer(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.db._validate_source('other', 2, 'T-whatevs') + + def test_validate_source_wrong_txid(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.assertRaises( + errors.InvalidTransactionId, + self.db._validate_source, 'other', 1, 'T-sad') + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests): + # test supporting/functionality around storing conflicts + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_get_docs_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual([doc2], list(self.db.get_docs([doc1.doc_id]))) + + def test_get_docs_conflicts_ignored(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + no_conflict_doc = self.make_document(doc1.doc_id, 'alternate:1', + nested_doc) + self.assertEqual([no_conflict_doc, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id], + check_for_conflicts=False))) + + def test_get_doc_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual([alt_doc, doc], + self.db.get_doc_conflicts(doc.doc_id)) + + def test_get_all_docs_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + _, docs = self.db.get_all_docs() + self.assertTrue(list(docs)[0].has_conflicts) + + def test_get_doc_conflicts_unconflicted(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertEqual([], self.db.get_doc_conflicts(doc.doc_id)) + + def test_get_doc_conflicts_no_such_id(self): + self.assertEqual([], self.db.get_doc_conflicts('doc-id')) + + def test_resolve_doc(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc.doc_id, + [('alternate:1', nested_doc), + (doc.rev, simple_doc)]) + orig_rev = doc.rev + self.db.resolve_doc(doc, [alt_doc.rev, doc.rev]) + self.assertNotEqual(orig_rev, doc.rev) + self.assertFalse(doc.has_conflicts) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + self.assertGetDocConflicts(self.db, doc.doc_id, []) + + def test_resolve_doc_picks_biggest_vcr(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, simple_doc)]) + orig_doc1_rev = doc1.rev + self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) + self.assertFalse(doc1.has_conflicts) + self.assertNotEqual(orig_doc1_rev, doc1.rev) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + vcr_1 = vectorclock.VectorClockRev(orig_doc1_rev) + vcr_2 = vectorclock.VectorClockRev(doc2.rev) + vcr_new = vectorclock.VectorClockRev(doc1.rev) + self.assertTrue(vcr_new.is_newer(vcr_1)) + self.assertTrue(vcr_new.is_newer(vcr_2)) + + def test_resolve_doc_partial_not_winning(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, simple_doc)]) + content3 = '{"key": "valin3"}' + doc3 = self.make_document(doc1.doc_id, 'third:1', content3) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) + self.assertTrue(doc1.has_conflicts) + self.assertGetDoc(self.db, doc1.doc_id, doc3.rev, content3, True) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc)]) + + def test_resolve_doc_partial_winning(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + content3 = '{"key": "valin3"}' + doc3 = self.make_document(doc1.doc_id, 'third:1', content3) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + self.db.resolve_doc(doc1, [doc3.rev, doc1.rev]) + self.assertTrue(doc1.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + + def test_resolve_doc_with_delete_conflict(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, None)]) + self.db.resolve_doc(doc2, [doc1.rev, doc2.rev]) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, False) + + def test_resolve_doc_with_delete_to_delete(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, None)]) + self.db.resolve_doc(doc1, [doc1.rev, doc2.rev]) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + self.assertGetDocIncludeDeleted( + self.db, doc1.doc_id, doc1.rev, None, False) + + def test_put_doc_if_newer_save_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + # Document is inserted as a conflict + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + state, _ = self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('conflicted', state) + # The database was updated + self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, nested_doc, True) + + def test_force_doc_conflict_supersedes_properly(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', '{"b": 1}') + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + doc3 = self.make_document(doc1.doc_id, 'altalt:1', '{"c": 1}') + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + doc22 = self.make_document(doc1.doc_id, 'alternate:2', '{"b": 2}') + self.db._put_doc_if_newer( + doc22, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='zed') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:2', doc22.get_json()), + ('altalt:1', doc3.get_json()), + (doc1.rev, simple_doc)]) + + def test_put_doc_if_newer_save_conflict_was_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertTrue(doc2.has_conflicts) + self.assertGetDoc( + self.db, doc1.doc_id, 'alternate:1', nested_doc, True) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:1', nested_doc), + (doc1.rev, None)]) + + def test_put_doc_if_newer_propagates_full_resolution(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + resolved_vcr = vectorclock.VectorClockRev(doc1.rev) + vcr_2 = vectorclock.VectorClockRev(doc2.rev) + resolved_vcr.maximize(vcr_2) + resolved_vcr.increment('alternate') + doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), + '{"good": 1}') + state, _ = self.db._put_doc_if_newer( + doc_resolved, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual('inserted', state) + self.assertFalse(doc_resolved.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + doc3 = self.db.get_doc(doc1.doc_id) + self.assertFalse(doc3.has_conflicts) + + def test_put_doc_if_newer_propagates_partial_resolution(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'altalt:1', '{}') + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + doc3 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:1', nested_doc), + ('test:1', simple_doc), + ('altalt:1', '{}')]) + resolved_vcr = vectorclock.VectorClockRev(doc1.rev) + vcr_3 = vectorclock.VectorClockRev(doc3.rev) + resolved_vcr.maximize(vcr_3) + resolved_vcr.increment('alternate') + doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), + '{"good": 1}') + state, _ = self.db._put_doc_if_newer( + doc_resolved, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual('inserted', state) + self.assertTrue(doc_resolved.has_conflicts) + doc4 = self.db.get_doc(doc1.doc_id) + self.assertTrue(doc4.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:2|test:1', '{"good": 1}'), + ('altalt:1', '{}')]) + + def test_put_doc_if_newer_replica_uid(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db._set_replica_gen_and_trans_id('other', 1, 'T-id') + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', + nested_doc) + self.db._put_doc_if_newer(doc2, save_conflict=True, + replica_uid='other', replica_gen=2, + replica_trans_id='T-id2') + # Conflict vs the current update + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|third:3', + '{}') + self.assertEqual('conflicted', + self.db._put_doc_if_newer( + doc2, + save_conflict=True, + replica_uid='other', + replica_gen=3, + replica_trans_id='T-id3')[0]) + self.assertEqual( + (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) + + def test_put_doc_if_newer_autoresolve_2(self): + # this is an ordering variant of _3, but that already works + # adding the test explicitly to catch the regression easily + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', "{}") + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', + '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:2|other:1', "{}") + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'inserted') + self.assertFalse(self.db.get_doc(doc_a1.doc_id).has_conflicts) + + def test_put_doc_if_newer_autoresolve_3(self): + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', "{}") + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', "{}") + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'superseded') + doc = self.db.get_doc(doc_a1.doc_id, True) + self.assertFalse(doc.has_conflicts) + rev = vectorclock.VectorClockRev(doc.rev) + rev_a3 = vectorclock.VectorClockRev('test:3') + rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') + self.assertTrue(rev.is_newer(rev_a3)) + self.assertTrue('test:4' in doc.rev) # locally increased + self.assertTrue(rev.is_newer(rev_a1b1)) + + def test_put_doc_if_newer_autoresolve_4(self): + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', None) + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', None) + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'superseded') + doc = self.db.get_doc(doc_a1.doc_id, True) + self.assertFalse(doc.has_conflicts) + rev = vectorclock.VectorClockRev(doc.rev) + rev_a3 = vectorclock.VectorClockRev('test:3') + rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') + self.assertTrue(rev.is_newer(rev_a3)) + self.assertTrue('test:4' in doc.rev) # locally increased + self.assertTrue(rev.is_newer(rev_a1b1)) + + def test_put_refuses_to_update_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + content2 = '{"key": "altval"}' + doc2 = self.make_document(doc1.doc_id, 'altrev:1', content2) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, content2, True) + content3 = '{"key": "local"}' + doc2.set_json(content3) + self.assertRaises(errors.ConflictedDoc, self.db.put_doc, doc2) + + def test_delete_refuses_for_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'altrev:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, True) + self.assertRaises(errors.ConflictedDoc, self.db.delete_doc, doc2) + + +@skip("Skiping tests imported from U1DB.") +class DatabaseIndexTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def assertParseError(self, definition): + self.db.create_doc_from_json(nested_doc) + self.assertRaises( + errors.IndexDefinitionParseError, self.db.create_index, 'idx', + definition) + + def assertIndexCreatable(self, definition): + name = "idx" + self.db.create_doc_from_json(nested_doc) + self.db.create_index(name, definition) + self.assertEqual( + [(name, [definition])], self.db.list_indexes()) + + def test_create_index(self): + self.db.create_index('test-idx', 'name') + self.assertEqual([('test-idx', ['name'])], + self.db.list_indexes()) + + def test_create_index_on_non_ascii_field_name(self): + doc = self.db.create_doc_from_json(json.dumps({u'\xe5': 'value'})) + self.db.create_index('test-idx', u'\xe5') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_list_indexes_with_non_ascii_field_names(self): + self.db.create_index('test-idx', u'\xe5') + self.assertEqual( + [('test-idx', [u'\xe5'])], self.db.list_indexes()) + + def test_create_index_evaluates_it(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_wildcard_matches_unicode_value(self): + doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) + + def test_retrieve_unicode_value_from_index(self): + doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', u"valu\xe5")) + + def test_create_index_fails_if_name_taken(self): + self.db.create_index('test-idx', 'key') + self.assertRaises(errors.IndexNameTakenError, + self.db.create_index, + 'test-idx', 'stuff') + + def test_create_index_does_not_fail_if_name_taken_with_same_index(self): + self.db.create_index('test-idx', 'key') + self.db.create_index('test-idx', 'key') + self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) + + def test_create_index_does_not_duplicate_indexed_fields(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.db.delete_index('test-idx') + self.db.create_index('test-idx', 'key') + self.assertEqual(1, len(self.db.get_from_index('test-idx', 'value'))) + + def test_delete_index_does_not_remove_fields_from_other_indexes(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.db.create_index('test-idx2', 'key') + self.db.delete_index('test-idx') + self.assertEqual(1, len(self.db.get_from_index('test-idx2', 'value'))) + + def test_create_index_after_deleting_document(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc2) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_delete_index(self): + self.db.create_index('test-idx', 'key') + self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) + self.db.delete_index('test-idx') + self.assertEqual([], self.db.list_indexes()) + + def test_create_adds_to_index(self): + self.db.create_index('test-idx', 'key') + doc = self.db.create_doc_from_json(simple_doc) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_unmatched(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_from_index('test-idx', 'novalue')) + + def test_create_index_multiple_exact_matches(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'value'))) + + def test_get_from_index(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_multi(self): + content = '{"key": "value", "key2": "value2"}' + doc = self.db.create_doc_from_json(content) + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2')) + + def test_get_from_index_multi_list(self): + doc = self.db.create_doc_from_json( + '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-1')) + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-2')) + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-3')) + self.assertEqual( + [('value', 'value2-1'), ('value', 'value2-2'), + ('value', 'value2-3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_from_index_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key', 'key2') + alt_doc = self.make_document( + doc.doc_id, 'alternate:1', + '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + docs = self.db.get_from_index('test-idx', 'value', 'value2-1') + self.assertTrue(docs[0].has_conflicts) + + def test_get_index_keys_multi_list_list(self): + self.db.create_doc_from_json( + '{"key": "value1-1 value1-2 value1-3", ' + '"key2": ["value2-1", "value2-2", "value2-3"]}') + self.db.create_index('test-idx', 'split_words(key)', 'key2') + self.assertEqual( + [(u'value1-1', u'value2-1'), (u'value1-1', u'value2-2'), + (u'value1-1', u'value2-3'), (u'value1-2', u'value2-1'), + (u'value1-2', u'value2-2'), (u'value1-2', u'value2-3'), + (u'value1-3', u'value2-1'), (u'value1-3', u'value2-2'), + (u'value1-3', u'value2-3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_from_index_multi_ordered(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2, doc1], + self.db.get_from_index('test-idx', 'v*', '*')) + + def test_get_range_from_index_start_end(self): + doc1 = self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + self.db.create_doc_from_json('{"key": "value4"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', 'value2', 'value3')) + + def test_get_range_from_index_start(self): + doc1 = self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + doc3 = self.db.create_doc_from_json('{"key": "value4"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc2, doc1, doc3], + self.db.get_range_from_index('test-idx', 'value2')) + + def test_get_range_from_index_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + alt_doc = self.make_document( + doc.doc_id, 'alternate:1', '{"key": "valuedepalue"}') + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + docs = self.db.get_range_from_index('test-idx', 'a') + self.assertTrue(docs[0].has_conflicts) + + def test_get_range_from_index_end(self): + self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + self.db.create_doc_from_json('{"key": "value4"}') + doc4 = self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc4, doc2], + self.db.get_range_from_index('test-idx', None, 'value2')) + + def test_get_wildcard_range_from_index_start(self): + doc1 = self.db.create_doc_from_json('{"key": "value4"}') + doc2 = self.db.create_doc_from_json('{"key": "value23"}') + doc3 = self.db.create_doc_from_json('{"key": "value2"}') + doc4 = self.db.create_doc_from_json('{"key": "value22"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc3, doc4, doc2, doc1], + self.db.get_range_from_index('test-idx', 'value2*')) + + def test_get_wildcard_range_from_index_end(self): + self.db.create_doc_from_json('{"key": "value4"}') + doc2 = self.db.create_doc_from_json('{"key": "value23"}') + doc3 = self.db.create_doc_from_json('{"key": "value2"}') + doc4 = self.db.create_doc_from_json('{"key": "value22"}') + doc5 = self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc5, doc3, doc4, doc2], + self.db.get_range_from_index('test-idx', None, 'value2*')) + + def test_get_wildcard_range_from_index_start_end(self): + self.db.create_doc_from_json('{"key": "a"}') + self.db.create_doc_from_json('{"key": "boo3"}') + doc3 = self.db.create_doc_from_json('{"key": "catalyst"}') + doc4 = self.db.create_doc_from_json('{"key": "whaever"}') + self.db.create_doc_from_json('{"key": "zerg"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc3, doc4], + self.db.get_range_from_index('test-idx', 'cat*', 'zap*')) + + def test_get_range_from_index_multi_column_start_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc3, doc2], + self.db.get_range_from_index( + 'test-idx', ('value2', 'value2'), ('value2', 'value3'))) + + def test_get_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + self.db.create_doc_from_json('{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', 'value3'))) + + def test_get_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index( + 'test-idx', None, ('value2', 'value3'))) + + def test_get_wildcard_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc3, doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', 'value2*'))) + + def test_get_wildcard_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index( + 'test-idx', None, ('value2', 'value2*'))) + + def test_get_glob_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', '*'))) + + def test_get_glob_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index('test-idx', None, ('value2', '*'))) + + def test_get_range_from_index_illegal_wildcard_order(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', ('*', 'v2')) + + def test_get_range_from_index_illegal_glob_after_wildcard(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', ('*', 'v*')) + + def test_get_range_from_index_illegal_wildcard_order_end(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', None, ('*', 'v2')) + + def test_get_range_from_index_illegal_glob_after_wildcard_end(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', None, ('*', 'v*')) + + def test_get_from_index_fails_if_no_index(self): + self.assertRaises( + errors.IndexDoesNotExist, self.db.get_from_index, 'foo') + + def test_get_index_keys_fails_if_no_index(self): + self.assertRaises(errors.IndexDoesNotExist, + self.db.get_index_keys, + 'foo') + + def test_get_index_keys_works_if_no_docs(self): + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_index_keys('test-idx')) + + def test_put_updates_index(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + new_content = '{"key": "altval"}' + doc.set_json(new_content) + self.db.put_doc(doc) + self.assertEqual([], self.db.get_from_index('test-idx', 'value')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'altval')) + + def test_delete_updates_index(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'value'))) + self.db.delete_doc(doc) + self.assertEqual([doc2], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_illegal_number_of_entries(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidValueForIndex, self.db.get_from_index, 'test-idx') + self.assertRaises( + errors.InvalidValueForIndex, + self.db.get_from_index, 'test-idx', 'v1') + self.assertRaises( + errors.InvalidValueForIndex, + self.db.get_from_index, 'test-idx', 'v1', 'v2', 'v3') + + def test_get_from_index_illegal_wildcard_order(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', '*', 'v2') + + def test_get_from_index_illegal_glob_after_wildcard(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', '*', 'v*') + + def test_get_all_from_index(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + # This one should not be in the index + self.db.create_doc_from_json('{"no": "key"}') + diff_value_doc = '{"key": "diff value"}' + doc4 = self.db.create_doc_from_json(diff_value_doc) + # This is essentially a 'prefix' match, but we match every entry. + self.assertEqual( + sorted([doc1, doc2, doc4]), + sorted(self.db.get_from_index('test-idx', '*'))) + + def test_get_all_from_index_ordered(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json('{"key": "value x"}') + doc2 = self.db.create_doc_from_json('{"key": "value b"}') + doc3 = self.db.create_doc_from_json('{"key": "value a"}') + doc4 = self.db.create_doc_from_json('{"key": "value m"}') + # This is essentially a 'prefix' match, but we match every entry. + self.assertEqual( + [doc3, doc2, doc4, doc1], self.db.get_from_index('test-idx', '*')) + + def test_put_updates_when_adding_key(self): + doc = self.db.create_doc_from_json("{}") + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + doc.set_json(simple_doc) + self.db.put_doc(doc) + self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) + + def test_get_from_index_empty_string(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + content2 = '{"key": ""}' + doc2 = self.db.create_doc_from_json(content2) + self.assertEqual([doc2], self.db.get_from_index('test-idx', '')) + # Empty string matches the wildcard. + self.assertEqual( + sorted([doc1, doc2]), + sorted(self.db.get_from_index('test-idx', '*'))) + + def test_get_from_index_not_null(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json('{"key": null}') + self.assertEqual([doc1], self.db.get_from_index('test-idx', '*')) + + def test_get_partial_from_index(self): + content1 = '{"k1": "v1", "k2": "v2"}' + content2 = '{"k1": "v1", "k2": "x2"}' + content3 = '{"k1": "v1", "k2": "y2"}' + # doc4 has a different k1 value, so it doesn't match the prefix. + content4 = '{"k1": "NN", "k2": "v2"}' + doc1 = self.db.create_doc_from_json(content1) + doc2 = self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.db.create_index('test-idx', 'k1', 'k2') + self.assertEqual( + sorted([doc1, doc2, doc3]), + sorted(self.db.get_from_index('test-idx', "v1", "*"))) + + def test_get_glob_match(self): + # Note: the exact glob syntax is probably subject to change + content1 = '{"k1": "v1", "k2": "v1"}' + content2 = '{"k1": "v1", "k2": "v2"}' + content3 = '{"k1": "v1", "k2": "v3"}' + # doc4 has a different k2 prefix value, so it doesn't match + content4 = '{"k1": "v1", "k2": "ZZ"}' + self.db.create_index('test-idx', 'k1', 'k2') + doc1 = self.db.create_doc_from_json(content1) + doc2 = self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.assertEqual( + sorted([doc1, doc2, doc3]), + sorted(self.db.get_from_index('test-idx', "v1", "v*"))) + + def test_nested_index(self): + doc = self.db.create_doc_from_json(nested_doc) + self.db.create_index('test-idx', 'sub.doc') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'underneath')) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'underneath'))) + + def test_nested_nonexistent(self): + self.db.create_doc_from_json(nested_doc) + # sub exists, but sub.foo does not: + self.db.create_index('test-idx', 'sub.foo') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + + def test_nested_nonexistent2(self): + self.db.create_doc_from_json(nested_doc) + self.db.create_index('test-idx', 'sub.foo.bar.baz.qux.fnord') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + + def test_nested_traverses_lists(self): + # subpath finds dicts in list + doc = self.db.create_doc_from_json( + '{"foo": [{"zap": "bar"}, {"zap": "baz"}]}') + # subpath only finds dicts in list + self.db.create_doc_from_json('{"foo": ["zap", "baz"]}') + self.db.create_index('test-idx', 'foo.zap') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'bar')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'baz')) + + def test_nested_list_traversal(self): + # subpath finds dicts in list + doc = self.db.create_doc_from_json( + '{"foo": [{"zap": [{"qux": "fnord"}, {"qux": "zombo"}]},' + '{"zap": "baz"}]}') + # subpath only finds dicts in list + self.db.create_index('test-idx', 'foo.zap.qux') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'fnord')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'zombo')) + + def test_index_list1(self): + self.db.create_index("index", "name") + content = '{"name": ["foo", "bar"]}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_list2(self): + self.db.create_index("index", "name") + content = '{"name": ["foo", "bar"]}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_get_from_index_case_sensitive(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + self.assertEqual([], self.db.get_from_index('test-idx', 'V*')) + self.assertEqual([doc1], self.db.get_from_index('test-idx', 'v*')) + + def test_get_from_index_illegal_glob_before_value(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', 'v*', 'v2') + + def test_get_from_index_illegal_glob_after_glob(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', 'v*', 'v*') + + def test_get_from_index_with_sql_wildcards(self): + self.db.create_index('test-idx', 'key') + content1 = '{"key": "va%lue"}' + content2 = '{"key": "value"}' + content3 = '{"key": "va_lue"}' + doc1 = self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + # The '%' in the search should be treated literally, not as a sql + # globbing character. + self.assertEqual([doc1], self.db.get_from_index('test-idx', 'va%*')) + # Same for '_' + self.assertEqual([doc3], self.db.get_from_index('test-idx', 'va_*')) + + def test_get_from_index_with_lower(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "Foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_get_from_index_with_lower_matches_same_case(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_lower_doesnt_match_different_case(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "Foo"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "Foo") + self.assertEqual([], rows) + + def test_index_lower_doesnt_match_other_index(self): + self.db.create_index("index", "lower(name)") + self.db.create_index("other_index", "name") + content = '{"name": "Foo"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "Foo") + self.assertEqual(0, len(rows)) + + def test_index_split_words_match_first(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_match_second(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_split_words_match_both(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_double_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_split_words_leading_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": " foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_trailing_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar "}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_get_from_index_with_number(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 12}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "00012") + self.assertEqual([doc], rows) + + def test_get_from_index_with_number_bigger_than_padding(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 123456}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "123456") + self.assertEqual([doc], rows) + + def test_number_mapping_ignores_non_numbers(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 56}' + doc1 = self.db.create_doc_from_json(content) + content = '{"foo": "this is not a maigret painting"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "*") + self.assertEqual([doc1], rows) + + def test_get_from_index_with_bool(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": true}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "1") + self.assertEqual([doc], rows) + + def test_get_from_index_with_bool_false(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": false}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "0") + self.assertEqual([doc], rows) + + def test_get_from_index_with_non_bool(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": 42}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "*") + self.assertEqual([], rows) + + def test_get_from_index_with_combine(self): + self.db.create_index("index", "combine(foo, bar)") + content = '{"foo": "value1", "bar": "value2"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "value1") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "value2") + self.assertEqual([doc], rows) + + def test_get_complex_combine(self): + self.db.create_index( + "index", "combine(number(foo, 5), lower(bar), split_words(baz))") + content = '{"foo": 12, "bar": "ALLCAPS", "baz": "qux nox"}' + doc = self.db.create_doc_from_json(content) + content = '{"foo": "not a number", "bar": "something"}' + doc2 = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "00012") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "allcaps") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "nox") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "something") + self.assertEqual([doc2], rows) + + def test_get_index_keys_from_index(self): + self.db.create_index('test-idx', 'key') + content1 = '{"key": "value1"}' + content2 = '{"key": "value2"}' + content3 = '{"key": "value2"}' + self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + self.db.create_doc_from_json(content3) + self.assertEqual( + [('value1',), ('value2',)], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_index_keys_from_multicolumn_index(self): + self.db.create_index('test-idx', 'key1', 'key2') + content1 = '{"key1": "value1", "key2": "val2-1"}' + content2 = '{"key1": "value2", "key2": "val2-2"}' + content3 = '{"key1": "value2", "key2": "val2-2"}' + content4 = '{"key1": "value2", "key2": "val3"}' + self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.assertEqual([ + ('value1', 'val2-1'), + ('value2', 'val2-2'), + ('value2', 'val3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_empty_expr(self): + self.assertParseError('') + + def test_nested_unknown_operation(self): + self.assertParseError('unknown_operation(field1)') + + def test_parse_missing_close_paren(self): + self.assertParseError("lower(a") + + def test_parse_trailing_close_paren(self): + self.assertParseError("lower(ab))") + + def test_parse_trailing_chars(self): + self.assertParseError("lower(ab)adsf") + + def test_parse_empty_op(self): + self.assertParseError("(ab)") + + def test_parse_top_level_commas(self): + self.assertParseError("a, b") + + def test_invalid_field_name(self): + self.assertParseError("a.") + + def test_invalid_inner_field_name(self): + self.assertParseError("lower(a.)") + + def test_gobbledigook(self): + self.assertParseError("(@#@cc @#!*DFJSXV(()jccd") + + def test_leading_space(self): + self.assertIndexCreatable(" lower(a)") + + def test_trailing_space(self): + self.assertIndexCreatable("lower(a) ") + + def test_spaces_before_open_paren(self): + self.assertIndexCreatable("lower (a)") + + def test_spaces_after_open_paren(self): + self.assertIndexCreatable("lower( a)") + + def test_spaces_before_close_paren(self): + self.assertIndexCreatable("lower(a )") + + def test_spaces_before_comma(self): + self.assertIndexCreatable("combine(a , b , c)") + + def test_spaces_after_comma(self): + self.assertIndexCreatable("combine(a, b, c)") + + def test_all_together_now(self): + self.assertParseError(' (a) ') + + def test_all_together_now2(self): + self.assertParseError('combine(lower(x)x,foo)') + + +@skip("Skiping tests imported from U1DB.") +class PythonBackendTests(tests.DatabaseBaseTests): + + def setUp(self): + super(PythonBackendTests, self).setUp() + self.simple_doc = json.loads(simple_doc) + + def test_create_doc_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') + self.assertTrue(isinstance(doc, TestAlternativeDocument)) + + def test_get_doc_after_put_with_factory(self): + doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') + self.db.set_document_factory(TestAlternativeDocument) + result = self.db.get_doc('my_doc_id') + self.assertTrue(isinstance(result, TestAlternativeDocument)) + self.assertEqual(doc.doc_id, result.doc_id) + self.assertEqual(doc.rev, result.rev) + self.assertEqual(doc.get_json(), result.get_json()) + self.assertEqual(False, result.has_conflicts) + + def test_get_doc_nonexisting_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.assertIs(None, self.db.get_doc('non-existing')) + + def test_get_all_docs_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.db.create_doc(self.simple_doc) + self.assertTrue(isinstance( + list(self.db.get_all_docs()[1])[0], TestAlternativeDocument)) + + def test_get_docs_conflicted_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + doc1 = self.db.create_doc(self.simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertTrue( + isinstance( + list(self.db.get_docs([doc1.doc_id]))[0], + TestAlternativeDocument)) + + def test_get_from_index_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.db.create_doc(self.simple_doc) + self.db.create_index('test-idx', 'key') + self.assertTrue( + isinstance( + self.db.get_from_index('test-idx', 'value')[0], + TestAlternativeDocument)) + + def test_sync_exchange_updates_indexes(self): + doc = self.db.create_doc(self.simple_doc) + self.db.create_index('test-idx', 'key') + new_content = '{"key": "altval"}' + other_rev = 'test:1|z:2' + st = self.db.get_sync_target() + + def ignore(doc_id, doc_rev, doc): + pass + + doc_other = self.make_document(doc.doc_id, other_rev, new_content) + docs_by_gen = [(doc_other, 10, 'T-sid')] + st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=ignore) + self.assertGetDoc(self.db, doc.doc_id, other_rev, new_content, False) + self.assertEqual( + [doc_other], self.db.get_from_index('test-idx', 'altval')) + self.assertEqual([], self.db.get_from_index('test-idx', 'value')) + + +# Use a custom loader to apply the scenarios at load time. +load_tests = tests.load_with_scenarios diff --git a/tests/test_soledad/u1db_tests/test_document.py b/tests/test_soledad/u1db_tests/test_document.py new file mode 100644 index 00000000..a7ead2d1 --- /dev/null +++ b/tests/test_soledad/u1db_tests/test_document.py @@ -0,0 +1,153 @@ +# Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of leap.soledad.common +# +# leap.soledad.common is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . +from unittest import skip + +from leap.soledad.common.l2db import errors + +from test_soledad import u1db_tests as tests + + +@skip("Skiping tests imported from U1DB.") +class TestDocument(tests.TestCase): + + scenarios = ([( + 'py', {'make_document_for_test': tests.make_document_for_test})]) # + + # tests.C_DATABASE_SCENARIOS) + + def test_create_doc(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) + self.assertEqual('doc-id', doc.doc_id) + self.assertEqual('uid:1', doc.rev) + self.assertEqual(tests.simple_doc, doc.get_json()) + self.assertFalse(doc.has_conflicts) + + def test__repr__(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) + self.assertEqual( + '%s(doc-id, uid:1, \'{"key": "value"}\')' + % (doc.__class__.__name__,), + repr(doc)) + + def test__repr__conflicted(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc, + has_conflicts=True) + self.assertEqual( + '%s(doc-id, uid:1, conflicted, \'{"key": "value"}\')' + % (doc.__class__.__name__,), + repr(doc)) + + def test__lt__(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('b', 'b', '{}') + self.assertTrue(doc_a < doc_b) + self.assertTrue(doc_b > doc_a) + doc_aa = self.make_document('a', 'a', '{}') + self.assertTrue(doc_aa < doc_a) + + def test__eq__(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('a', 'b', '{}') + self.assertTrue(doc_a == doc_b) + doc_b = self.make_document('a', 'b', '{}', has_conflicts=True) + self.assertFalse(doc_a == doc_b) + + def test_non_json_dict(self): + self.assertRaises( + errors.InvalidJSON, self.make_document, 'id', 'uid:1', + '"not a json dictionary"') + + def test_non_json(self): + self.assertRaises( + errors.InvalidJSON, self.make_document, 'id', 'uid:1', + 'not a json dictionary') + + def test_get_size(self): + doc_a = self.make_document('a', 'b', '{"some": "content"}') + self.assertEqual( + len('a' + 'b' + '{"some": "content"}'), doc_a.get_size()) + + def test_get_size_empty_document(self): + doc_a = self.make_document('a', 'b', None) + self.assertEqual(len('a' + 'b'), doc_a.get_size()) + + +@skip("Skiping tests imported from U1DB.") +class TestPyDocument(tests.TestCase): + + scenarios = ([( + 'py', {'make_document_for_test': tests.make_document_for_test})]) + + def test_get_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertEqual({"content": ""}, doc.content) + doc.set_json('{"content": "new"}') + self.assertEqual({"content": "new"}, doc.content) + + def test_set_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + doc.content = {"content": "new"} + self.assertEqual('{"content": "new"}', doc.get_json()) + + def test_set_bad_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises( + errors.InvalidContent, setattr, doc, 'content', + '{"content": "new"}') + + def test_is_tombstone(self): + doc_a = self.make_document('a', 'b', '{}') + self.assertFalse(doc_a.is_tombstone()) + doc_a.set_json(None) + self.assertTrue(doc_a.is_tombstone()) + + def test_make_tombstone(self): + doc_a = self.make_document('a', 'b', '{}') + self.assertFalse(doc_a.is_tombstone()) + doc_a.make_tombstone() + self.assertTrue(doc_a.is_tombstone()) + + def test_same_content_as(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('d', 'e', '{}') + self.assertTrue(doc_a.same_content_as(doc_b)) + doc_b = self.make_document('p', 'q', '{}', has_conflicts=True) + self.assertTrue(doc_a.same_content_as(doc_b)) + doc_b.content['key'] = 'value' + self.assertFalse(doc_a.same_content_as(doc_b)) + + def test_same_content_as_json_order(self): + doc_a = self.make_document( + 'a', 'b', '{"key1": "val1", "key2": "val2"}') + doc_b = self.make_document( + 'c', 'd', '{"key2": "val2", "key1": "val1"}') + self.assertTrue(doc_a.same_content_as(doc_b)) + + def test_set_json(self): + doc = self.make_document('id', 'rev', '{"content":""}') + doc.set_json('{"content": "new"}') + self.assertEqual('{"content": "new"}', doc.get_json()) + + def test_set_json_non_dict(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises(errors.InvalidJSON, doc.set_json, '"is not a dict"') + + def test_set_json_error(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises(errors.InvalidJSON, doc.set_json, 'is not json') + + +load_tests = tests.load_with_scenarios diff --git a/tests/test_soledad/u1db_tests/test_http_client.py b/tests/test_soledad/u1db_tests/test_http_client.py new file mode 100644 index 00000000..e9516236 --- /dev/null +++ b/tests/test_soledad/u1db_tests/test_http_client.py @@ -0,0 +1,304 @@ +# Copyright 2011-2012 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +""" +Tests for HTTPDatabase +""" +import json + +from unittest import skip + +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db.remote import http_client + +from test_soledad import u1db_tests as tests + + +@skip("Skiping tests imported from U1DB.") +class TestEncoder(tests.TestCase): + + def test_encode_string(self): + self.assertEqual("foo", http_client._encode_query_parameter("foo")) + + def test_encode_true(self): + self.assertEqual("true", http_client._encode_query_parameter(True)) + + def test_encode_false(self): + self.assertEqual("false", http_client._encode_query_parameter(False)) + + +@skip("Skiping tests imported from U1DB.") +class TestHTTPClientBase(tests.TestCaseWithServer): + + def setUp(self): + super(TestHTTPClientBase, self).setUp() + self.errors = 0 + + def app(self, environ, start_response): + if environ['PATH_INFO'].endswith('echo'): + start_response("200 OK", [('Content-Type', 'application/json')]) + ret = {} + for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): + ret[name] = environ[name] + if environ['REQUEST_METHOD'] in ('PUT', 'POST'): + ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] + content_length = int(environ['CONTENT_LENGTH']) + ret['body'] = environ['wsgi.input'].read(content_length) + return [json.dumps(ret)] + elif environ['PATH_INFO'].endswith('error_then_accept'): + if self.errors >= 3: + start_response( + "200 OK", [('Content-Type', 'application/json')]) + ret = {} + for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): + ret[name] = environ[name] + if environ['REQUEST_METHOD'] in ('PUT', 'POST'): + ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] + content_length = int(environ['CONTENT_LENGTH']) + ret['body'] = '{"oki": "doki"}' + return [json.dumps(ret)] + self.errors += 1 + content_length = int(environ['CONTENT_LENGTH']) + error = json.loads( + environ['wsgi.input'].read(content_length)) + response = error['response'] + # In debug mode, wsgiref has an assertion that the status parameter + # is a 'str' object. However error['status'] returns a unicode + # object. + status = str(error['status']) + if isinstance(response, unicode): + response = str(response) + if isinstance(response, str): + start_response(status, [('Content-Type', 'text/plain')]) + return [str(response)] + else: + start_response(status, [('Content-Type', 'application/json')]) + return [json.dumps(response)] + elif environ['PATH_INFO'].endswith('error'): + self.errors += 1 + content_length = int(environ['CONTENT_LENGTH']) + error = json.loads( + environ['wsgi.input'].read(content_length)) + response = error['response'] + # In debug mode, wsgiref has an assertion that the status parameter + # is a 'str' object. However error['status'] returns a unicode + # object. + status = str(error['status']) + if isinstance(response, unicode): + response = str(response) + if isinstance(response, str): + start_response(status, [('Content-Type', 'text/plain')]) + return [str(response)] + else: + start_response(status, [('Content-Type', 'application/json')]) + return [json.dumps(response)] + + def make_app(self): + return self.app + + def getClient(self, **kwds): + self.startServer() + return http_client.HTTPClientBase(self.getURL('dbase'), **kwds) + + def test_construct(self): + self.startServer() + url = self.getURL() + cli = http_client.HTTPClientBase(url) + self.assertEqual(url, cli._url.geturl()) + self.assertIs(None, cli._conn) + + def test_parse_url(self): + cli = http_client.HTTPClientBase( + '%s://127.0.0.1:12345/' % self.url_scheme) + self.assertEqual(self.url_scheme, cli._url.scheme) + self.assertEqual('127.0.0.1', cli._url.hostname) + self.assertEqual(12345, cli._url.port) + self.assertEqual('/', cli._url.path) + + def test__ensure_connection(self): + cli = self.getClient() + self.assertIs(None, cli._conn) + cli._ensure_connection() + self.assertIsNot(None, cli._conn) + conn = cli._conn + cli._ensure_connection() + self.assertIs(conn, cli._conn) + + def test_close(self): + cli = self.getClient() + cli._ensure_connection() + cli.close() + self.assertIs(None, cli._conn) + + def test__request(self): + cli = self.getClient() + res, headers = cli._request('PUT', ['echo'], {}, {}) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': '', + 'body': '{}', + 'REQUEST_METHOD': 'PUT'}, json.loads(res)) + + res, headers = cli._request('GET', ['doc', 'echo'], {'a': 1}) + self.assertEqual({'PATH_INFO': '/dbase/doc/echo', + 'QUERY_STRING': 'a=1', + 'REQUEST_METHOD': 'GET'}, json.loads(res)) + + res, headers = cli._request('GET', ['doc', '%FFFF', 'echo'], {'a': 1}) + self.assertEqual({'PATH_INFO': '/dbase/doc/%FFFF/echo', + 'QUERY_STRING': 'a=1', + 'REQUEST_METHOD': 'GET'}, json.loads(res)) + + res, headers = cli._request('POST', ['echo'], {'b': 2}, 'Body', + 'application/x-test') + self.assertEqual({'CONTENT_TYPE': 'application/x-test', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': 'b=2', + 'body': 'Body', + 'REQUEST_METHOD': 'POST'}, json.loads(res)) + + def test__request_json(self): + cli = self.getClient() + res, headers = cli._request_json( + 'POST', ['echo'], {'b': 2}, {'a': 'x'}) + self.assertEqual('application/json', headers['content-type']) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': 'b=2', + 'body': '{"a": "x"}', + 'REQUEST_METHOD': 'POST'}, res) + + def test_unspecified_http_error(self): + cli = self.getClient() + self.assertRaises(errors.HTTPError, + cli._request_json, 'POST', ['error'], {}, + {'status': "500 Internal Error", + 'response': "Crash."}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "500 Internal Error", + 'response': "Fail."}) + except errors.HTTPError, e: + pass + + self.assertEqual(500, e.status) + self.assertEqual("Fail.", e.message) + self.assertTrue("content-type" in e.headers) + + def test_revision_conflict(self): + cli = self.getClient() + self.assertRaises(errors.RevisionConflict, + cli._request_json, 'POST', ['error'], {}, + {'status': "409 Conflict", + 'response': {"error": "revision conflict"}}) + + def test_unavailable_proper(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + self.assertRaises(errors.Unavailable, + cli._request_json, 'POST', ['error'], {}, + {'status': "503 Service Unavailable", + 'response': {"error": "unavailable"}}) + self.assertEqual(5, self.errors) + + def test_unavailable_then_available(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + res, headers = cli._request_json( + 'POST', ['error_then_accept'], {'b': 2}, + {'status': "503 Service Unavailable", + 'response': {"error": "unavailable"}}) + self.assertEqual('application/json', headers['content-type']) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/error_then_accept', + 'QUERY_STRING': 'b=2', + 'body': '{"oki": "doki"}', + 'REQUEST_METHOD': 'POST'}, res) + self.assertEqual(3, self.errors) + + def test_unavailable_random_source(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + try: + cli._request_json('POST', ['error'], {}, + {'status': "503 Service Unavailable", + 'response': "random unavailable."}) + except errors.Unavailable, e: + pass + + self.assertEqual(503, e.status) + self.assertEqual("random unavailable.", e.message) + self.assertTrue("content-type" in e.headers) + self.assertEqual(5, self.errors) + + def test_document_too_big(self): + cli = self.getClient() + self.assertRaises(errors.DocumentTooBig, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "document too big"}}) + + def test_user_quota_exceeded(self): + cli = self.getClient() + self.assertRaises(errors.UserQuotaExceeded, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "user quota exceeded"}}) + + def test_user_needs_subscription(self): + cli = self.getClient() + self.assertRaises(errors.SubscriptionNeeded, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "user needs subscription"}}) + + def test_generic_u1db_error(self): + cli = self.getClient() + self.assertRaises(errors.U1DBError, + cli._request_json, 'POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': {"error": "error"}}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': {"error": "error"}}) + except errors.U1DBError, e: + pass + self.assertIs(e.__class__, errors.U1DBError) + + def test_unspecified_bad_request(self): + cli = self.getClient() + self.assertRaises(errors.HTTPError, + cli._request_json, 'POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': ""}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': ""}) + except errors.HTTPError, e: + pass + + self.assertEqual(400, e.status) + self.assertEqual("", e.message) + self.assertTrue("content-type" in e.headers) + + def test_unknown_creds(self): + self.assertRaises(errors.UnknownAuthMethod, + self.getClient, creds={'foo': {}}) + self.assertRaises(errors.UnknownAuthMethod, + self.getClient, creds={}) diff --git a/tests/test_soledad/u1db_tests/test_http_database.py b/tests/test_soledad/u1db_tests/test_http_database.py new file mode 100644 index 00000000..a3ed9361 --- /dev/null +++ b/tests/test_soledad/u1db_tests/test_http_database.py @@ -0,0 +1,233 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Tests for HTTPDatabase""" + +import inspect +import json + +from unittest import skip + +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import Document +from leap.soledad.common.l2db.remote import http_database +from leap.soledad.common.l2db.remote import http_target +from test_soledad import u1db_tests as tests +from test_soledad.u1db_tests import make_http_app + + +@skip("Skiping tests imported from U1DB.") +class TestHTTPDatabaseSimpleOperations(tests.TestCase): + + def setUp(self): + super(TestHTTPDatabaseSimpleOperations, self).setUp() + self.db = http_database.HTTPDatabase('dbase') + self.db._conn = object() # crash if used + self.got = None + self.response_val = None + + def _request(method, url_parts, params=None, body=None, + content_type=None): + self.got = method, url_parts, params, body, content_type + if isinstance(self.response_val, Exception): + raise self.response_val + return self.response_val + + def _request_json(method, url_parts, params=None, body=None, + content_type=None): + self.got = method, url_parts, params, body, content_type + if isinstance(self.response_val, Exception): + raise self.response_val + return self.response_val + + self.db._request = _request + self.db._request_json = _request_json + + def test__sanity_same_signature(self): + my_request_sig = inspect.getargspec(self.db._request) + my_request_sig = (['self'] + my_request_sig[0],) + my_request_sig[1:] + self.assertEqual( + my_request_sig, + inspect.getargspec(http_database.HTTPDatabase._request)) + my_request_json_sig = inspect.getargspec(self.db._request_json) + my_request_json_sig = ((['self'] + my_request_json_sig[0],) + + my_request_json_sig[1:]) + self.assertEqual( + my_request_json_sig, + inspect.getargspec(http_database.HTTPDatabase._request_json)) + + def test__ensure(self): + self.response_val = {'ok': True}, {} + self.db._ensure() + self.assertEqual(('PUT', [], {}, {}, None), self.got) + + def test__delete(self): + self.response_val = {'ok': True}, {} + self.db._delete() + self.assertEqual(('DELETE', [], {}, {}, None), self.got) + + def test__check(self): + self.response_val = {}, {} + res = self.db._check() + self.assertEqual({}, res) + self.assertEqual(('GET', [], None, None, None), self.got) + + def test_put_doc(self): + self.response_val = {'rev': 'doc-rev'}, {} + doc = Document('doc-id', None, '{"v": 1}') + res = self.db.put_doc(doc) + self.assertEqual('doc-rev', res) + self.assertEqual('doc-rev', doc.rev) + self.assertEqual(('PUT', ['doc', 'doc-id'], {}, + '{"v": 1}', 'application/json'), self.got) + + self.response_val = {'rev': 'doc-rev-2'}, {} + doc.content = {"v": 2} + res = self.db.put_doc(doc) + self.assertEqual('doc-rev-2', res) + self.assertEqual('doc-rev-2', doc.rev) + self.assertEqual(('PUT', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, + '{"v": 2}', 'application/json'), self.got) + + def test_get_doc(self): + self.response_val = '{"v": 2}', {'x-u1db-rev': 'doc-rev', + 'x-u1db-has-conflicts': 'false'} + self.assertGetDoc(self.db, 'doc-id', 'doc-rev', '{"v": 2}', False) + self.assertEqual( + ('GET', ['doc', 'doc-id'], {'include_deleted': False}, None, None), + self.got) + + def test_get_doc_non_existing(self): + self.response_val = errors.DocumentDoesNotExist() + self.assertIs(None, self.db.get_doc('not-there')) + self.assertEqual( + ('GET', ['doc', 'not-there'], {'include_deleted': False}, None, + None), self.got) + + def test_get_doc_deleted(self): + self.response_val = errors.DocumentDoesNotExist() + self.assertIs(None, self.db.get_doc('deleted')) + self.assertEqual( + ('GET', ['doc', 'deleted'], {'include_deleted': False}, None, + None), self.got) + + def test_get_doc_deleted_include_deleted(self): + self.response_val = errors.HTTPError( + 404, + json.dumps({"error": errors.DOCUMENT_DELETED}), + {'x-u1db-rev': 'doc-rev-gone', + 'x-u1db-has-conflicts': 'false'}) + doc = self.db.get_doc('deleted', include_deleted=True) + self.assertEqual('deleted', doc.doc_id) + self.assertEqual('doc-rev-gone', doc.rev) + self.assertIs(None, doc.content) + self.assertEqual( + ('GET', ['doc', 'deleted'], {'include_deleted': True}, None, None), + self.got) + + def test_get_doc_pass_through_errors(self): + self.response_val = errors.HTTPError(500, 'Crash.') + self.assertRaises(errors.HTTPError, + self.db.get_doc, 'something-something') + + def test_create_doc_with_id(self): + self.response_val = {'rev': 'doc-rev'}, {} + new_doc = self.db.create_doc_from_json('{"v": 1}', doc_id='doc-id') + self.assertEqual('doc-rev', new_doc.rev) + self.assertEqual('doc-id', new_doc.doc_id) + self.assertEqual('{"v": 1}', new_doc.get_json()) + self.assertEqual(('PUT', ['doc', 'doc-id'], {}, + '{"v": 1}', 'application/json'), self.got) + + def test_create_doc_without_id(self): + self.response_val = {'rev': 'doc-rev-2'}, {} + new_doc = self.db.create_doc_from_json('{"v": 3}') + self.assertEqual('D-', new_doc.doc_id[:2]) + self.assertEqual('doc-rev-2', new_doc.rev) + self.assertEqual('{"v": 3}', new_doc.get_json()) + self.assertEqual(('PUT', ['doc', new_doc.doc_id], {}, + '{"v": 3}', 'application/json'), self.got) + + def test_delete_doc(self): + self.response_val = {'rev': 'doc-rev-gone'}, {} + doc = Document('doc-id', 'doc-rev', None) + self.db.delete_doc(doc) + self.assertEqual('doc-rev-gone', doc.rev) + self.assertEqual(('DELETE', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, + None, None), self.got) + + def test_get_sync_target(self): + st = self.db.get_sync_target() + self.assertIsInstance(st, http_target.HTTPSyncTarget) + self.assertEqual(st._url, self.db._url) + + +@skip("Skiping tests imported from U1DB.") +class TestHTTPDatabaseIntegration(tests.TestCaseWithServer): + + make_app_with_state = staticmethod(make_http_app) + + def setUp(self): + super(TestHTTPDatabaseIntegration, self).setUp() + self.startServer() + + def test_non_existing_db(self): + db = http_database.HTTPDatabase(self.getURL('not-there')) + self.assertRaises(errors.DatabaseDoesNotExist, db.get_doc, 'doc1') + + def test__ensure(self): + db = http_database.HTTPDatabase(self.getURL('new')) + db._ensure() + self.assertIs(None, db.get_doc('doc1')) + + def test__delete(self): + self.request_state._create_database('db0') + db = http_database.HTTPDatabase(self.getURL('db0')) + db._delete() + self.assertRaises(errors.DatabaseDoesNotExist, + self.request_state.check_database, 'db0') + + def test_open_database_existing(self): + self.request_state._create_database('db0') + db = http_database.HTTPDatabase.open_database(self.getURL('db0'), + create=False) + self.assertIs(None, db.get_doc('doc1')) + + def test_open_database_non_existing(self): + self.assertRaises(errors.DatabaseDoesNotExist, + http_database.HTTPDatabase.open_database, + self.getURL('not-there'), + create=False) + + def test_open_database_create(self): + db = http_database.HTTPDatabase.open_database(self.getURL('new'), + create=True) + self.assertIs(None, db.get_doc('doc1')) + + def test_delete_database_existing(self): + self.request_state._create_database('db0') + http_database.HTTPDatabase.delete_database(self.getURL('db0')) + self.assertRaises(errors.DatabaseDoesNotExist, + self.request_state.check_database, 'db0') + + def test_doc_ids_needing_quoting(self): + db0 = self.request_state._create_database('db0') + db = http_database.HTTPDatabase.open_database(self.getURL('db0'), + create=False) + doc = Document('%fff', None, '{}') + db.put_doc(doc) + self.assertGetDoc(db0, '%fff', doc.rev, '{}', False) + self.assertGetDoc(db, '%fff', doc.rev, '{}', False) diff --git a/tests/test_soledad/u1db_tests/test_https.py b/tests/test_soledad/u1db_tests/test_https.py new file mode 100644 index 00000000..2e75afd1 --- /dev/null +++ b/tests/test_soledad/u1db_tests/test_https.py @@ -0,0 +1,105 @@ +"""Test support for client-side https support.""" + +import os +import ssl +import sys + +from paste import httpserver +from unittest import skip + +from leap.soledad.common.l2db.remote import http_client + +from leap import soledad +from test_soledad import u1db_tests as tests + + +def https_server_def(): + def make_server(host_port, application): + from OpenSSL import SSL + cert_file = os.path.join(os.path.dirname(__file__), 'testing-certs', + 'testing.cert') + key_file = os.path.join(os.path.dirname(__file__), 'testing-certs', + 'testing.key') + ssl_context = SSL.Context(SSL.SSLv23_METHOD) + ssl_context.use_privatekey_file(key_file) + ssl_context.use_certificate_chain_file(cert_file) + srv = httpserver.WSGIServerBase(application, host_port, + httpserver.WSGIHandler, + ssl_context=ssl_context + ) + + def shutdown_request(req): + req.shutdown() + srv.close_request(req) + + srv.shutdown_request = shutdown_request + application.base_url = "https://localhost:%s" % srv.server_address[1] + return srv + return make_server, "shutdown", "https" + + +@skip("Skiping tests imported from U1DB.") +class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): + + scenarios = [] + + def setUp(self): + try: + import OpenSSL # noqa + except ImportError: + self.skipTest("Requires pyOpenSSL") + self.cacert_pem = os.path.join(os.path.dirname(__file__), + 'testing-certs', 'cacert.pem') + # The default u1db http_client class for doing HTTPS only does HTTPS + # if the platform is linux. Because of this, soledad replaces that + # class with one that will do HTTPS independent of the platform. In + # order to maintain the compatibility with u1db default tests, we undo + # that replacement here. + http_client._VerifiedHTTPSConnection = \ + soledad.client.api.old__VerifiedHTTPSConnection + super(TestHttpSyncTargetHttpsSupport, self).setUp() + + def getSyncTarget(self, host, path=None, cert_file=None): + if self.server is None: + self.startServer() + return self.sync_target(self, host, path, cert_file=cert_file) + + def test_working(self): + self.startServer() + db = self.request_state._create_database('test') + self.patch(http_client, 'CA_CERTS', self.cacert_pem) + remote_target = self.getSyncTarget('localhost', 'test') + remote_target.record_sync_info('other-id', 2, 'T-id') + self.assertEqual( + (2, 'T-id'), db._get_replica_gen_and_trans_id('other-id')) + + def test_cannot_verify_cert(self): + if not sys.platform.startswith('linux'): + self.skipTest( + "XXX certificate verification happens on linux only for now") + self.startServer() + # don't print expected traceback server-side + self.server.handle_error = lambda req, cli_addr: None + self.request_state._create_database('test') + remote_target = self.getSyncTarget('localhost', 'test') + try: + remote_target.record_sync_info('other-id', 2, 'T-id') + except ssl.SSLError as e: + self.assertIn("certificate verify failed", str(e)) + else: + self.fail("certificate verification should have failed.") + + def test_host_mismatch(self): + if not sys.platform.startswith('linux'): + self.skipTest( + "XXX certificate verification happens on linux only for now") + self.startServer() + self.request_state._create_database('test') + self.patch(http_client, 'CA_CERTS', self.cacert_pem) + remote_target = self.getSyncTarget('127.0.0.1', 'test') + self.assertRaises( + http_client.CertificateError, remote_target.record_sync_info, + 'other-id', 2, 'T-id') + + +load_tests = tests.load_with_scenarios diff --git a/tests/test_soledad/u1db_tests/test_open.py b/tests/test_soledad/u1db_tests/test_open.py new file mode 100644 index 00000000..4ca0c4a7 --- /dev/null +++ b/tests/test_soledad/u1db_tests/test_open.py @@ -0,0 +1,74 @@ +# Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Test u1db.open""" + +import os +import pytest + +from unittest import skip + +from test_soledad import u1db_tests as tests +from test_soledad.u1db_tests.test_backends import TestAlternativeDocument + +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import open as u1db_open + +from leap.soledad.client._db import sqlite + + +@skip("Skiping tests imported from U1DB.") +@pytest.mark.usefixtures('method_tmpdir') +class TestU1DBOpen(tests.TestCase): + + def setUp(self): + super(TestU1DBOpen, self).setUp() + self.db_path = self.tempdir + '/test.db' + + def test_open_no_create(self): + self.assertRaises(errors.DatabaseDoesNotExist, + u1db_open, self.db_path, create=False) + self.assertFalse(os.path.exists(self.db_path)) + + def test_open_create(self): + db = u1db_open(self.db_path, create=True) + self.addCleanup(db.close) + self.assertTrue(os.path.exists(self.db_path)) + self.assertIsInstance(db, sqlite.SQLiteDatabase) + + def test_open_with_factory(self): + db = u1db_open(self.db_path, create=True, + document_factory=TestAlternativeDocument) + self.addCleanup(db.close) + self.assertEqual(TestAlternativeDocument, db._factory) + + def test_open_existing(self): + db = sqlite.SQLitePartialExpandDatabase(self.db_path) + self.addCleanup(db.close) + doc = db.create_doc_from_json(tests.simple_doc) + # Even though create=True, we shouldn't wipe the db + db2 = u1db_open(self.db_path, create=True) + self.addCleanup(db2.close) + doc2 = db2.get_doc(doc.doc_id) + self.assertEqual(doc, doc2) + + def test_open_existing_no_create(self): + db = sqlite.SQLitePartialExpandDatabase(self.db_path) + self.addCleanup(db.close) + db2 = u1db_open(self.db_path, create=False) + self.addCleanup(db2.close) + self.assertIsInstance(db2, sqlite.SQLitePartialExpandDatabase) diff --git a/tests/test_soledad/u1db_tests/testing-certs/Makefile b/tests/test_soledad/u1db_tests/testing-certs/Makefile new file mode 100644 index 00000000..2385e75b --- /dev/null +++ b/tests/test_soledad/u1db_tests/testing-certs/Makefile @@ -0,0 +1,35 @@ +CATOP=./demoCA +ORIG_CONF=/usr/lib/ssl/openssl.cnf +ELEVEN_YEARS=-days 4015 + +init: + cp $(ORIG_CONF) ca.conf + install -d $(CATOP) + install -d $(CATOP)/certs + install -d $(CATOP)/crl + install -d $(CATOP)/newcerts + install -d $(CATOP)/private + touch $(CATOP)/index.txt + echo 01>$(CATOP)/crlnumber + @echo '**** Making CA certificate ...' + openssl req -nodes -new \ + -newkey rsa -keyout $(CATOP)/private/cakey.pem \ + -out $(CATOP)/careq.pem \ + -multivalue-rdn \ + -subj "/C=UK/ST=-/O=u1db LOCAL TESTING ONLY, DO NO TRUST/CN=u1db testing CA" + openssl ca -config ./ca.conf -create_serial \ + -out $(CATOP)/cacert.pem $(ELEVEN_YEARS) -batch \ + -keyfile $(CATOP)/private/cakey.pem -selfsign \ + -extensions v3_ca -infiles $(CATOP)/careq.pem + +pems: + cp ./demoCA/cacert.pem . + openssl req -new -config ca.conf \ + -multivalue-rdn \ + -subj "/O=u1db LOCAL TESTING ONLY, DO NOT TRUST/CN=localhost" \ + -nodes -keyout testing.key -out newreq.pem $(ELEVEN_YEARS) + openssl ca -batch -config ./ca.conf $(ELEVEN_YEARS) \ + -policy policy_anything \ + -out testing.cert -infiles newreq.pem + +.PHONY: init pems diff --git a/tests/test_soledad/u1db_tests/testing-certs/cacert.pem b/tests/test_soledad/u1db_tests/testing-certs/cacert.pem new file mode 100644 index 00000000..c019a730 --- /dev/null +++ b/tests/test_soledad/u1db_tests/testing-certs/cacert.pem @@ -0,0 +1,58 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + e4:de:01:76:c4:78:78:7e + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA + Validity + Not Before: May 3 11:11:11 2012 GMT + Not After : May 1 11:11:11 2023 GMT + Subject: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:bc:91:a5:7f:7d:37:f7:06:c7:db:5b:83:6a:6b: + 63:c3:8b:5c:f7:84:4d:97:6d:d4:be:bf:e7:79:a8: + c1:03:57:ec:90:d4:20:e7:02:95:d9:a6:49:e3:f9: + 9a:ea:37:b9:b2:02:62:ab:40:d3:42:bb:4a:4e:a2: + 47:71:0f:1d:a2:c5:94:a1:cf:35:d3:23:32:42:c0: + 1e:8d:cb:08:58:fb:8a:5c:3e:ea:eb:d5:2c:ed:d6: + aa:09:b4:b5:7d:e3:45:c9:ae:c2:82:b2:ae:c0:81: + bc:24:06:65:a9:e7:e0:61:ac:25:ee:53:d3:d7:be: + 22:f7:00:a2:ad:c6:0e:3a:39 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D + X509v3 Authority Key Identifier: + keyid:DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 72:9b:c1:f7:07:65:83:36:25:4e:01:2f:b7:4a:f2:a4:00:28: + 80:c7:56:2c:32:39:90:13:61:4b:bb:12:c5:44:9d:42:57:85: + 28:19:70:69:e1:43:c8:bd:11:f6:94:df:91:2d:c3:ea:82:8d: + b4:8f:5d:47:a3:00:99:53:29:93:27:6c:c5:da:c1:20:6f:ab: + ec:4a:be:34:f3:8f:02:e5:0c:c0:03:ac:2b:33:41:71:4f:0a: + 72:5a:b4:26:1a:7f:81:bc:c0:95:8a:06:87:a8:11:9f:5c:73: + 38:df:5a:69:40:21:29:ad:46:23:56:75:e1:e9:8b:10:18:4c: + 7b:54 +-----BEGIN CERTIFICATE----- +MIICkjCCAfugAwIBAgIJAOTeAXbEeHh+MA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlVLMQowCAYDVQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcg +T05MWSwgRE8gTk8gVFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTAeFw0x +MjA1MDMxMTExMTFaFw0yMzA1MDExMTExMTFaMGIxCzAJBgNVBAYTAlVLMQowCAYD +VQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcgT05MWSwgRE8gTk8g +VFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTCBnzANBgkqhkiG9w0BAQEF +AAOBjQAwgYkCgYEAvJGlf3039wbH21uDamtjw4tc94RNl23Uvr/neajBA1fskNQg +5wKV2aZJ4/ma6je5sgJiq0DTQrtKTqJHcQ8dosWUoc810yMyQsAejcsIWPuKXD7q +69Us7daqCbS1feNFya7CgrKuwIG8JAZlqefgYawl7lPT174i9wCircYOOjkCAwEA +AaNQME4wHQYDVR0OBBYEFNs9k1FsMhVUjxBQ/ElPNhUou5VtMB8GA1UdIwQYMBaA +FNs9k1FsMhVUjxBQ/ElPNhUou5VtMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEF +BQADgYEAcpvB9wdlgzYlTgEvt0rypAAogMdWLDI5kBNhS7sSxUSdQleFKBlwaeFD +yL0R9pTfkS3D6oKNtI9dR6MAmVMpkydsxdrBIG+r7Eq+NPOPAuUMwAOsKzNBcU8K +clq0Jhp/gbzAlYoGh6gRn1xzON9aaUAhKa1GI1Z14emLEBhMe1Q= +-----END CERTIFICATE----- diff --git a/tests/test_soledad/u1db_tests/testing-certs/testing.cert b/tests/test_soledad/u1db_tests/testing-certs/testing.cert new file mode 100644 index 00000000..985684fb --- /dev/null +++ b/tests/test_soledad/u1db_tests/testing-certs/testing.cert @@ -0,0 +1,61 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + e4:de:01:76:c4:78:78:7f + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA + Validity + Not Before: May 3 11:11:14 2012 GMT + Not After : May 1 11:11:14 2023 GMT + Subject: O=u1db LOCAL TESTING ONLY, DO NOT TRUST, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:c6:1d:72:d3:c5:e4:fc:d1:4c:d9:e4:08:3e:90: + 10:ce:3f:1f:87:4a:1d:4f:7f:2a:5a:52:c9:65:4f: + d9:2c:bf:69:75:18:1a:b5:c9:09:32:00:47:f5:60: + aa:c6:dd:3a:87:37:5f:16:be:de:29:b5:ea:fc:41: + 7e:eb:77:bb:df:63:c3:06:1e:ed:e9:a0:67:1a:f1: + ec:e1:9d:f7:9c:8f:1c:fa:c3:66:7b:39:dc:70:ae: + 09:1b:9c:c0:9a:c4:90:77:45:8e:39:95:a9:2f:92: + 43:bd:27:07:5a:99:51:6e:76:a0:af:dd:b1:2c:8f: + ca:8b:8c:47:0d:f6:6e:fc:69 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + OpenSSL Generated Certificate + X509v3 Subject Key Identifier: + 1C:63:85:E1:1D:F3:89:2E:6C:4E:3F:FB:D0:10:64:5A:C1:22:6A:2A + X509v3 Authority Key Identifier: + keyid:DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D + + Signature Algorithm: sha1WithRSAEncryption + 1d:6d:3e:bd:93:fd:bd:3e:17:b8:9f:f0:99:7f:db:50:5c:b2: + 01:42:03:b5:d5:94:05:d3:f6:8e:80:82:55:47:1f:58:f2:18: + 6c:ab:ef:43:2c:2f:10:e1:7c:c4:5c:cc:ac:50:50:22:42:aa: + 35:33:f5:b9:f3:a6:66:55:d9:36:f4:f2:e4:d4:d9:b5:2c:52: + 66:d4:21:17:97:22:b8:9b:d7:0e:7c:3d:ce:85:19:ca:c4:d2: + 58:62:31:c6:18:3e:44:fc:f4:30:b6:95:87:ee:21:4a:08:f0: + af:3c:8f:c4:ba:5e:a1:5c:37:1a:7d:7b:fe:66:ae:62:50:17: + 31:ca +-----BEGIN CERTIFICATE----- +MIICnzCCAgigAwIBAgIJAOTeAXbEeHh/MA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlVLMQowCAYDVQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcg +T05MWSwgRE8gTk8gVFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTAeFw0x +MjA1MDMxMTExMTRaFw0yMzA1MDExMTExMTRaMEQxLjAsBgNVBAoMJXUxZGIgTE9D +QUwgVEVTVElORyBPTkxZLCBETyBOT1QgVFJVU1QxEjAQBgNVBAMMCWxvY2FsaG9z +dDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxh1y08Xk/NFM2eQIPpAQzj8f +h0odT38qWlLJZU/ZLL9pdRgatckJMgBH9WCqxt06hzdfFr7eKbXq/EF+63e732PD +Bh7t6aBnGvHs4Z33nI8c+sNmeznccK4JG5zAmsSQd0WOOZWpL5JDvScHWplRbnag +r92xLI/Ki4xHDfZu/GkCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0E +HxYdT3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFBxjheEd +84kubE4/+9AQZFrBImoqMB8GA1UdIwQYMBaAFNs9k1FsMhVUjxBQ/ElPNhUou5Vt +MA0GCSqGSIb3DQEBBQUAA4GBAB1tPr2T/b0+F7if8Jl/21BcsgFCA7XVlAXT9o6A +glVHH1jyGGyr70MsLxDhfMRczKxQUCJCqjUz9bnzpmZV2Tb08uTU2bUsUmbUIReX +Irib1w58Pc6FGcrE0lhiMcYYPkT89DC2lYfuIUoI8K88j8S6XqFcNxp9e/5mrmJQ +FzHK +-----END CERTIFICATE----- diff --git a/tests/test_soledad/u1db_tests/testing-certs/testing.key b/tests/test_soledad/u1db_tests/testing-certs/testing.key new file mode 100644 index 00000000..d83d4920 --- /dev/null +++ b/tests/test_soledad/u1db_tests/testing-certs/testing.key @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMYdctPF5PzRTNnk +CD6QEM4/H4dKHU9/KlpSyWVP2Sy/aXUYGrXJCTIAR/VgqsbdOoc3Xxa+3im16vxB +fut3u99jwwYe7emgZxrx7OGd95yPHPrDZns53HCuCRucwJrEkHdFjjmVqS+SQ70n +B1qZUW52oK/dsSyPyouMRw32bvxpAgMBAAECgYBs3lXxhjg1rhabTjIxnx19GTcM +M3Az9V+izweZQu3HJ1CeZiaXauhAr+LbNsniCkRVddotN6oCJdQB10QVxXBZc9Jz +HPJ4zxtZfRZlNMTMmG7eLWrfxpgWnb/BUjDb40yy1nhr9yhDUnI/8RoHDRHnAEHZ +/CnHGUrqcVcrY5zJAQJBAPLhBJg9W88JVmcOKdWxRgs7dLHnZb999Kv1V5mczmAi +jvGvbUmucqOqke6pTUHNYyNHqU6pySzGUi2cH+BAkFECQQDQ0VoAOysg6FVoT15v +tGh57t5sTiCZZ7PS8jwvtThsgA+vcf6c16XWzXgjGXSap4r2QDOY2rI5lsWLaQ8T ++fyZAkAfyFJRmbXp4c7srW3MCOahkaYzoZQu+syJtBFCiMJ40gzik5I5khpuUGPI +V19EvRu8AiSlppIsycb3MPb64XgBAkEAy7DrUf5le5wmc7G4NM6OeyJ+5LbxJbL6 +vnJ8My1a9LuWkVVpQCU7J+UVo2dZTuLPspW9vwTVhUeFOxAoHRxlQQJAFem93f7m +el2BkB2EFqU3onPejkZ5UrDmfmeOQR1axMQNSXqSxcJxqa16Ru1BWV2gcWRbwajQ +oc+kuJThu/r/Ug== +-----END PRIVATE KEY----- diff --git a/tests/test_soledad/util.py b/tests/test_soledad/util.py new file mode 100644 index 00000000..ca8d098d --- /dev/null +++ b/tests/test_soledad/util.py @@ -0,0 +1,399 @@ +# -*- CODING: UTF-8 -*- +# util.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Utilities used by multiple test suites. +""" + +import os +import random +import string +import couchdb +import pytest +import sys + +from six.moves.urllib.parse import urljoin +from six import StringIO +from uuid import uuid4 +from mock import Mock + +from twisted.trial import unittest + +from leap.common.testing.basetest import BaseLeapTest + +from leap.soledad.common import l2db +from leap.soledad.common.l2db import sync +from leap.soledad.common.l2db.remote import http_database + +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.couch import CouchDatabase +from leap.soledad.common.couch.state import CouchServerState + +from leap.soledad.client import Soledad +from leap.soledad.client import http_target +from leap.soledad.client import auth +from leap.soledad.client._crypto import is_symmetrically_encrypted +from leap.soledad.client._db.sqlcipher import SQLCipherDatabase +from leap.soledad.client._db.sqlcipher import SQLCipherOptions + +from leap.soledad.server import SoledadApp + +if sys.version_info[0] < 3: + from pysqlcipher import dbapi2 +else: + from pysqlcipher3 import dbapi2 + + +PASSWORD = '123456' +ADDRESS = 'user-1234' + + +def make_local_db_and_target(test): + db = test.create_database('test') + st = db.get_sync_target() + return db, st + + +def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): + return SoledadDocument(doc_id, rev, content, has_conflicts=has_conflicts) + + +def make_sqlcipher_database_for_test(test, replica_uid): + db = SQLCipherDatabase( + SQLCipherOptions(':memory:', PASSWORD)) + db._set_replica_uid(replica_uid) + return db + + +def copy_sqlcipher_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = make_sqlcipher_database_for_test(test, None) + tmpfile = StringIO() + for line in db._db_handle.iterdump(): + if 'sqlite_sequence' not in line: # work around bug in iterdump + tmpfile.write('%s\n' % line) + tmpfile.seek(0) + new_db._db_handle = dbapi2.connect(':memory:') + new_db._db_handle.cursor().executescript(tmpfile.read()) + new_db._db_handle.commit() + new_db._set_replica_uid(db._replica_uid) + new_db._factory = db._factory + return new_db + + +SQLCIPHER_SCENARIOS = [ + ('sqlcipher', {'make_database_for_test': make_sqlcipher_database_for_test, + 'copy_database_for_test': copy_sqlcipher_database_for_test, + 'make_document_for_test': make_document_for_test, }), +] + + +def make_soledad_app(state): + return SoledadApp(state) + + +def make_token_soledad_app(state): + application = SoledadApp(state) + + def _verify_authentication_data(uuid, auth_data): + if uuid.startswith('user-') and auth_data == 'auth-token': + return True + return False + + # we test for action authorization in leap.soledad.common.tests.test_server + def _verify_authorization(uuid, environ): + return True + + application._verify_authentication_data = _verify_authentication_data + application._verify_authorization = _verify_authorization + return application + + +def make_soledad_document_for_test(test, doc_id, rev, content, + has_conflicts=False): + return SoledadDocument( + doc_id, rev, content, has_conflicts=has_conflicts) + + +def make_token_http_database_for_test(test, replica_uid): + test.startServer() + test.request_state._create_database(replica_uid) + + class _HTTPDatabaseWithToken( + http_database.HTTPDatabase, auth.TokenBasedAuth): + + def set_token_credentials(self, uuid, token): + auth.TokenBasedAuth.set_token_credentials(self, uuid, token) + + def _sign_request(self, method, url_query, params): + return auth.TokenBasedAuth._sign_request( + self, method, url_query, params) + + http_db = _HTTPDatabaseWithToken(test.getURL('test')) + http_db.set_token_credentials('user-uuid', 'auth-token') + return http_db + + +def copy_token_http_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + http_db = test.request_state._copy_database(db) + http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') + return http_db + + +def sync_via_synchronizer(test, db_source, db_target, trace_hook=None, + trace_hook_shallow=None): + target = db_target.get_sync_target() + trace_hook = trace_hook or trace_hook_shallow + if trace_hook: + target._set_trace_hook(trace_hook) + return sync.Synchronizer(db_source, target).sync() + + +class MockedSharedDBTest(object): + + def get_default_shared_mock(self, put_doc_side_effect=None, + get_doc_return_value=None): + """ + Get a default class for mocking the shared DB + """ + class defaultMockSharedDB(object): + get_doc = Mock(return_value=get_doc_return_value) + put_doc = Mock(side_effect=put_doc_side_effect) + open = Mock(return_value=None) + close = Mock(return_value=None) + + def __call__(self): + return self + return defaultMockSharedDB + + +def soledad_sync_target( + test, path, source_replica_uid=uuid4().hex): + creds = {'token': { + 'uuid': 'user-uuid', + 'token': 'auth-token', + }} + return http_target.SoledadHTTPSyncTarget( + test.getURL(path), + source_replica_uid, + creds, + test._soledad._crypto, + None) # cert_file + + +# redefine the base leap test class so it inherits from twisted trial's +# TestCase. This is needed so trial knows that it has to manage a reactor and +# wait for deferreds returned by tests to be fired. + +BaseLeapTest = type( + 'BaseLeapTest', (unittest.TestCase,), dict(BaseLeapTest.__dict__)) + + +class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest): + + """ + Instantiates Soledad for usage in tests. + """ + + @pytest.mark.usefixtures("method_tmpdir") + def setUp(self): + # The following snippet comes from BaseLeapTest.setUpClass, but we + # repeat it here because twisted.trial does not work with + # setUpClass/tearDownClass. + + self.home = self.tempdir + + # config info + self.db1_file = os.path.join(self.tempdir, "db1.u1db") + self.db2_file = os.path.join(self.tempdir, "db2.u1db") + self.email = ADDRESS + # open test dbs + self._db1 = l2db.open(self.db1_file, create=True, + document_factory=SoledadDocument) + self._db2 = l2db.open(self.db2_file, create=True, + document_factory=SoledadDocument) + # get a random prefix for each test, so we do not mess with + # concurrency during initialization and shutting down of + # each local db. + self.rand_prefix = ''.join( + map(lambda x: random.choice(string.ascii_letters), range(6))) + + # initialize soledad by hand so we can control keys + # XXX check if this soledad is actually used + self._soledad = self._soledad_instance( + prefix=self.rand_prefix, user=self.email) + + def tearDown(self): + self._db1.close() + self._db2.close() + self._soledad.close() + + def _delete_temporary_dirs(): + # XXX should not access "private" attrs + for f in [self._soledad.local_db_path, + self._soledad.secrets.secrets_path]: + if os.path.isfile(f): + os.unlink(f) + + from twisted.internet import reactor + reactor.addSystemEventTrigger( + "after", "shutdown", _delete_temporary_dirs) + + def _soledad_instance(self, user=ADDRESS, passphrase=u'123', + prefix='', + secrets_path='secrets.json', + local_db_path='soledad.u1db', + server_url='https://127.0.0.1/', + cert_file=None, + shared_db_class=None, + auth_token='auth-token'): + + def _put_doc_side_effect(doc): + self._doc_put = doc + + if shared_db_class is not None: + MockSharedDB = shared_db_class + else: + MockSharedDB = self.get_default_shared_mock( + _put_doc_side_effect) + + soledad = Soledad( + user, + passphrase, + secrets_path=os.path.join( + self.tempdir, prefix, secrets_path), + local_db_path=os.path.join( + self.tempdir, prefix, local_db_path), + server_url=server_url, # Soledad will fail if not given an url + cert_file=cert_file, + shared_db=MockSharedDB(), + auth_token=auth_token, + with_blobs=True) + self.addCleanup(soledad.close) + return soledad + + @pytest.inlineCallbacks + def assertGetEncryptedDoc( + self, db, doc_id, doc_rev, content, has_conflicts): + """ + Assert that the document in the database looks correct. + """ + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + doc = db.get_doc(doc_id) + + if is_symmetrically_encrypted(doc.content['raw']): + crypt = self._soledad._crypto + decrypted = yield crypt.decrypt_doc(doc) + doc.set_json(decrypted) + self.assertEqual(exp_doc.doc_id, doc.doc_id) + self.assertEqual(exp_doc.rev, doc.rev) + self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts) + self.assertEqual(exp_doc.content, doc.content) + + +@pytest.mark.usefixtures("couch_url") +class CouchDBTestCase(unittest.TestCase, MockedSharedDBTest): + + """ + TestCase base class for tests against a real CouchDB server. + """ + + def setUp(self): + """ + Make sure we have a CouchDB instance for a test. + """ + self.couch_server = couchdb.Server(self.couch_url) + + def delete_db(self, name): + try: + self.couch_server.delete(name) + except: + # ignore if already missing + pass + + +class CouchServerStateForTests(CouchServerState): + + """ + This is a slightly modified CouchDB server state that allows for creating + a database. + + Ordinarily, the CouchDB server state does not allow some operations, + because for security purposes the Soledad Server should not even have + enough permissions to perform them. For tests, we allow database creation, + otherwise we'd have to create those databases in setUp/tearDown methods, + which is less pleasant than allowing the db to be automatically created. + """ + + def __init__(self, *args, **kwargs): + self.dbs = [] + super(CouchServerStateForTests, self).__init__(*args, **kwargs) + + def _create_database(self, replica_uid=None, dbname=None): + """ + Create db and append to a list, allowing test to close it later + """ + dbname = dbname or ('test-%s' % uuid4().hex) + db = CouchDatabase.open_database( + urljoin(self.couch_url, dbname), + True, + replica_uid=replica_uid or 'test') + self.dbs.append(db) + return db + + def ensure_database(self, dbname): + db = self._create_database(dbname=dbname) + return db, db.replica_uid + + +class SoledadWithCouchServerMixin( + BaseSoledadTest, + CouchDBTestCase): + + def setUp(self): + CouchDBTestCase.setUp(self) + BaseSoledadTest.setUp(self) + main_test_class = getattr(self, 'main_test_class', None) + if main_test_class is not None: + main_test_class.setUp(self) + + def tearDown(self): + main_test_class = getattr(self, 'main_test_class', None) + if main_test_class is not None: + main_test_class.tearDown(self) + # delete the test database + BaseSoledadTest.tearDown(self) + CouchDBTestCase.tearDown(self) + + def make_app(self): + self.request_state = CouchServerStateForTests(self.couch_url) + self.addCleanup(self.delete_dbs) + return self.make_app_with_state(self.request_state) + + def delete_dbs(self): + for db in self.request_state.dbs: + self.delete_db(db._dbname) -- cgit v1.2.3