From 8a3bbc6c81f10d8e00fcdd779784f327425f1942 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 5 Jul 2016 08:37:45 +0200 Subject: [refactor] remove u1db dep from support code --- client/pkg/generate_wheels.sh | 2 +- client/pkg/pip_install_requirements.sh | 2 +- client/pkg/requirements-latest.pip | 1 - .../examples/benchmarks/measure_index_times.py | 4 ++-- .../benchmarks/measure_index_times_custom_docid.py | 4 ++-- .../src/leap/soledad/client/examples/use_adbapi.py | 4 ++-- client/src/leap/soledad/client/sqlcipher.py | 2 +- common/pkg/generate_wheels.sh | 2 +- common/pkg/pip_install_requirements.sh | 2 +- common/pkg/requirements-latest.pip | 1 - scripts/docker/files/bin/setup-test-env.py | 19 ++++++++++--------- .../profiling/backends_cpu_usage/test_u1db_sync.py | 21 ++++++++------------- server/pkg/generate_wheels.sh | 2 +- server/pkg/pip_install_requirements.sh | 2 +- server/pkg/requirements-latest.pip | 1 - 15 files changed, 31 insertions(+), 38 deletions(-) diff --git a/client/pkg/generate_wheels.sh b/client/pkg/generate_wheels.sh index e29c327e..496f8e01 100755 --- a/client/pkg/generate_wheels.sh +++ b/client/pkg/generate_wheels.sh @@ -7,7 +7,7 @@ if [ "$WHEELHOUSE" = "" ]; then fi pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip +pip wheel --wheel-dir $WHEELHOUSE --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip if [ -f pkg/requirements-testing.pip ]; then pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip fi diff --git a/client/pkg/pip_install_requirements.sh b/client/pkg/pip_install_requirements.sh index 1f5ac5f6..b97c826f 100755 --- a/client/pkg/pip_install_requirements.sh +++ b/client/pkg/pip_install_requirements.sh @@ -4,7 +4,7 @@ # Use at your own risk. # See $usage for help -insecure_packages="u1db dirspec" +insecure_packages="dirspec" leap_wheelhouse=https://lizard.leap.se/wheels show_help() { diff --git a/client/pkg/requirements-latest.pip b/client/pkg/requirements-latest.pip index a629aa57..fa483db7 100644 --- a/client/pkg/requirements-latest.pip +++ b/client/pkg/requirements-latest.pip @@ -1,6 +1,5 @@ --index-url https://pypi.python.org/simple/ ---allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py index 08775580..4fc91d9d 100644 --- a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py @@ -24,9 +24,9 @@ import hashlib import os import sys -import u1db from twisted.internet import defer, reactor +from leap.soledad.common import l2db from leap.soledad.client import adbapi from leap.soledad.client.sqlcipher import SQLCipherOptions @@ -135,7 +135,7 @@ def countDocs(_): def printResult(r, **kwargs): if kwargs: debug(*kwargs.values()) - elif isinstance(r, u1db.Document): + elif isinstance(r, l2db.Document): debug(r.doc_id, r.content['number']) else: len_results = len(r[1]) diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py index 9deba136..38ea18a3 100644 --- a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py @@ -24,11 +24,11 @@ import hashlib import os import sys -import u1db from twisted.internet import defer, reactor from leap.soledad.client import adbapi from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.common import l2db folder = os.environ.get("TMPDIR", "tmp") @@ -135,7 +135,7 @@ def countDocs(_): def printResult(r, **kwargs): if kwargs: debug(*kwargs.values()) - elif isinstance(r, u1db.Document): + elif isinstance(r, l2db.Document): debug(r.doc_id, r.content['number']) else: len_results = len(r[1]) diff --git a/client/src/leap/soledad/client/examples/use_adbapi.py b/client/src/leap/soledad/client/examples/use_adbapi.py index d7bd21f2..a2683836 100644 --- a/client/src/leap/soledad/client/examples/use_adbapi.py +++ b/client/src/leap/soledad/client/examples/use_adbapi.py @@ -21,11 +21,11 @@ from __future__ import print_function import datetime import os -import u1db from twisted.internet import defer, reactor from leap.soledad.client import adbapi from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.common import l2db folder = os.environ.get("TMPDIR", "tmp") @@ -68,7 +68,7 @@ def countDocs(_): def printResult(r): - if isinstance(r, u1db.Document): + if isinstance(r, l2db.Document): debug(r.doc_id, r.content['number']) else: len_results = len(r[1]) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index bf2a50f1..f36c0b6a 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -594,7 +594,7 @@ class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase): self._db_handle = conn self._real_replica_uid = None self._ensure_schema() - self._factory = u1db.Document + self._factory = l2db.Document class SoledadSQLCipherWrapper(SQLCipherDatabase): diff --git a/common/pkg/generate_wheels.sh b/common/pkg/generate_wheels.sh index e29c327e..496f8e01 100755 --- a/common/pkg/generate_wheels.sh +++ b/common/pkg/generate_wheels.sh @@ -7,7 +7,7 @@ if [ "$WHEELHOUSE" = "" ]; then fi pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip +pip wheel --wheel-dir $WHEELHOUSE --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip if [ -f pkg/requirements-testing.pip ]; then pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip fi diff --git a/common/pkg/pip_install_requirements.sh b/common/pkg/pip_install_requirements.sh index 1f5ac5f6..b97c826f 100755 --- a/common/pkg/pip_install_requirements.sh +++ b/common/pkg/pip_install_requirements.sh @@ -4,7 +4,7 @@ # Use at your own risk. # See $usage for help -insecure_packages="u1db dirspec" +insecure_packages="dirspec" leap_wheelhouse=https://lizard.leap.se/wheels show_help() { diff --git a/common/pkg/requirements-latest.pip b/common/pkg/requirements-latest.pip index 9de17382..9b579503 100644 --- a/common/pkg/requirements-latest.pip +++ b/common/pkg/requirements-latest.pip @@ -1,6 +1,5 @@ --index-url https://pypi.python.org/simple/ ---allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' -e . diff --git a/scripts/docker/files/bin/setup-test-env.py b/scripts/docker/files/bin/setup-test-env.py index 0569b65d..0f3ea6f4 100755 --- a/scripts/docker/files/bin/setup-test-env.py +++ b/scripts/docker/files/bin/setup-test-env.py @@ -31,7 +31,8 @@ from couchdb.http import PreconditionFailed from couchdb.http import ResourceConflict from couchdb.http import ResourceNotFound from hashlib import sha512 -from u1db.errors import DatabaseDoesNotExist + +from leap.soledad.common.l2db.errors import DatabaseDoesNotExist # @@ -65,15 +66,15 @@ def pidfile_is_running(pidfile): return False -def status_from_pidfile(args, default_basedir): +def status_from_pidfile(args, default_basedir, name): basedir = _get_basedir(args, default_basedir) pidfile = os.path.join(basedir, args.pidfile) try: pid = get_pid(pidfile) psutil.Process(pid) - print "[+] running - pid: %d" % pid + print "[+] %s is running with pid %d" % (name, pid) except (IOError, psutil.NoSuchProcess): - print "[-] stopped" + print "[-] %s stopped" % name def kill_all_executables(args): @@ -163,7 +164,7 @@ def couch_server_start(args): except: time.sleep(0.1) - print '[+] running - pid: %d' % pid + print '[+] couch is running with pid: %d' % pid def couch_server_stop(args): @@ -177,11 +178,11 @@ def couch_server_stop(args): args.executable, '-p %s' % pidfile, # set the background PID FILE '-k']) # kill the background process, will respawn if needed - print '[-] stopped - pid: %d ' % pid + print '[-] stopped couch server with pid %d ' % pid def couch_status_from_pidfile(args): - status_from_pidfile(args, COUCH_BASEDIR) + status_from_pidfile(args, COUCH_BASEDIR, 'couch') # @@ -264,7 +265,7 @@ def soledad_server_start(args): call([args.executable] + params) pid = get_pid(pidfile) - print '[+] running - pid: %d' % pid + print '[+] soledad-server is running with pid %d' % pid def soledad_server_stop(args): @@ -279,7 +280,7 @@ def soledad_server_stop(args): def soledad_server_status_from_pidfile(args): - status_from_pidfile(args, SOLEDAD_SERVER_BASEDIR) + status_from_pidfile(args, SOLEDAD_SERVER_BASEDIR, 'soledad-server') # couch helpers diff --git a/scripts/profiling/backends_cpu_usage/test_u1db_sync.py b/scripts/profiling/backends_cpu_usage/test_u1db_sync.py index 26ef8f9f..5ae68c81 100755 --- a/scripts/profiling/backends_cpu_usage/test_u1db_sync.py +++ b/scripts/profiling/backends_cpu_usage/test_u1db_sync.py @@ -1,18 +1,16 @@ #!/usr/bin/python -import u1db import tempfile import logging import shutil import os -import argparse import time import binascii -import random - +from leap.soledad.common import l2db from leap.soledad.client.sqlcipher import open as sqlcipher_open + from log_cpu_usage import LogCpuUsage from u1dblite import open as u1dblite_open from u1dbcipher import open as u1dbcipher_open @@ -24,10 +22,10 @@ BIGGEST_DOC_SIZE = 100 * 1024 # 100 KB def get_data(size): - return binascii.hexlify(os.urandom(size/2)) + return binascii.hexlify(os.urandom(size / 2)) -def run_test(testname, open_fun, tempdir, docs, *args): +def run_test(testname, open_fun, tempdir, docs, *args): logger.info('Starting test \"%s\".' % testname) # instantiate dbs @@ -36,8 +34,7 @@ def run_test(testname, open_fun, tempdir, docs, *args): # get sync target and synchsonizer target = db2.get_sync_target() - synchronizer = u1db.sync.Synchronizer(db1, target) - + synchronizer = l2db.sync.Synchronizer(db1, target) # generate lots of small documents logger.info('Creating %d documents in source db...' % DOCS_TO_SYNC) @@ -80,30 +77,28 @@ def run_test(testname, open_fun, tempdir, docs, *args): if __name__ == '__main__': - + # configure logger logger = logging.getLogger(__name__) LOG_FORMAT = '%(asctime)s %(message)s' logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) - # get a temporary dir tempdir = tempfile.mkdtemp() logger.info('Using temporary directory %s' % tempdir) - # create a lot of documents with random sizes docs = [] for i in xrange(DOCS_TO_SYNC): docs.append({ 'index': i, - #'data': get_data( + # 'data': get_data( # random.randrange( # SMALLEST_DOC_SIZE, BIGGEST_DOC_SIZE)) }) # run tests - run_test('sqlite', u1db.open, tempdir, docs, True) + run_test('sqlite', l2db.open, tempdir, docs, True) run_test('sqlcipher', sqlcipher_open, tempdir, docs, '123456', True) run_test('u1dblite', u1dblite_open, tempdir, docs) run_test('u1dbcipher', u1dbcipher_open, tempdir, docs, '123456', True) diff --git a/server/pkg/generate_wheels.sh b/server/pkg/generate_wheels.sh index e29c327e..496f8e01 100755 --- a/server/pkg/generate_wheels.sh +++ b/server/pkg/generate_wheels.sh @@ -7,7 +7,7 @@ if [ "$WHEELHOUSE" = "" ]; then fi pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip +pip wheel --wheel-dir $WHEELHOUSE --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip if [ -f pkg/requirements-testing.pip ]; then pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip fi diff --git a/server/pkg/pip_install_requirements.sh b/server/pkg/pip_install_requirements.sh index 1f5ac5f6..b97c826f 100755 --- a/server/pkg/pip_install_requirements.sh +++ b/server/pkg/pip_install_requirements.sh @@ -4,7 +4,7 @@ # Use at your own risk. # See $usage for help -insecure_packages="u1db dirspec" +insecure_packages="dirspec" leap_wheelhouse=https://lizard.leap.se/wheels show_help() { diff --git a/server/pkg/requirements-latest.pip b/server/pkg/requirements-latest.pip index a629aa57..fa483db7 100644 --- a/server/pkg/requirements-latest.pip +++ b/server/pkg/requirements-latest.pip @@ -1,6 +1,5 @@ --index-url https://pypi.python.org/simple/ ---allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' -- cgit v1.2.3