diff options
author | Micah Anderson <micah@riseup.net> | 2013-08-22 14:45:35 -0400 |
---|---|---|
committer | Micah Anderson <micah@riseup.net> | 2013-08-22 14:45:35 -0400 |
commit | 74e3e8ce861d20edc19ac7f858ba0f016f73b1b2 (patch) | |
tree | 12f3530b968837a26e49e339843eb2fba9b466dc | |
parent | ae049361d2eead016ac2a963add0516494ef5a72 (diff) | |
parent | 59f337a29202bb80287a31d6c5b942b74375f33f (diff) |
Merge remote-tracking branch 'origin/develop' into debian
Conflicts:
client/src/leap/__init__.py
client/src/leap/soledad/client/__init__.py
client/src/leap/soledad/client/auth.py
client/src/leap/soledad/client/crypto.py
client/src/leap/soledad/client/shared_db.py
client/src/leap/soledad/client/sqlcipher.py
client/src/leap/soledad/client/target.py
common/src/leap/soledad/common/couch.py
common/src/leap/soledad/common/crypto.py
common/src/leap/soledad/common/objectstore.py
common/src/leap/soledad/common/tests/__init__.py
common/src/leap/soledad/common/tests/couchdb.ini.template
common/src/leap/soledad/common/tests/test_couch.py
common/src/leap/soledad/common/tests/test_server.py
common/src/leap/soledad/common/tests/test_soledad.py
common/src/leap/soledad/common/tests/test_sqlcipher.py
common/src/leap/soledad/common/tests/test_target.py
common/src/leap/soledad/common/tests/u1db_tests/README
common/src/leap/soledad/common/tests/u1db_tests/__init__.py
common/src/leap/soledad/common/tests/u1db_tests/test_backends.py
common/src/leap/soledad/common/tests/u1db_tests/test_document.py
common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py
common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py
common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py
common/src/leap/soledad/common/tests/u1db_tests/test_https.py
common/src/leap/soledad/common/tests/u1db_tests/test_open.py
common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py
common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py
common/src/leap/soledad/common/tests/u1db_tests/test_sync.py
common/src/leap/soledad/common/tests/u1db_tests/testing-certs/Makefile
common/src/leap/soledad/common/tests/u1db_tests/testing-certs/cacert.pem
common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.cert
common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.key
pkg/soledad
server/pkg/soledad
server/setup.py
server/src/leap/soledad/server/auth.py
setup.py
soledad/setup.py
soledad/src/leap/__init__.py
soledad/src/leap/soledad/__init__.py
soledad/src/leap/soledad/auth.py
soledad/src/leap/soledad/crypto.py
soledad/src/leap/soledad/shared_db.py
soledad/src/leap/soledad/sqlcipher.py
soledad/src/leap/soledad/target.py
soledad/src/leap/soledad/tests/__init__.py
soledad/src/leap/soledad/tests/couchdb.ini.template
soledad/src/leap/soledad/tests/test_couch.py
soledad/src/leap/soledad/tests/test_crypto.py
soledad/src/leap/soledad/tests/test_server.py
soledad/src/leap/soledad/tests/test_soledad.py
soledad/src/leap/soledad/tests/test_sqlcipher.py
soledad/src/leap/soledad/tests/test_target.py
soledad/src/leap/soledad/tests/u1db_tests/README
soledad/src/leap/soledad/tests/u1db_tests/__init__.py
soledad/src/leap/soledad/tests/u1db_tests/test_backends.py
soledad/src/leap/soledad/tests/u1db_tests/test_document.py
soledad/src/leap/soledad/tests/u1db_tests/test_http_app.py
soledad/src/leap/soledad/tests/u1db_tests/test_http_client.py
soledad/src/leap/soledad/tests/u1db_tests/test_http_database.py
soledad/src/leap/soledad/tests/u1db_tests/test_https.py
soledad/src/leap/soledad/tests/u1db_tests/test_open.py
soledad/src/leap/soledad/tests/u1db_tests/test_remote_sync_target.py
soledad/src/leap/soledad/tests/u1db_tests/test_sqlite_backend.py
soledad/src/leap/soledad/tests/u1db_tests/test_sync.py
soledad/src/leap/soledad/tests/u1db_tests/testing-certs/Makefile
soledad/src/leap/soledad/tests/u1db_tests/testing-certs/cacert.pem
soledad/src/leap/soledad/tests/u1db_tests/testing-certs/testing.cert
soledad/src/leap/soledad/tests/u1db_tests/testing-certs/testing.key
soledad_server/pkg/soledad
soledad_server/src/leap/soledad_server/auth.py
soledad_server/src/leap/soledad_server/couch.py
soledad_server/src/leap/soledad_server/objectstore.py
src/leap/__init__.py
src/leap/soledad/__init__.py
src/leap/soledad/auth.py
src/leap/soledad/backends/couch.py
src/leap/soledad/backends/leap_backend.py
src/leap/soledad/backends/objectstore.py
src/leap/soledad/backends/sqlcipher.py
src/leap/soledad/crypto.py
src/leap/soledad/server.py
src/leap/soledad/shared_db.py
src/leap/soledad/tests/__init__.py
src/leap/soledad/tests/couchdb.ini.template
src/leap/soledad/tests/test_couch.py
src/leap/soledad/tests/test_leap_backend.py
src/leap/soledad/tests/test_server.py
src/leap/soledad/tests/test_soledad.py
src/leap/soledad/tests/test_sqlcipher.py
src/leap/soledad/tests/u1db_tests/README
src/leap/soledad/tests/u1db_tests/__init__.py
src/leap/soledad/tests/u1db_tests/test_backends.py
src/leap/soledad/tests/u1db_tests/test_document.py
src/leap/soledad/tests/u1db_tests/test_http_app.py
src/leap/soledad/tests/u1db_tests/test_http_client.py
src/leap/soledad/tests/u1db_tests/test_http_database.py
src/leap/soledad/tests/u1db_tests/test_https.py
src/leap/soledad/tests/u1db_tests/test_open.py
src/leap/soledad/tests/u1db_tests/test_remote_sync_target.py
src/leap/soledad/tests/u1db_tests/test_sqlite_backend.py
src/leap/soledad/tests/u1db_tests/test_sync.py
src/leap/soledad/tests/u1db_tests/testing-certs/Makefile
src/leap/soledad/tests/u1db_tests/testing-certs/cacert.pem
src/leap/soledad/tests/u1db_tests/testing-certs/testing.cert
src/leap/soledad/tests/u1db_tests/testing-certs/testing.key
-rw-r--r-- | .gitignore | 6 | ||||
-rw-r--r-- | README.rst | 20 | ||||
-rw-r--r-- | changes/bug-update-readme-requirements | 1 | ||||
-rw-r--r-- | changes/bug_3497-check-for-none-in-priv-db | 1 | ||||
-rw-r--r-- | changes/feature_add-xsalsa20-encryption-method | 1 | ||||
-rw-r--r-- | changes/feature_use-pycryptopp-for-symmetric-encryption | 1 | ||||
-rw-r--r-- | client/changes/feature_3118-provide-a-way-to-access-the-saved-password | 1 | ||||
-rw-r--r-- | client/changes/feature_3487-split-soledad-into-common-client-and-server | 1 | ||||
-rw-r--r-- | client/setup.py (renamed from soledad/setup.py) | 23 | ||||
-rw-r--r-- | client/src/leap/__init__.py (renamed from soledad/src/leap/__init__.py) | 0 | ||||
-rw-r--r-- | client/src/leap/soledad/__init__.py (renamed from soledad_server/src/leap/__init__.py) | 0 | ||||
-rw-r--r-- | client/src/leap/soledad/client/__init__.py (renamed from soledad/src/leap/soledad/__init__.py) | 111 | ||||
-rw-r--r-- | client/src/leap/soledad/client/auth.py (renamed from soledad/src/leap/soledad/auth.py) | 3 | ||||
-rw-r--r-- | client/src/leap/soledad/client/crypto.py (renamed from soledad/src/leap/soledad/crypto.py) | 56 | ||||
-rw-r--r-- | client/src/leap/soledad/client/shared_db.py (renamed from soledad/src/leap/soledad/shared_db.py) | 2 | ||||
-rw-r--r-- | client/src/leap/soledad/client/sqlcipher.py (renamed from soledad/src/leap/soledad/sqlcipher.py) | 117 | ||||
-rw-r--r-- | client/src/leap/soledad/client/target.py (renamed from soledad/src/leap/soledad/target.py) | 52 | ||||
-rw-r--r-- | common/changes/feature_3487-split-soledad-into-common-client-and-server | 1 | ||||
-rw-r--r-- | common/setup.py | 79 | ||||
-rw-r--r-- | common/src/leap/__init__.py | 6 | ||||
-rw-r--r-- | common/src/leap/soledad/__init__.py | 6 | ||||
-rw-r--r-- | common/src/leap/soledad/common/__init__.py | 64 | ||||
-rw-r--r-- | common/src/leap/soledad/common/couch.py (renamed from soledad_server/src/leap/soledad_server/couch.py) | 3 | ||||
-rw-r--r-- | common/src/leap/soledad/common/crypto.py | 55 | ||||
-rw-r--r-- | common/src/leap/soledad/common/document.py (renamed from soledad/src/leap/soledad/document.py) | 0 | ||||
-rw-r--r-- | common/src/leap/soledad/common/objectstore.py (renamed from soledad_server/src/leap/soledad_server/objectstore.py) | 3 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/__init__.py (renamed from soledad/src/leap/soledad/tests/__init__.py) | 24 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/couchdb.ini.template (renamed from soledad/src/leap/soledad/tests/couchdb.ini.template) | 0 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/test_couch.py (renamed from soledad/src/leap/soledad/tests/test_couch.py) | 10 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/test_crypto.py (renamed from soledad/src/leap/soledad/tests/test_crypto.py) | 66 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/test_server.py (renamed from soledad/src/leap/soledad/tests/test_server.py) | 22 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/test_soledad.py (renamed from soledad/src/leap/soledad/tests/test_soledad.py) | 118 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/test_sqlcipher.py (renamed from soledad/src/leap/soledad/tests/test_sqlcipher.py) | 25 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/test_target.py (renamed from soledad/src/leap/soledad/tests/test_target.py) | 44 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/README (renamed from soledad/src/leap/soledad/tests/u1db_tests/README) | 0 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/__init__.py (renamed from soledad/src/leap/soledad/tests/u1db_tests/__init__.py) | 7 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/test_backends.py (renamed from soledad/src/leap/soledad/tests/u1db_tests/test_backends.py) | 4 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/test_document.py (renamed from soledad/src/leap/soledad/tests/u1db_tests/test_document.py) | 2 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py (renamed from soledad/src/leap/soledad/tests/u1db_tests/test_http_app.py) | 2 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py (renamed from soledad/src/leap/soledad/tests/u1db_tests/test_http_client.py) | 2 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py (renamed from soledad/src/leap/soledad/tests/u1db_tests/test_http_database.py) | 4 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/test_https.py (renamed from soledad/src/leap/soledad/tests/u1db_tests/test_https.py) | 6 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/test_open.py (renamed from soledad/src/leap/soledad/tests/u1db_tests/test_open.py) | 4 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py (renamed from soledad/src/leap/soledad/tests/u1db_tests/test_remote_sync_target.py) | 2 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py (renamed from soledad/src/leap/soledad/tests/u1db_tests/test_sqlite_backend.py) | 4 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/test_sync.py (renamed from soledad/src/leap/soledad/tests/u1db_tests/test_sync.py) | 10 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/testing-certs/Makefile (renamed from soledad/src/leap/soledad/tests/u1db_tests/testing-certs/Makefile) | 0 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/testing-certs/cacert.pem (renamed from soledad/src/leap/soledad/tests/u1db_tests/testing-certs/cacert.pem) | 0 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.cert (renamed from soledad/src/leap/soledad/tests/u1db_tests/testing-certs/testing.cert) | 0 | ||||
-rw-r--r-- | common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.key (renamed from soledad/src/leap/soledad/tests/u1db_tests/testing-certs/testing.key) | 0 | ||||
-rw-r--r-- | server/changes/feature_3487-split-soledad-into-common-client-and-server | 1 | ||||
-rw-r--r-- | server/pkg/soledad (renamed from soledad_server/pkg/soledad) | 0 | ||||
-rw-r--r-- | server/setup.py (renamed from soledad_server/setup.py) | 10 | ||||
-rw-r--r-- | server/src/leap/__init__.py | 6 | ||||
-rw-r--r-- | server/src/leap/soledad/__init__.py | 6 | ||||
-rw-r--r-- | server/src/leap/soledad/server/__init__.py (renamed from soledad_server/src/leap/soledad_server/__init__.py) | 6 | ||||
-rw-r--r-- | server/src/leap/soledad/server/auth.py (renamed from soledad_server/src/leap/soledad_server/auth.py) | 0 | ||||
-rw-r--r-- | soledad/src/leap/soledad/backends/__init__.py | 0 | ||||
-rw-r--r-- | soledad/src/leap/soledad/backends/leap_backend.py | 73 | ||||
-rw-r--r-- | soledad/src/leap/soledad/dbwrapper.py | 183 |
60 files changed, 659 insertions, 595 deletions
@@ -5,4 +5,10 @@ build/ MANIFEST *.egg-info/ *.egg +*.swp +*.swo +*.pyc +*.log +*.*~ + @@ -7,21 +7,23 @@ Soledad This software is under development. +Library dependencies +-------------------- +* ``libsqlite3-dev`` + Tests ----- -Client and server tests are both included in leap.soledad. Because -soledad_server depends on soledad and soledad tests depend on soledad_server, -if you want to run tests in development mode you must first install soledad, -then soledad_server, and then run the tests. - -Therefore, tests must be run with:: +Client and server tests are both included in leap.soledad.common. If you want +to run tests in development mode you must do the following:: - cd soledad + cd common + python setup.py develop + cd ../client python setup.py develop - cd ../soledad_server + cd ../server python setup.py develop - cd ../soledad + cd ../common python setup.py test Note that to run CouchDB tests, be sure you have ``CouchDB`` installed on your diff --git a/changes/bug-update-readme-requirements b/changes/bug-update-readme-requirements new file mode 100644 index 00000000..fedb78cf --- /dev/null +++ b/changes/bug-update-readme-requirements @@ -0,0 +1 @@ + o Add libsqlite3-dev requirement for soledad. diff --git a/changes/bug_3497-check-for-none-in-priv-db b/changes/bug_3497-check-for-none-in-priv-db new file mode 100644 index 00000000..26365ec8 --- /dev/null +++ b/changes/bug_3497-check-for-none-in-priv-db @@ -0,0 +1 @@ + o Check for None in private methods that depend on _db. Closes: #3497 diff --git a/changes/feature_add-xsalsa20-encryption-method b/changes/feature_add-xsalsa20-encryption-method new file mode 100644 index 00000000..66b97de8 --- /dev/null +++ b/changes/feature_add-xsalsa20-encryption-method @@ -0,0 +1 @@ + o Add XSalsa20 symmetric encryption method. diff --git a/changes/feature_use-pycryptopp-for-symmetric-encryption b/changes/feature_use-pycryptopp-for-symmetric-encryption new file mode 100644 index 00000000..1f28db97 --- /dev/null +++ b/changes/feature_use-pycryptopp-for-symmetric-encryption @@ -0,0 +1 @@ + o Use pycryptopp for symmetric encryption. diff --git a/client/changes/feature_3118-provide-a-way-to-access-the-saved-password b/client/changes/feature_3118-provide-a-way-to-access-the-saved-password new file mode 100644 index 00000000..69cb0b1d --- /dev/null +++ b/client/changes/feature_3118-provide-a-way-to-access-the-saved-password @@ -0,0 +1 @@ + o Add public method to access the saved password. Closes #3118. diff --git a/client/changes/feature_3487-split-soledad-into-common-client-and-server b/client/changes/feature_3487-split-soledad-into-common-client-and-server new file mode 100644 index 00000000..2eab6b56 --- /dev/null +++ b/client/changes/feature_3487-split-soledad-into-common-client-and-server @@ -0,0 +1 @@ + o Split soledad package into common, client and server. Closes #3487. diff --git a/soledad/setup.py b/client/setup.py index f2291662..291c95fe 100644 --- a/soledad/setup.py +++ b/client/setup.py @@ -29,20 +29,10 @@ install_requirements = [ 'oauth', # this is not strictly needed by us, but we need it # until u1db adds it to its release as a dep. 'u1db', - 'six==1.1.0', 'scrypt', 'pyxdg', - 'pycrypto', - 'pyOpenSSL', -] - - -tests_requirements = [ - 'mock', - 'nose2', - 'testscenarios', - 'leap.common', - 'leap.soledad_server', + 'pycryptopp', + 'leap.soledad.common>=0.3.0', ] @@ -60,8 +50,9 @@ trove_classifiers = ( "Topic :: Software Development :: Libraries :: Python Modules" ) + setup( - name='leap.soledad', + name='leap.soledad.client', version='0.3.0', url='https://leap.se/', license='GPLv3+', @@ -73,12 +64,10 @@ setup( "securely shared among devices. It provides, to other parts of the " "LEAP client, an API for data storage and sync." ), - namespace_packages=["leap"], - packages=find_packages('src', exclude=['leap.soledad.tests']), + namespace_packages=["leap", "leap.soledad"], + packages=find_packages('src'), package_dir={'': 'src'}, - test_suite='leap.soledad.tests', install_requires=install_requirements, - tests_require=tests_requirements, classifiers=trove_classifiers, extras_require={'signaling': ['leap.common']}, ) diff --git a/soledad/src/leap/__init__.py b/client/src/leap/__init__.py index f48ad105..f48ad105 100644 --- a/soledad/src/leap/__init__.py +++ b/client/src/leap/__init__.py diff --git a/soledad_server/src/leap/__init__.py b/client/src/leap/soledad/__init__.py index f48ad105..f48ad105 100644 --- a/soledad_server/src/leap/__init__.py +++ b/client/src/leap/soledad/__init__.py diff --git a/soledad/src/leap/soledad/__init__.py b/client/src/leap/soledad/client/__init__.py index 00ac21f8..a3b26689 100644 --- a/soledad/src/leap/soledad/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -45,49 +45,6 @@ from u1db.remote.ssl_match_hostname import match_hostname # -# Assert functions -# - -# we want to use leap.common.check.leap_assert in case it is available, -# because it also logs in a way other parts of leap can access log messages. - -try: - from leap.common.check import leap_assert as soledad_assert - -except ImportError: - - def soledad_assert(condition, message): - """ - Asserts the condition and displays the message if that's not - met. - - @param condition: condition to check - @type condition: bool - @param message: message to display if the condition isn't met - @type message: str - """ - assert condition, message - -try: - from leap.common.check import leap_assert_type as soledad_assert_type - -except ImportError: - - def soledad_assert_type(var, expectedType): - """ - Helper assert check for a variable's expected type - - @param var: variable to check - @type var: any - @param expectedType: type to check agains - @type expectedType: type - """ - soledad_assert(isinstance(var, expectedType), - "Expected type %r instead of %r" % - (expectedType, type(var))) - - -# # Signaling function # @@ -124,13 +81,13 @@ except ImportError: logger.info("Would signal: %s - %s." % (str(signal), content)) -from leap.soledad.crypto import SoledadCrypto -from leap.soledad.dbwrapper import SQLCipherWrapper -from leap.soledad.document import SoledadDocument -from leap.soledad.shared_db import SoledadSharedDatabase -from leap.soledad.sqlcipher import open as sqlcipher_open -from leap.soledad.sqlcipher import SQLCipherDatabase -from leap.soledad.target import SoledadSyncTarget +from leap.soledad.common import soledad_assert, soledad_assert_type +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client.crypto import SoledadCrypto +from leap.soledad.client.shared_db import SoledadSharedDatabase +from leap.soledad.client.sqlcipher import open as sqlcipher_open +from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client.target import SoledadSyncTarget logger = logging.getLogger(name=__name__) @@ -238,7 +195,7 @@ class Soledad(object): MINIMUM_PASSPHRASE_LENGTH = 6 """ The minimum length for a passphrase. The passphrase length is only checked - when the user changes her passphras, not when she instantiates Soledad. + when the user changes her passphrase, not when she instantiates Soledad. """ IV_SEPARATOR = ":" @@ -429,8 +386,7 @@ class Soledad(object): buflen=32, # we need a key with 256 bits (32 bytes) ) - # Instantiate a thread-safe wrapper - self._db = SQLCipherWrapper( + self._db = sqlcipher_open( self._local_db_path, binascii.b2a_hex(key), # sqlcipher only accepts the hex version create=True, @@ -444,7 +400,7 @@ class Soledad(object): """ if hasattr(self, '_db') and isinstance( self._db, - SQLCipherWrapper): + SQLCipherDatabase): self._db.close() def __del__(self): @@ -650,6 +606,7 @@ class Soledad(object): self.SECRET_KEY: '%s%s%s' % ( str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), } + self._store_secrets() self._passphrase = new_passphrase @@ -855,7 +812,8 @@ class Soledad(object): "number(fieldname, width)", "lower(fieldname)" """ - return self._db.create_index(index_name, *index_expressions) + if self._db: + return self._db.create_index(index_name, *index_expressions) def delete_index(self, index_name): """ @@ -864,7 +822,8 @@ class Soledad(object): @param index_name: The name of the index we are removing @type index_name: str """ - return self._db.delete_index(index_name) + if self._db: + return self._db.delete_index(index_name) def list_indexes(self): """ @@ -873,7 +832,8 @@ class Soledad(object): @return: A list of [('index-name', ['field', 'field2'])] definitions. @rtype: list """ - return self._db.list_indexes() + if self._db: + return self._db.list_indexes() def get_from_index(self, index_name, *key_values): """ @@ -895,7 +855,8 @@ class Soledad(object): @return: List of [Document] @rtype: list """ - return self._db.get_from_index(index_name, *key_values) + if self._db: + return self._db.get_from_index(index_name, *key_values) def get_range_from_index(self, index_name, start_value, end_value): """ @@ -924,8 +885,9 @@ class Soledad(object): @return: List of [Document] @rtype: list """ - return self._db.get_range_from_index( - index_name, start_value, end_value) + if self._db: + return self._db.get_range_from_index( + index_name, start_value, end_value) def get_index_keys(self, index_name): """ @@ -936,7 +898,8 @@ class Soledad(object): @return: [] A list of tuples of indexed keys. @rtype: list """ - return self._db.get_index_keys(index_name) + if self._db: + return self._db.get_index_keys(index_name) def get_doc_conflicts(self, doc_id): """ @@ -948,7 +911,8 @@ class Soledad(object): @return: a list of the document entries that are conflicted @rtype: list """ - return self._db.get_doc_conflicts(doc_id) + if self._db: + return self._db.get_doc_conflicts(doc_id) def resolve_doc(self, doc, conflicted_doc_revs): """ @@ -960,7 +924,8 @@ class Soledad(object): supersedes. @type conflicted_doc_revs: list """ - return self._db.resolve_doc(doc, conflicted_doc_revs) + if self._db: + return self._db.resolve_doc(doc, conflicted_doc_revs) def sync(self): """ @@ -973,11 +938,12 @@ class Soledad(object): performed. @rtype: str """ - local_gen = self._db.sync( - urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), - creds=self._creds, autocreate=True) - signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) - return local_gen + if self._db: + local_gen = self._db.sync( + urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), + creds=self._creds, autocreate=True) + signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) + return local_gen def need_sync(self, url): """ @@ -1120,6 +1086,14 @@ class Soledad(object): _get_storage_secret, doc='The secret used for symmetric encryption.') + def _get_passphrase(self): + return self._passphrase + + passphrase = property( + _get_passphrase, + doc='The passphrase for locking and unlocking encryption secrets for ' + 'local and remote storage.') + #----------------------------------------------------------------------------- # Monkey patching u1db to be able to provide a custom SSL cert @@ -1128,6 +1102,7 @@ class Soledad(object): # We need a more reasonable timeout (in seconds) SOLEDAD_TIMEOUT = 10 + class VerifiedHTTPSConnection(httplib.HTTPSConnection): """HTTPSConnection verifying server side certificates.""" # derived from httplib.py diff --git a/soledad/src/leap/soledad/auth.py b/client/src/leap/soledad/client/auth.py index 81e838d2..3cd6dabe 100644 --- a/soledad/src/leap/soledad/auth.py +++ b/client/src/leap/soledad/client/auth.py @@ -67,4 +67,5 @@ class TokenBasedAuth(object): return [('Authorization', 'Token %s' % auth.encode('base64')[:-1])] else: raise errors.UnknownAuthMethod( - 'Wrong credentials: %s' % self._creds)
\ No newline at end of file + 'Wrong credentials: %s' % self._creds) + diff --git a/soledad/src/leap/soledad/crypto.py b/client/src/leap/soledad/client/crypto.py index bfad66d1..9fcff8e9 100644 --- a/soledad/src/leap/soledad/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -27,11 +27,11 @@ import hmac import hashlib -from Crypto.Cipher import AES -from Crypto.Util import Counter +from pycryptopp.cipher.aes import AES +from pycryptopp.cipher.xsalsa20 import XSalsa20 -from leap.soledad import ( +from leap.soledad.common import ( soledad_assert, soledad_assert_type, ) @@ -43,6 +43,7 @@ class EncryptionMethods(object): """ AES_256_CTR = 'aes-256-ctr' + XSALSA20 = 'xsalsa20' class UnknownEncryptionMethod(Exception): @@ -93,19 +94,23 @@ class SoledadCrypto(object): """ soledad_assert_type(key, str) + soledad_assert( + len(key) == 32, # 32 x 8 = 256 bits. + 'Wrong key size: %s bits (must be 256 bits long).' % + (len(key) * 8)) + iv = None # AES-256 in CTR mode if method == EncryptionMethods.AES_256_CTR: - soledad_assert( - len(key) == 32, # 32 x 8 = 256 bits. - 'Wrong key size: %s bits (must be 256 bits long).' % - (len(key) * 8)) - iv = os.urandom(8) - ctr = Counter.new(64, prefix=iv) - cipher = AES.new(key=key, mode=AES.MODE_CTR, counter=ctr) - return binascii.b2a_base64(iv), cipher.encrypt(data) - - # raise if method is unknown - raise UnknownEncryptionMethod('Unkwnown method: %s' % method) + iv = os.urandom(16) + ciphertext = AES(key=key, iv=iv).process(data) + # XSalsa20 + elif method == EncryptionMethods.XSALSA20: + iv = os.urandom(24) + ciphertext = XSalsa20(key=key, iv=iv).process(data) + else: + # raise if method is unknown + raise UnknownEncryptionMethod('Unkwnown method: %s' % method) + return binascii.b2a_base64(iv), ciphertext def decrypt_sym(self, data, key, method=EncryptionMethods.AES_256_CTR, **kwargs): @@ -127,19 +132,20 @@ class SoledadCrypto(object): @rtype: str """ soledad_assert_type(key, str) - + # assert params + soledad_assert( + len(key) == 32, # 32 x 8 = 256 bits. + 'Wrong key size: %s (must be 256 bits long).' % len(key)) + soledad_assert( + 'iv' in kwargs, + '%s needs an initial value.' % method) # AES-256 in CTR mode if method == EncryptionMethods.AES_256_CTR: - # assert params - soledad_assert( - len(key) == 32, # 32 x 8 = 256 bits. - 'Wrong key size: %s (must be 256 bits long).' % len(key)) - soledad_assert( - 'iv' in kwargs, - 'AES-256-CTR needs an initial value.') - ctr = Counter.new(64, prefix=binascii.a2b_base64(kwargs['iv'])) - cipher = AES.new(key=key, mode=AES.MODE_CTR, counter=ctr) - return cipher.decrypt(data) + return AES( + key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data) + elif method == EncryptionMethods.XSALSA20: + return XSalsa20( + key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data) # raise if method is unknown raise UnknownEncryptionMethod('Unkwnown method: %s' % method) diff --git a/soledad/src/leap/soledad/shared_db.py b/client/src/leap/soledad/client/shared_db.py index 33c5c484..a6ca504d 100644 --- a/soledad/src/leap/soledad/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -26,7 +26,7 @@ import simplejson as json from u1db.remote import http_database -from leap.soledad.auth import TokenBasedAuth +from leap.soledad.client.auth import TokenBasedAuth #----------------------------------------------------------------------------- diff --git a/soledad/src/leap/soledad/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index acbeabe6..c605c28c 100644 --- a/soledad/src/leap/soledad/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -43,23 +43,33 @@ So, as the statements above were introduced for backwards compatibility with SLCipher 1.1 databases, we do not implement them as all SQLCipher databases handled by Soledad should be created by SQLCipher >= 2.0. """ - +import logging import os import time import string +import threading from u1db.backends import sqlite_backend from pysqlcipher import dbapi2 -from u1db import ( - errors, -) -from leap.soledad.document import SoledadDocument +from u1db import errors as u1db_errors +from leap.soledad.common.document import SoledadDocument + +logger = logging.getLogger(__name__) # Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2 sqlite_backend.dbapi2 = dbapi2 +# It seems that, as long as we are not using old sqlite versions, serialized +# mode is enabled by default at compile time. So accessing db connections from +# different threads should be safe, as long as no attempt is made to use them +# from multiple threads with no locking. +# See https://sqlite.org/threadsafe.html +# and http://bugs.python.org/issue16509 + +SQLITE_CHECK_SAME_THREAD = False + def open(path, password, create=True, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, @@ -125,6 +135,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """A U1DB implementation that uses SQLCipher as its persistence layer.""" _index_storage_value = 'expand referenced encrypted' + k_lock = threading.Lock() def __init__(self, sqlcipher_file, password, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', @@ -158,20 +169,23 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): sqlcipher_file, password, raw_key, cipher, kdf_iter, cipher_page_size) # connect to the database - self._db_handle = dbapi2.connect(sqlcipher_file) - # set SQLCipher cryptographic parameters - self._set_crypto_pragmas( - self._db_handle, password, raw_key, cipher, kdf_iter, - cipher_page_size) - self._real_replica_uid = None - self._ensure_schema() - self._crypto = crypto + with self.k_lock: + self._db_handle = dbapi2.connect( + sqlcipher_file, + check_same_thread=SQLITE_CHECK_SAME_THREAD) + # set SQLCipher cryptographic parameters + self._set_crypto_pragmas( + self._db_handle, password, raw_key, cipher, kdf_iter, + cipher_page_size) + self._real_replica_uid = None + self._ensure_schema() + self._crypto = crypto def factory(doc_id=None, rev=None, json='{}', has_conflicts=False, syncable=True): return SoledadDocument(doc_id=doc_id, rev=rev, json=json, - has_conflicts=has_conflicts, - syncable=syncable) + has_conflicts=has_conflicts, + syncable=syncable) self.set_document_factory(factory) @classmethod @@ -205,22 +219,35 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): @rtype: SQLCipherDatabase """ if not os.path.isfile(sqlcipher_file): - raise errors.DatabaseDoesNotExist() + raise u1db_errors.DatabaseDoesNotExist() + tries = 2 + # Note: There seems to be a bug in sqlite 3.5.9 (with python2.6) + # where without re-opening the database on Windows, it + # doesn't see the transaction that was just committed while True: - # Note: There seems to be a bug in sqlite 3.5.9 (with python2.6) - # where without re-opening the database on Windows, it - # doesn't see the transaction that was just committed - db_handle = dbapi2.connect(sqlcipher_file) - # set cryptographic params - cls._set_crypto_pragmas( - db_handle, password, raw_key, cipher, kdf_iter, - cipher_page_size) - c = db_handle.cursor() - v, err = cls._which_index_storage(c) - db_handle.close() - if v is not None: - break + + with cls.k_lock: + db_handle = dbapi2.connect( + sqlcipher_file, + check_same_thread=SQLITE_CHECK_SAME_THREAD) + + try: + # set cryptographic params + cls._set_crypto_pragmas( + db_handle, password, raw_key, cipher, kdf_iter, + cipher_page_size) + c = db_handle.cursor() + # XXX if we use it here, it should be public + v, err = cls._which_index_storage(c) + except Exception as exc: + logger.warning("ERROR OPENING DATABASE!") + logger.debug("error was: %r" % exc) + v, err = None, exc + finally: + db_handle.close() + if v is not None: + break # possibly another process is initializing it, wait for it to be # done if tries == 0: @@ -273,7 +300,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): sqlcipher_file, password, document_factory=document_factory, crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter, cipher_page_size=cipher_page_size) - except errors.DatabaseDoesNotExist: + except u1db_errors.DatabaseDoesNotExist: if not create: raise # TODO: remove backend class from here. @@ -301,17 +328,21 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): @rtype: int """ from u1db.sync import Synchronizer - from leap.soledad.target import SoledadSyncTarget + from leap.soledad.client.target import SoledadSyncTarget return Synchronizer( self, SoledadSyncTarget(url, - creds=creds, - crypto=self._crypto)).sync(autocreate=autocreate) + creds=creds, + crypto=self._crypto)).sync(autocreate=autocreate) def _extra_schema_init(self, c): """ Add any extra fields, etc to the basic table definitions. + This method is called by u1db.backends.sqlite_backend._initialize() + method, which is executed when the database schema is created. Here, + we use it to include the "syncable" property for LeapDocuments. + @param c: The cursor for querying the database. @type c: dbapi2.cursor """ @@ -402,10 +433,15 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): except dbapi2.DatabaseError: # assert that we can access it using SQLCipher with the given # key - db_handle = dbapi2.connect(sqlcipher_file) - cls._set_crypto_pragmas( - db_handle, key, raw_key, cipher, kdf_iter, cipher_page_size) - db_handle.cursor().execute('SELECT count(*) FROM sqlite_master') + with cls.k_lock: + db_handle = dbapi2.connect( + sqlcipher_file, + check_same_thread=SQLITE_CHECK_SAME_THREAD) + cls._set_crypto_pragmas( + db_handle, key, raw_key, cipher, + kdf_iter, cipher_page_size) + db_handle.cursor().execute( + 'SELECT count(*) FROM sqlite_master') @classmethod def _set_crypto_pragmas(cls, db_handle, key, raw_key, cipher, kdf_iter, @@ -649,5 +685,12 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): raise NotAnHexString(key) db_handle.cursor().execute('PRAGMA rekey = "x\'%s"' % passphrase) + def __del__(self): + """ + Closes db_handle upon object destruction. + """ + if self._db_handle is not None: + self._db_handle.close() + sqlite_backend.SQLiteDatabase.register_implementation(SQLCipherDatabase) diff --git a/soledad/src/leap/soledad/target.py b/client/src/leap/soledad/client/target.py index 9fac9f54..d0bc3706 100644 --- a/soledad/src/leap/soledad/target.py +++ b/client/src/leap/soledad/client/target.py @@ -32,13 +32,23 @@ from u1db.errors import BrokenSyncStream from u1db.remote.http_target import HTTPSyncTarget -from leap.soledad import soledad_assert -from leap.soledad.document import SoledadDocument -from leap.soledad.crypto import ( +from leap.soledad.common import soledad_assert +from leap.soledad.common.crypto import ( + EncryptionSchemes, + MacMethods, + ENC_JSON_KEY, + ENC_SCHEME_KEY, + ENC_METHOD_KEY, + ENC_IV_KEY, + MAC_KEY, + MAC_METHOD_KEY, +) +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client.auth import TokenBasedAuth +from leap.soledad.client.crypto import ( EncryptionMethods, UnknownEncryptionMethod, ) -from leap.soledad.auth import TokenBasedAuth # @@ -74,38 +84,9 @@ class WrongMac(Exception): # -# Encryption schemes used for encryption. -# - -class EncryptionSchemes(object): - """ - Representation of encryption schemes used to encrypt documents. - """ - - NONE = 'none' - SYMKEY = 'symkey' - PUBKEY = 'pubkey' - - -class MacMethods(object): - """ - Representation of MAC methods used to authenticate document's contents. - """ - - HMAC = 'hmac' - - -# # Crypto utilities for a SoledadDocument. # -ENC_JSON_KEY = '_enc_json' -ENC_SCHEME_KEY = '_enc_scheme' -ENC_METHOD_KEY = '_enc_method' -ENC_IV_KEY = '_enc_iv' -MAC_KEY = '_mac' -MAC_METHOD_KEY = '_mac_method' - def mac_doc(crypto, doc_id, doc_rev, ciphertext, mac_method): """ @@ -168,7 +149,7 @@ def encrypt_doc(crypto, doc): soledad_assert(doc.is_tombstone() is False) # encrypt content using AES-256 CTR mode iv, ciphertext = crypto.encrypt_sym( - doc.get_json(), + str(doc.get_json()), # encryption/decryption routines expect str crypto.doc_passphrase(doc.doc_id), method=EncryptionMethods.AES_256_CTR) # Return a representation for the encrypted content. In the following, we @@ -367,7 +348,8 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): #------------------------------------------------------------- # if arriving content was symmetrically encrypted, we decrypt # it. - doc = SoledadDocument(entry['id'], entry['rev'], entry['content']) + doc = SoledadDocument( + entry['id'], entry['rev'], entry['content']) if doc.content and ENC_SCHEME_KEY in doc.content: if doc.content[ENC_SCHEME_KEY] == \ EncryptionSchemes.SYMKEY: diff --git a/common/changes/feature_3487-split-soledad-into-common-client-and-server b/common/changes/feature_3487-split-soledad-into-common-client-and-server new file mode 100644 index 00000000..2eab6b56 --- /dev/null +++ b/common/changes/feature_3487-split-soledad-into-common-client-and-server @@ -0,0 +1 @@ + o Split soledad package into common, client and server. Closes #3487. diff --git a/common/setup.py b/common/setup.py new file mode 100644 index 00000000..6a432608 --- /dev/null +++ b/common/setup.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +# setup.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +from setuptools import ( + setup, + find_packages +) + + +install_requirements = [ + 'simplejson', + 'oauth', # this is not strictly needed by us, but we need it + # until u1db adds it to its release as a dep. + 'u1db', + 'six==1.1.0', # some tests are incompatible with newer versions of six. +] + + +tests_requirements = [ + 'mock', + 'nose2', + 'testscenarios', + 'leap.common', + 'leap.soledad.server', + 'leap.soledad.client', +] + + +trove_classifiers = ( + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: " + "GNU General Public License v3 or later (GPLv3+)", + "Environment :: Console", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Topic :: Database :: Front-Ends", + "Topic :: Software Development :: Libraries :: Python Modules" +) + + +setup( + name='leap.soledad.common', + version='0.3.0', + url='https://leap.se/', + license='GPLv3+', + description='Synchronization of locally encrypted data among devices.', + author='The LEAP Encryption Access Project', + author_email='info@leap.se', + long_description=( + "Soledad is the part of LEAP that allows application data to be " + "securely shared among devices. It provides, to other parts of the " + "LEAP client, an API for data storage and sync." + ), + namespace_packages=["leap", "leap.soledad"], + packages=find_packages('src', exclude=['leap.soledad.common.tests']), + package_dir={'': 'src'}, + test_suite='leap.soledad.common.tests', + install_requires=install_requirements, + tests_require=tests_requirements, + classifiers=trove_classifiers, +) diff --git a/common/src/leap/__init__.py b/common/src/leap/__init__.py new file mode 100644 index 00000000..f48ad105 --- /dev/null +++ b/common/src/leap/__init__.py @@ -0,0 +1,6 @@ +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/common/src/leap/soledad/__init__.py b/common/src/leap/soledad/__init__.py new file mode 100644 index 00000000..f48ad105 --- /dev/null +++ b/common/src/leap/soledad/__init__.py @@ -0,0 +1,6 @@ +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py new file mode 100644 index 00000000..4e45b828 --- /dev/null +++ b/common/src/leap/soledad/common/__init__.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +Soledad routines common to client and server. +""" + + +# +# Assert functions +# + +# we want to use leap.common.check.leap_assert in case it is available, +# because it also logs in a way other parts of leap can access log messages. + +try: + from leap.common.check import leap_assert as soledad_assert + +except ImportError: + + def soledad_assert(condition, message): + """ + Asserts the condition and displays the message if that's not + met. + + @param condition: condition to check + @type condition: bool + @param message: message to display if the condition isn't met + @type message: str + """ + assert condition, message + +try: + from leap.common.check import leap_assert_type as soledad_assert_type + +except ImportError: + + def soledad_assert_type(var, expectedType): + """ + Helper assert check for a variable's expected type + + @param var: variable to check + @type var: any + @param expectedType: type to check agains + @type expectedType: type + """ + soledad_assert(isinstance(var, expectedType), + "Expected type %r instead of %r" % + (expectedType, type(var))) diff --git a/soledad_server/src/leap/soledad_server/couch.py b/common/src/leap/soledad/common/couch.py index ed5ad6b3..9642e8f3 100644 --- a/soledad_server/src/leap/soledad_server/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -18,7 +18,6 @@ """A U1DB backend that uses CouchDB as its persistence layer.""" -# general imports import uuid import re import simplejson as json @@ -34,7 +33,7 @@ from couchdb.client import Server, Document as CouchDocument from couchdb.http import ResourceNotFound -from leap.soledad_server.objectstore import ( +from leap.soledad.common.objectstore import ( ObjectStoreDatabase, ObjectStoreSyncTarget, ) diff --git a/common/src/leap/soledad/common/crypto.py b/common/src/leap/soledad/common/crypto.py new file mode 100644 index 00000000..2c6bd7a3 --- /dev/null +++ b/common/src/leap/soledad/common/crypto.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# crypto.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +""" +Soledad common crypto bits. +""" + + +# +# Encryption schemes used for encryption. +# + +class EncryptionSchemes(object): + """ + Representation of encryption schemes used to encrypt documents. + """ + + NONE = 'none' + SYMKEY = 'symkey' + PUBKEY = 'pubkey' + + +class MacMethods(object): + """ + Representation of MAC methods used to authenticate document's contents. + """ + + HMAC = 'hmac' + + +# +# Crypto utilities for a SoledadDocument. +# + +ENC_JSON_KEY = '_enc_json' +ENC_SCHEME_KEY = '_enc_scheme' +ENC_METHOD_KEY = '_enc_method' +ENC_IV_KEY = '_enc_iv' +MAC_KEY = '_mac' +MAC_METHOD_KEY = '_mac_method' diff --git a/soledad/src/leap/soledad/document.py b/common/src/leap/soledad/common/document.py index cc24b53a..cc24b53a 100644 --- a/soledad/src/leap/soledad/document.py +++ b/common/src/leap/soledad/common/document.py diff --git a/soledad_server/src/leap/soledad_server/objectstore.py b/common/src/leap/soledad/common/objectstore.py index 8afac3ec..921cf075 100644 --- a/soledad_server/src/leap/soledad_server/objectstore.py +++ b/common/src/leap/soledad/common/objectstore.py @@ -26,11 +26,12 @@ extended to implement OpenStack or Amazon S3 storage, for example. See U1DB documentation for more information on how to use databases. """ + +from u1db import errors from u1db.backends.inmemory import ( InMemoryDatabase, InMemorySyncTarget, ) -from u1db import errors class ObjectStoreDatabase(InMemoryDatabase): diff --git a/soledad/src/leap/soledad/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py index b33f866c..9f47d74a 100644 --- a/soledad/src/leap/soledad/tests/__init__.py +++ b/common/src/leap/soledad/common/tests/__init__.py @@ -3,14 +3,16 @@ Tests to make sure Soledad provides U1DB functionality and more. """ import os +import random +import string import u1db from mock import Mock -from leap.soledad import Soledad -from leap.soledad.document import SoledadDocument -from leap.soledad.crypto import SoledadCrypto -from leap.soledad.target import ( +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client import Soledad +from leap.soledad.client.crypto import SoledadCrypto +from leap.soledad.client.target import ( decrypt_doc, ENC_SCHEME_KEY, ) @@ -40,16 +42,23 @@ class BaseSoledadTest(BaseLeapTest): document_factory=SoledadDocument) self._db2 = u1db.open(self.db2_file, create=True, document_factory=SoledadDocument) + # get a random prefix for each test, so we do not mess with + # concurrency during initialization and shutting down of + # each local db. + self.rand_prefix = ''.join( + map(lambda x: random.choice(string.ascii_letters), range(6))) # initialize soledad by hand so we can control keys - self._soledad = self._soledad_instance(user=self.email) + self._soledad = self._soledad_instance( + prefix=self.rand_prefix, user=self.email) def tearDown(self): self._db1.close() self._db2.close() + self._soledad.close() + # XXX should not access "private" attrs for f in [self._soledad._local_db_path, self._soledad._secrets_path]: if os.path.isfile(f): os.unlink(f) - self._soledad.close() def _soledad_instance(self, user=ADDRESS, passphrase='123', prefix='', @@ -72,7 +81,8 @@ class BaseSoledadTest(BaseLeapTest): return Soledad( user, passphrase, - secrets_path=os.path.join(self.tempdir, prefix, secrets_path), + secrets_path=os.path.join( + self.tempdir, prefix, secrets_path), local_db_path=os.path.join( self.tempdir, prefix, local_db_path), server_url=server_url, # Soledad will fail if not given an url. diff --git a/soledad/src/leap/soledad/tests/couchdb.ini.template b/common/src/leap/soledad/common/tests/couchdb.ini.template index 7d0316f0..7d0316f0 100644 --- a/soledad/src/leap/soledad/tests/couchdb.ini.template +++ b/common/src/leap/soledad/common/tests/couchdb.ini.template diff --git a/soledad/src/leap/soledad/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index a84bb46c..2fb799b7 100644 --- a/soledad/src/leap/soledad/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -27,12 +27,12 @@ from base64 import b64decode from leap.common.files import mkdir_p -from leap.soledad_server import couch -from leap.soledad.tests import u1db_tests as tests -from leap.soledad.tests.u1db_tests import test_backends -from leap.soledad.tests.u1db_tests import test_sync +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.tests import u1db_tests as tests +from leap.soledad.common.tests.u1db_tests import test_backends +from leap.soledad.common.tests.u1db_tests import test_sync +from leap.soledad.common import couch import simplejson as json -from leap.soledad.document import SoledadDocument #----------------------------------------------------------------------------- diff --git a/soledad/src/leap/soledad/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index c727a2ff..01b43299 100644 --- a/soledad/src/leap/soledad/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -32,19 +32,18 @@ from leap.common.testing.basetest import BaseLeapTest from Crypto import Random -from leap.common.testing.basetest import BaseLeapTest -from leap.soledad import ( +from leap.soledad.client import ( Soledad, crypto, target, ) -from leap.soledad.document import SoledadDocument -from leap.soledad.tests import ( +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.tests import ( BaseSoledadTest, KEY_FINGERPRINT, PRIVATE_KEY, ) -from leap.soledad.tests.u1db_tests import ( +from leap.soledad.common.tests.u1db_tests import ( simple_doc, nested_doc, TestCaseWithServer, @@ -137,7 +136,8 @@ class SoledadSecretsTestCase(BaseSoledadTest): secret_id_2 == hashlib.sha256(sol.storage_secret).hexdigest()) def test__has_secret(self): - sol = self._soledad_instance(user='user@leap.se') + sol = self._soledad_instance( + user='user@leap.se', prefix=self.rand_prefix) self.assertTrue(sol._has_secret(), "Should have a secret at " "this point") # setting secret id to None should not interfere in the fact we have a @@ -189,7 +189,7 @@ class MacAuthTestCase(BaseSoledadTest): target.decrypt_doc, self._soledad._crypto, doc) -class SoledadCryptoTestCase(BaseSoledadTest): +class SoledadCryptoAESTestCase(BaseSoledadTest): def test_encrypt_decrypt_sym(self): # generate 256-bit key @@ -239,3 +239,55 @@ class SoledadCryptoTestCase(BaseSoledadTest): cyphertext, wrongkey, iv=iv, method=crypto.EncryptionMethods.AES_256_CTR) self.assertNotEqual('data', plaintext) + + +class SoledadCryptoXSalsa20TestCase(BaseSoledadTest): + + def test_encrypt_decrypt_sym(self): + # generate 256-bit key + key = Random.new().read(32) + iv, cyphertext = self._soledad._crypto.encrypt_sym( + 'data', key, + method=crypto.EncryptionMethods.XSALSA20) + self.assertTrue(cyphertext is not None) + self.assertTrue(cyphertext != '') + self.assertTrue(cyphertext != 'data') + plaintext = self._soledad._crypto.decrypt_sym( + cyphertext, key, iv=iv, + method=crypto.EncryptionMethods.XSALSA20) + self.assertEqual('data', plaintext) + + def test_decrypt_with_wrong_iv_fails(self): + key = Random.new().read(32) + iv, cyphertext = self._soledad._crypto.encrypt_sym( + 'data', key, + method=crypto.EncryptionMethods.XSALSA20) + self.assertTrue(cyphertext is not None) + self.assertTrue(cyphertext != '') + self.assertTrue(cyphertext != 'data') + # get a different iv by changing the first byte + rawiv = binascii.a2b_base64(iv) + wrongiv = rawiv + while wrongiv == rawiv: + wrongiv = os.urandom(1) + rawiv[1:] + plaintext = self._soledad._crypto.decrypt_sym( + cyphertext, key, iv=binascii.b2a_base64(wrongiv), + method=crypto.EncryptionMethods.XSALSA20) + self.assertNotEqual('data', plaintext) + + def test_decrypt_with_wrong_key_fails(self): + key = Random.new().read(32) + iv, cyphertext = self._soledad._crypto.encrypt_sym( + 'data', key, + method=crypto.EncryptionMethods.XSALSA20) + self.assertTrue(cyphertext is not None) + self.assertTrue(cyphertext != '') + self.assertTrue(cyphertext != 'data') + wrongkey = Random.new().read(32) # 256-bits key + # ensure keys are different in case we are extremely lucky + while wrongkey == key: + wrongkey = Random.new().read(32) + plaintext = self._soledad._crypto.decrypt_sym( + cyphertext, wrongkey, iv=iv, + method=crypto.EncryptionMethods.XSALSA20) + self.assertNotEqual('data', plaintext) diff --git a/soledad/src/leap/soledad/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index 24cd68dc..beb7e04d 100644 --- a/soledad/src/leap/soledad/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -26,27 +26,27 @@ import simplejson as json import mock -from leap.soledad import Soledad -from leap.soledad_server import SoledadApp -from leap.soledad_server.auth import URLToAuthorization -from leap.soledad_server.couch import ( +from leap.common.testing.basetest import BaseLeapTest +from leap.soledad.common.couch import ( CouchServerState, CouchDatabase, ) -from leap.soledad import target - - -from leap.common.testing.basetest import BaseLeapTest -from leap.soledad.tests.u1db_tests import ( +from leap.soledad.common.tests.u1db_tests import ( TestCaseWithServer, simple_doc, ) -from leap.soledad.tests.test_couch import CouchDBTestCase -from leap.soledad.tests.test_target import ( +from leap.soledad.common.tests.test_couch import CouchDBTestCase +from leap.soledad.common.tests.test_target import ( make_token_soledad_app, make_leap_document_for_test, token_leap_sync_target, ) +from leap.soledad.client import ( + Soledad, + target, +) +from leap.soledad.server import SoledadApp +from leap.soledad.server.auth import URLToAuthorization class ServerAuthorizationTestCase(BaseLeapTest): diff --git a/soledad/src/leap/soledad/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index 875ecc56..0b753647 100644 --- a/soledad/src/leap/soledad/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -22,25 +22,21 @@ Tests for general Soledad functionality. import os -import re -import tempfile -import simplejson as json from mock import Mock from pysqlcipher.dbapi2 import DatabaseError -from leap.common.testing.basetest import BaseLeapTest from leap.common.events import events_pb2 as proto -from leap.soledad.tests import ( +from leap.soledad.common.tests import ( BaseSoledadTest, ADDRESS, ) from leap import soledad -from leap.soledad import Soledad -from leap.soledad.document import SoledadDocument -from leap.soledad.crypto import SoledadCrypto -from leap.soledad.shared_db import SoledadSharedDatabase -from leap.soledad.target import SoledadSyncTarget +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client import Soledad, PassphraseTooShort +from leap.soledad.client.crypto import SoledadCrypto +from leap.soledad.client.shared_db import SoledadSharedDatabase +from leap.soledad.client.target import SoledadSyncTarget class AuxMethodsTestCase(BaseSoledadTest): @@ -62,7 +58,7 @@ class AuxMethodsTestCase(BaseSoledadTest): sol._gen_secret() sol._load_secrets() sol._init_db() - from leap.soledad.sqlcipher import SQLCipherDatabase + from leap.soledad.client.sqlcipher import SQLCipherDatabase self.assertIsInstance(sol._db, SQLCipherDatabase) def test__init_config_defaults(self): @@ -113,17 +109,26 @@ class AuxMethodsTestCase(BaseSoledadTest): """ sol = self._soledad_instance( 'leap@leap.se', - passphrase='123') + passphrase='123', + prefix=self.rand_prefix, + ) doc = sol.create_doc({'simple': 'doc'}) doc_id = doc.doc_id + # change the passphrase sol.change_passphrase('654321') - # assert we can not use the old passphrase anymore + self.assertRaises( DatabaseError, - self._soledad_instance, 'leap@leap.se', '123') + self._soledad_instance, 'leap@leap.se', + passphrase='123', + prefix=self.rand_prefix) + # use new passphrase and retrieve doc - sol2 = self._soledad_instance('leap@leap.se', '654321') + sol2 = self._soledad_instance( + 'leap@leap.se', + passphrase='654321', + prefix=self.rand_prefix) doc2 = sol2.get_doc(doc_id) self.assertEqual(doc, doc2) @@ -137,9 +142,16 @@ class AuxMethodsTestCase(BaseSoledadTest): passphrase='123') # check that soledad complains about new passphrase length self.assertRaises( - soledad.PassphraseTooShort, + PassphraseTooShort, sol.change_passphrase, '54321') + def test_get_passphrase(self): + """ + Assert passphrase getter works fine. + """ + sol = self._soledad_instance() + self.assertEqual('123', sol.passphrase) + class SoledadSharedDBTestCase(BaseSoledadTest): """ @@ -191,7 +203,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): def setUp(self): BaseSoledadTest.setUp(self) # mock signaling - soledad.signal = Mock() + soledad.client.signal = Mock() def tearDown(self): pass @@ -205,54 +217,54 @@ class SoledadSignalingTestCase(BaseSoledadTest): """ Test that a fresh soledad emits all bootstrap signals. """ - soledad.signal.reset_mock() + soledad.client.signal.reset_mock() # get a fresh instance so it emits all bootstrap signals sol = self._soledad_instance( secrets_path='alternative.json', local_db_path='alternative.u1db') # reverse call order so we can verify in the order the signals were # expected - soledad.signal.mock_calls.reverse() - soledad.signal.call_args = \ - soledad.signal.call_args_list[0] - soledad.signal.call_args_list.reverse() + soledad.client.signal.mock_calls.reverse() + soledad.client.signal.call_args = \ + soledad.client.signal.call_args_list[0] + soledad.client.signal.call_args_list.reverse() # assert signals - soledad.signal.assert_called_with( + soledad.client.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.signal) - soledad.signal.assert_called_with( + self._pop_mock_call(soledad.client.signal) + soledad.client.signal.assert_called_with( proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.signal) - soledad.signal.assert_called_with( + self._pop_mock_call(soledad.client.signal) + soledad.client.signal.assert_called_with( proto.SOLEDAD_CREATING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.signal) - soledad.signal.assert_called_with( + self._pop_mock_call(soledad.client.signal) + soledad.client.signal.assert_called_with( proto.SOLEDAD_DONE_CREATING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.signal) - soledad.signal.assert_called_with( + self._pop_mock_call(soledad.client.signal) + soledad.client.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.signal) - soledad.signal.assert_called_with( + self._pop_mock_call(soledad.client.signal) + soledad.client.signal.assert_called_with( proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.signal) - soledad.signal.assert_called_with( + self._pop_mock_call(soledad.client.signal) + soledad.client.signal.assert_called_with( proto.SOLEDAD_UPLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.signal) - soledad.signal.assert_called_with( + self._pop_mock_call(soledad.client.signal) + soledad.client.signal.assert_called_with( proto.SOLEDAD_DONE_UPLOADING_KEYS, ADDRESS, ) @@ -261,32 +273,32 @@ class SoledadSignalingTestCase(BaseSoledadTest): """ Test that an existent soledad emits some of the bootstrap signals. """ - soledad.signal.reset_mock() + soledad.client.signal.reset_mock() # get an existent instance so it emits only some of bootstrap signals sol = self._soledad_instance() # reverse call order so we can verify in the order the signals were # expected - soledad.signal.mock_calls.reverse() - soledad.signal.call_args = \ - soledad.signal.call_args_list[0] - soledad.signal.call_args_list.reverse() + soledad.client.signal.mock_calls.reverse() + soledad.client.signal.call_args = \ + soledad.client.signal.call_args_list[0] + soledad.client.signal.call_args_list.reverse() # assert signals - soledad.signal.assert_called_with( + soledad.client.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.signal) - soledad.signal.assert_called_with( + self._pop_mock_call(soledad.client.signal) + soledad.client.signal.assert_called_with( proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.signal) - soledad.signal.assert_called_with( + self._pop_mock_call(soledad.client.signal) + soledad.client.signal.assert_called_with( proto.SOLEDAD_UPLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.signal) - soledad.signal.assert_called_with( + self._pop_mock_call(soledad.client.signal) + soledad.client.signal.assert_called_with( proto.SOLEDAD_DONE_UPLOADING_KEYS, ADDRESS, ) @@ -295,7 +307,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): """ Test Soledad emits SOLEDAD_CREATING_KEYS signal. """ - soledad.signal.reset_mock() + soledad.client.signal.reset_mock() # get a fresh instance so it emits all bootstrap signals sol = self._soledad_instance() # mock the actual db sync so soledad does not try to connect to the @@ -304,7 +316,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): # do the sync sol.sync() # assert the signal has been emitted - soledad.signal.assert_called_with( + soledad.client.signal.assert_called_with( proto.SOLEDAD_DONE_DATA_SYNC, ADDRESS, ) @@ -313,7 +325,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): """ Test Soledad emits SOLEDAD_CREATING_KEYS signal. """ - soledad.signal.reset_mock() + soledad.client.signal.reset_mock() sol = self._soledad_instance() # mock the sync target old_get_sync_info = SoledadSyncTarget.get_sync_info @@ -323,7 +335,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): # check for new data to sync sol.need_sync('http://provider/userdb') # assert the signal has been emitted - soledad.signal.assert_called_with( + soledad.client.signal.assert_called_with( proto.SOLEDAD_NEW_DATA_TO_SYNC, ADDRESS, ) diff --git a/soledad/src/leap/soledad/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py index 25d04861..66a673b6 100644 --- a/soledad/src/leap/soledad/tests/test_sqlcipher.py +++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py @@ -43,27 +43,27 @@ from u1db.backends.sqlite_backend import SQLitePartialExpandDatabase # soledad stuff. -from leap.soledad.document import SoledadDocument -from leap.soledad.sqlcipher import ( +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client.sqlcipher import ( SQLCipherDatabase, DatabaseIsNotEncrypted, open as u1db_open, ) -from leap.soledad.target import ( +from leap.soledad.common.crypto import ( EncryptionSchemes, - decrypt_doc, ENC_JSON_KEY, ENC_SCHEME_KEY, ) +from leap.soledad.client.target import decrypt_doc # u1db tests stuff. -from leap.soledad.tests import u1db_tests as tests, BaseSoledadTest -from leap.soledad.tests.u1db_tests import test_sqlite_backend -from leap.soledad.tests.u1db_tests import test_backends -from leap.soledad.tests.u1db_tests import test_open -from leap.soledad.tests.u1db_tests import test_sync -from leap.soledad.target import SoledadSyncTarget +from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest +from leap.soledad.common.tests.u1db_tests import test_sqlite_backend +from leap.soledad.common.tests.u1db_tests import test_backends +from leap.soledad.common.tests.u1db_tests import test_open +from leap.soledad.common.tests.u1db_tests import test_sync +from leap.soledad.client.target import SoledadSyncTarget from leap.common.testing.basetest import BaseLeapTest PASSWORD = '123456' @@ -438,7 +438,8 @@ def sync_via_synchronizer_and_leap(test, db_source, db_target, if trace_hook: test.skipTest("full trace hook unsupported over http") path = test._http_at[db_target] - target = SoledadSyncTarget.connect(test.getURL(path), test._soledad._crypto) + target = SoledadSyncTarget.connect( + test.getURL(path), test._soledad._crypto) target.set_token_credentials('user-uuid', 'auth-token') if trace_hook_shallow: target._set_trace_hook_shallow(trace_hook_shallow) @@ -682,7 +683,7 @@ class SQLCipherSyncTargetTests( def setUp(self): test_sync.DatabaseSyncTargetTests.setUp(self) - BaseSoledadTest.setUp(self) + #BaseSoledadTest.setUp(self) def tearDown(self): test_sync.DatabaseSyncTargetTests.tearDown(self) diff --git a/soledad/src/leap/soledad/tests/test_target.py b/common/src/leap/soledad/common/tests/test_target.py index ca2878a5..5a541745 100644 --- a/soledad/src/leap/soledad/tests/test_target.py +++ b/common/src/leap/soledad/common/tests/test_target.py @@ -34,25 +34,26 @@ from u1db.remote import ( http_target, ) -from leap import soledad -from leap.soledad import ( +from leap.soledad import client +from leap.soledad.client import ( target, auth, + VerifiedHTTPSConnection, ) -from leap.soledad.document import SoledadDocument -from leap.soledad_server import SoledadApp -from leap.soledad_server.auth import SoledadTokenAuthMiddleware +from leap.soledad.common.document import SoledadDocument +from leap.soledad.server import SoledadApp +from leap.soledad.server.auth import SoledadTokenAuthMiddleware -from leap.soledad.tests import u1db_tests as tests -from leap.soledad.tests import BaseSoledadTest -from leap.soledad.tests.u1db_tests import test_backends -from leap.soledad.tests.u1db_tests import test_http_database -from leap.soledad.tests.u1db_tests import test_http_client -from leap.soledad.tests.u1db_tests import test_document -from leap.soledad.tests.u1db_tests import test_remote_sync_target -from leap.soledad.tests.u1db_tests import test_https -from leap.soledad.tests.u1db_tests import test_sync +from leap.soledad.common.tests import u1db_tests as tests +from leap.soledad.common.tests import BaseSoledadTest +from leap.soledad.common.tests.u1db_tests import test_backends +from leap.soledad.common.tests.u1db_tests import test_http_database +from leap.soledad.common.tests.u1db_tests import test_http_client +from leap.soledad.common.tests.u1db_tests import test_document +from leap.soledad.common.tests.u1db_tests import test_remote_sync_target +from leap.soledad.common.tests.u1db_tests import test_https +from leap.soledad.common.tests.u1db_tests import test_sync #----------------------------------------------------------------------------- @@ -77,7 +78,7 @@ def make_token_soledad_app(state): return True return False - # we test for action authorization in leap.soledad.tests.test_server + # we test for action authorization in leap.soledad.common.tests.test_server def _verify_authorization(uuid, environ): return True @@ -504,8 +505,9 @@ def token_leap_https_sync_target(test, host, path): return st -class TestSoledadSyncTargetHttpsSupport(test_https.TestHttpSyncTargetHttpsSupport, - BaseSoledadTest): +class TestSoledadSyncTargetHttpsSupport( + test_https.TestHttpSyncTargetHttpsSupport, + BaseSoledadTest): scenarios = [ ('token_soledad_https', @@ -520,8 +522,8 @@ class TestSoledadSyncTargetHttpsSupport(test_https.TestHttpSyncTargetHttpsSuppor # run smoothly with standard u1db. test_https.TestHttpSyncTargetHttpsSupport.setUp(self) # so here monkey patch again to test our functionality. - http_client._VerifiedHTTPSConnection = soledad.VerifiedHTTPSConnection - soledad.SOLEDAD_CERT = http_client.CA_CERTS + http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection + client.SOLEDAD_CERT = http_client.CA_CERTS def test_working(self): """ @@ -532,7 +534,7 @@ class TestSoledadSyncTargetHttpsSupport(test_https.TestHttpSyncTargetHttpsSuppor """ self.startServer() db = self.request_state._create_database('test') - self.patch(soledad, 'SOLEDAD_CERT', self.cacert_pem) + self.patch(client, 'SOLEDAD_CERT', self.cacert_pem) remote_target = self.getSyncTarget('localhost', 'test') remote_target.record_sync_info('other-id', 2, 'T-id') self.assertEqual( @@ -548,7 +550,7 @@ class TestSoledadSyncTargetHttpsSupport(test_https.TestHttpSyncTargetHttpsSuppor """ self.startServer() self.request_state._create_database('test') - self.patch(soledad, 'SOLEDAD_CERT', self.cacert_pem) + self.patch(client, 'SOLEDAD_CERT', self.cacert_pem) remote_target = self.getSyncTarget('127.0.0.1', 'test') self.assertRaises( http_client.CertificateError, remote_target.record_sync_info, diff --git a/soledad/src/leap/soledad/tests/u1db_tests/README b/common/src/leap/soledad/common/tests/u1db_tests/README index 605f01fa..605f01fa 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/README +++ b/common/src/leap/soledad/common/tests/u1db_tests/README diff --git a/soledad/src/leap/soledad/tests/u1db_tests/__init__.py b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py index 43304b43..3bc12487 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/__init__.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py @@ -189,8 +189,11 @@ class DatabaseBaseTests(TestCase): scenarios = LOCAL_DATABASES_SCENARIOS - def create_database(self, replica_uid): - return self.make_database_for_test(self, replica_uid) + def make_database_for_test(self, replica_uid): + return make_memory_database_for_test(self, replica_uid) + + def create_database(self, *args): + return self.make_database_for_test(self, *args) def copy_database(self, db): # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES diff --git a/soledad/src/leap/soledad/tests/u1db_tests/test_backends.py b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py index a53b01ba..d2a91d11 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/test_backends.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py @@ -26,12 +26,12 @@ from u1db import ( vectorclock, ) -from leap.soledad.tests import u1db_tests as tests +from leap.soledad.common.tests import u1db_tests as tests simple_doc = tests.simple_doc nested_doc = tests.nested_doc -from leap.soledad.tests.u1db_tests.test_remote_sync_target import ( +from leap.soledad.common.tests.u1db_tests.test_remote_sync_target import ( make_http_app, make_oauth_http_app, ) diff --git a/soledad/src/leap/soledad/tests/u1db_tests/test_document.py b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py index e706e1a9..8b30ed51 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/test_document.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py @@ -17,7 +17,7 @@ from u1db import errors -from leap.soledad.tests import u1db_tests as tests +from leap.soledad.common.tests import u1db_tests as tests class TestDocument(tests.TestCase): diff --git a/soledad/src/leap/soledad/tests/u1db_tests/test_http_app.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py index e0729aa2..789006ba 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/test_http_app.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py @@ -30,7 +30,7 @@ from u1db import ( sync, ) -from leap.soledad.tests import u1db_tests as tests +from leap.soledad.common.tests import u1db_tests as tests from u1db.remote import ( http_app, diff --git a/soledad/src/leap/soledad/tests/u1db_tests/test_http_client.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py index 42e98461..08e9714e 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/test_http_client.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py @@ -26,7 +26,7 @@ from u1db import ( errors, ) -from leap.soledad.tests import u1db_tests as tests +from leap.soledad.common.tests import u1db_tests as tests from u1db.remote import ( http_client, diff --git a/soledad/src/leap/soledad/tests/u1db_tests/test_http_database.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py index f21e6da1..3f3c7bba 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/test_http_database.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py @@ -27,13 +27,13 @@ from u1db import ( Document, ) -from leap.soledad.tests import u1db_tests as tests +from leap.soledad.common.tests import u1db_tests as tests from u1db.remote import ( http_database, http_target, ) -from leap.soledad.tests.u1db_tests.test_remote_sync_target import ( +from leap.soledad.common.tests.u1db_tests.test_remote_sync_target import ( make_http_app, ) diff --git a/soledad/src/leap/soledad/tests/u1db_tests/test_https.py b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py index 62180f8c..c086fbc0 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/test_https.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py @@ -12,8 +12,8 @@ from u1db.remote import ( ) from leap import soledad -from leap.soledad.tests import u1db_tests as tests -from leap.soledad.tests.u1db_tests.test_remote_sync_target import ( +from leap.soledad.common.tests import u1db_tests as tests +from leap.soledad.common.tests.u1db_tests.test_remote_sync_target import ( make_oauth_http_app, ) @@ -75,7 +75,7 @@ class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): # order to maintain the compatibility with u1db default tests, we undo # that replacement here. http_client._VerifiedHTTPSConnection = \ - soledad.old__VerifiedHTTPSConnection + soledad.client.old__VerifiedHTTPSConnection super(TestHttpSyncTargetHttpsSupport, self).setUp() def getSyncTarget(self, host, path=None): diff --git a/soledad/src/leap/soledad/tests/u1db_tests/test_open.py b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py index 0ff307e8..13425b4f 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/test_open.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py @@ -22,9 +22,9 @@ from u1db import ( errors, open as u1db_open, ) -from leap.soledad.tests import u1db_tests as tests +from leap.soledad.common.tests import u1db_tests as tests from u1db.backends import sqlite_backend -from leap.soledad.tests.u1db_tests.test_backends import TestAlternativeDocument +from leap.soledad.common.tests.u1db_tests.test_backends import TestAlternativeDocument class TestU1DBOpen(tests.TestCase): diff --git a/soledad/src/leap/soledad/tests/u1db_tests/test_remote_sync_target.py b/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py index 66d404d2..3793e0df 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/test_remote_sync_target.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py @@ -22,7 +22,7 @@ from u1db import ( errors, ) -from leap.soledad.tests import u1db_tests as tests +from leap.soledad.common.tests import u1db_tests as tests from u1db.remote import ( http_app, diff --git a/soledad/src/leap/soledad/tests/u1db_tests/test_sqlite_backend.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py index 1380e4b1..a53ea6cc 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/test_sqlite_backend.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py @@ -27,10 +27,10 @@ from u1db import ( query_parser, ) -from leap.soledad.tests import u1db_tests as tests +from leap.soledad.common.tests import u1db_tests as tests from u1db.backends import sqlite_backend -from leap.soledad.tests.u1db_tests.test_backends import TestAlternativeDocument +from leap.soledad.common.tests.u1db_tests.test_backends import TestAlternativeDocument simple_doc = '{"key": "value"}' diff --git a/soledad/src/leap/soledad/tests/u1db_tests/test_sync.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py index 96aa2736..5346d540 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/test_sync.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py @@ -26,7 +26,7 @@ from u1db import ( SyncTarget, ) -from leap.soledad.tests import u1db_tests as tests +from leap.soledad.common.tests import u1db_tests as tests from u1db.backends import ( inmemory, @@ -35,7 +35,7 @@ from u1db.remote import ( http_target, ) -from leap.soledad.tests.u1db_tests.test_remote_sync_target import ( +from leap.soledad.common.tests.u1db_tests.test_remote_sync_target import ( make_http_app, make_oauth_http_app, ) @@ -85,7 +85,8 @@ class DatabaseSyncTargetTests(tests.DatabaseBaseTests, whitebox = True def setUp(self): - super(DatabaseSyncTargetTests, self).setUp() + tests.DatabaseBaseTests.setUp(self) + tests.TestCaseWithServer.setUp(self) self.db, self.st = self.create_db_and_target(self) self.other_changes = [] @@ -96,6 +97,9 @@ class DatabaseSyncTargetTests(tests.DatabaseBaseTests, del self.db super(DatabaseSyncTargetTests, self).tearDown() + def create_db_and_target(self, *args): + return _make_local_db_and_target(*args) + def receive_doc(self, doc, gen, trans_id): self.other_changes.append( (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) diff --git a/soledad/src/leap/soledad/tests/u1db_tests/testing-certs/Makefile b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/Makefile index 2385e75b..2385e75b 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/testing-certs/Makefile +++ b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/Makefile diff --git a/soledad/src/leap/soledad/tests/u1db_tests/testing-certs/cacert.pem b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/cacert.pem index c019a730..c019a730 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/testing-certs/cacert.pem +++ b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/cacert.pem diff --git a/soledad/src/leap/soledad/tests/u1db_tests/testing-certs/testing.cert b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.cert index 985684fb..985684fb 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/testing-certs/testing.cert +++ b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.cert diff --git a/soledad/src/leap/soledad/tests/u1db_tests/testing-certs/testing.key b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.key index d83d4920..d83d4920 100644 --- a/soledad/src/leap/soledad/tests/u1db_tests/testing-certs/testing.key +++ b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.key diff --git a/server/changes/feature_3487-split-soledad-into-common-client-and-server b/server/changes/feature_3487-split-soledad-into-common-client-and-server new file mode 100644 index 00000000..2eab6b56 --- /dev/null +++ b/server/changes/feature_3487-split-soledad-into-common-client-and-server @@ -0,0 +1 @@ + o Split soledad package into common, client and server. Closes #3487. diff --git a/soledad_server/pkg/soledad b/server/pkg/soledad index c233731e..c233731e 100644 --- a/soledad_server/pkg/soledad +++ b/server/pkg/soledad diff --git a/soledad_server/setup.py b/server/setup.py index 16cebca1..89354a57 100644 --- a/soledad_server/setup.py +++ b/server/setup.py @@ -31,10 +31,9 @@ install_requirements = [ 'oauth', # this is not strictly needed by us, but we need it # until u1db adds it to its release as a dep. 'u1db', - 'six==1.1.0', 'routes', 'PyOpenSSL', - 'leap.soledad>=0.3.0', + 'leap.soledad.common>=0.3.0', ] @@ -44,6 +43,7 @@ else: # XXX this should go only for linux/mac data_files = [("/etc/init.d/", ["pkg/soledad"])] + trove_classifiers = ( "Development Status :: 3 - Alpha", "Intended Audience :: Developers", @@ -58,8 +58,9 @@ trove_classifiers = ( "Topic :: Software Development :: Libraries :: Python Modules" ) + setup( - name='leap.soledad_server', + name='leap.soledad.server', version='0.3.0', url='https://leap.se/', license='GPLv3+', @@ -71,9 +72,10 @@ setup( "securely shared among devices. It provides, to other parts of the " "LEAP client, an API for data storage and sync." ), - namespace_packages=["leap"], + namespace_packages=["leap", "leap.soledad"], packages=find_packages('src'), package_dir={'': 'src'}, install_requires=install_requirements, + data_files=data_files, classifiers=trove_classifiers, ) diff --git a/server/src/leap/__init__.py b/server/src/leap/__init__.py new file mode 100644 index 00000000..f48ad105 --- /dev/null +++ b/server/src/leap/__init__.py @@ -0,0 +1,6 @@ +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/server/src/leap/soledad/__init__.py b/server/src/leap/soledad/__init__.py new file mode 100644 index 00000000..f48ad105 --- /dev/null +++ b/server/src/leap/soledad/__init__.py @@ -0,0 +1,6 @@ +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/soledad_server/src/leap/soledad_server/__init__.py b/server/src/leap/soledad/server/__init__.py index 18a0546e..4ed8efc9 100644 --- a/soledad_server/src/leap/soledad_server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -43,8 +43,8 @@ if version.base() == "12.0.0": sys.modules['OpenSSL.tsafe'] = old_tsafe -from leap.soledad_server.auth import SoledadTokenAuthMiddleware -from leap.soledad_server.couch import CouchServerState +from leap.soledad.server.auth import SoledadTokenAuthMiddleware +from leap.soledad.common.couch import CouchServerState #----------------------------------------------------------------------------- @@ -122,4 +122,4 @@ state = CouchServerState(conf['couch_url']) # WSGI application that may be used by `twistd -web` application = SoledadTokenAuthMiddleware(SoledadApp(state)) -resource = WSGIResource(reactor, reactor.getThreadPool(), application)
\ No newline at end of file +resource = WSGIResource(reactor, reactor.getThreadPool(), application) diff --git a/soledad_server/src/leap/soledad_server/auth.py b/server/src/leap/soledad/server/auth.py index 3bcfcf04..3bcfcf04 100644 --- a/soledad_server/src/leap/soledad_server/auth.py +++ b/server/src/leap/soledad/server/auth.py diff --git a/soledad/src/leap/soledad/backends/__init__.py b/soledad/src/leap/soledad/backends/__init__.py deleted file mode 100644 index e69de29b..00000000 --- a/soledad/src/leap/soledad/backends/__init__.py +++ /dev/null diff --git a/soledad/src/leap/soledad/backends/leap_backend.py b/soledad/src/leap/soledad/backends/leap_backend.py deleted file mode 100644 index a8d76b67..00000000 --- a/soledad/src/leap/soledad/backends/leap_backend.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- -# leap_backend.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -This file exists to provide backwards compatibility with code that uses -Soledad before the refactor that removed the leap_backend module. -""" - - -import logging -import warnings - - -from leap.soledad import document -from leap.soledad import target - - -logger = logging.getLogger(name=__name__) - - -def warn(oldclass, newclass): - """ - Warns about deprecation of C{oldclass}, which must be substituted by - C{newclass}. - - @param oldclass: The class that is deprecated. - @type oldclass: type - @param newclass: The class that should be used instead. - @type newclass: type - """ - message = \ - "%s is deprecated and will be removed soon. Please use %s instead." \ - % (str(oldclass), str(newclass)) - print message - logger.warning(message) - warnings.warn(message, DeprecationWarning, stacklevel=2) - - -class LeapDocument(document.SoledadDocument): - """ - This class exists to provide backwards compatibility with code that still - uses C{leap.soledad.backends.leap_backend.LeapDocument}. - """ - - def __init__(self, *args, **kwargs): - warn(self.__class__, document.SoledadDocument) - document.SoledadDocument.__init__(self, *args, **kwargs) - - -class EncryptionSchemes(target.EncryptionSchemes): - """ - This class exists to provide backwards compatibility with code that still - uses C{leap.soledad.backends.leap_backend.EncryptionSchemes}. - """ - - def __init__(self, *args, **kwargs): - warn(self.__class__, target.EncryptionSchemes) - target.EncryptionSchemes.__init__(self, *args, **kwargs) diff --git a/soledad/src/leap/soledad/dbwrapper.py b/soledad/src/leap/soledad/dbwrapper.py deleted file mode 100644 index c16c4925..00000000 --- a/soledad/src/leap/soledad/dbwrapper.py +++ /dev/null @@ -1,183 +0,0 @@ -# -*- coding: utf-8 -*- -# dbwrapper.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Thread-safe wrapper for sqlite/pysqlcipher. - -*TODO* -At some point we surely will want to switch to a twisted way of dealing -with this, using defers and proper callbacks. But I had this tested for -some time so postponing that refactor. -""" -import logging -import threading -import Queue -import time - -import exceptions - -from functools import partial - -from leap.soledad import sqlcipher - -logger = logging.getLogger(__name__) - - -class SQLCipherWrapper(threading.Thread): - - def __init__(self, *args, **kwargs): - """ - Initializes a wrapper that proxies method and attribute - access to an underlying SQLCipher instance. We instantiate sqlcipher - in a thread, and all method accesses communicate with it using a - Queue. - - :param *args: position arguments to pass to pysqlcipher initialization - :type args: tuple - - :param **kwargs: keyword arguments to pass to pysqlcipher - initialization - :type kwargs: dict - """ - threading.Thread.__init__(self) - self._db = None - self._wrargs = args, kwargs - - self._queue = Queue.Queue() - self._stopped = threading.Event() - - self.start() - - def _init_db(self): - """ - Initializes sqlcipher database. - - This is called on a separate thread. - """ - # instantiate u1db - args, kwargs = self._wrargs - self._db = sqlcipher.open(*args, **kwargs) - - def run(self): - """ - Main loop for the sqlcipher thread. - """ - logger.debug("SQLCipherWrapper thread started.") - logger.debug("Initializing sqlcipher") - end_mths = ("__end_thread", "_SQLCipherWrapper__end_thread") - - self._init_db() - self._lock = threading.Lock() - - ct = 0 - started = False - - while True: - if self._db is None: - if started: - break - if ct > 10: - break # XXX DEBUG - logger.debug('db not ready yet, waiting...') - time.sleep(1) - ct += 1 - - started = True - - with self._lock: - try: - mth, q, wrargs = self._queue.get() - except: - logger.error("exception getting args from queue") - - res = None - attr = getattr(self._db, mth, None) - if not attr: - if mth not in end_mths: - logger.error('method %s does not exist' % (mth,)) - res = AttributeError( - "_db instance has no attribute %s" % mth) - - elif callable(attr): - # invoke the method with the passed args - args = wrargs.get('args', []) - kwargs = wrargs.get('kwargs', {}) - try: - res = attr(*args, **kwargs) - except Exception as e: - logger.error( - "Error on proxied method %s: '%r'." % ( - attr, e)) - res = e - else: - # non-callable attribute - res = attr - logger.debug('returning proxied db call...') - q.put(res) - - if mth in end_mths: - logger.debug('ending thread') - break - - logger.debug("SQLCipherWrapper thread terminated.") - self._stopped.set() - - def close(self): - """ - Closes the sqlcipher database and finishes the thread. This method - should always be called explicitely. - """ - self.__getattr__('close')() - self.__end_thread() - - def __getattr__(self, attr): - """ - Returns _db proxied attributes. - """ - - def __proxied_mth(method, *args, **kwargs): - if not self._stopped.isSet(): - wrargs = {'args': args, 'kwargs': kwargs} - q = Queue.Queue() - self._queue.put((method, q, wrargs)) - res = q.get() - q.task_done() - - if isinstance(res, exceptions.BaseException): - # XXX should get the original bt - raise res - return res - else: - logger.warning("tried to call proxied meth " - "but stopped is set: %s" % - (method,)) - - rgetattr = object.__getattribute__ - - if attr != "_db": - proxied = partial(__proxied_mth, attr) - return proxied - - # fallback to regular behavior - return rgetattr(self, attr) - - def __del__(self): - """ - Do not trust this get called. No guarantees given. Because of a funny - dance with the refs and the way the gc works, we should be calling the - close method explicitely. - """ - self.close() |