diff options
34 files changed, 487 insertions, 120 deletions
@@ -19,7 +19,9 @@ MANIFEST _trial_temp .DS_Store scripts/profiling/sync/profiles +tags testing/htmlcov testing/.coverage testing/test-env +testing/.benchmarks diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip index 8983b6b5..5a61a7b5 100644 --- a/client/pkg/requirements.pip +++ b/client/pkg/requirements.pip @@ -5,3 +5,5 @@ cryptography pysqlcipher;python_version=="2.7" pysqlcipher3;python_version=="3.4" treq +weakref +enum34 diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 3a114021..bcad78db 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -17,12 +17,14 @@ """ Soledad - Synchronization Of Locally Encrypted Data Among Devices. """ -from leap.soledad.client.api import Soledad from leap.soledad.common import soledad_assert +from .api import Soledad +from ._document import Document, AttachmentStates from ._version import get_versions __version__ = get_versions()['version'] del get_versions -__all__ = ['soledad_assert', 'Soledad', '__version__'] +__all__ = ['soledad_assert', 'Soledad', 'Document', 'AttachmentStates', + '__version__'] diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py index 7a1d508e..e66cc600 100644 --- a/client/src/leap/soledad/client/_crypto.py +++ b/client/src/leap/soledad/client/_crypto.py @@ -135,7 +135,7 @@ class SoledadCrypto(object): and wrapping the result as a simple JSON string with a "raw" key. :param doc: the document to be encrypted. - :type doc: SoledadDocument + :type doc: Document :return: A deferred whose callback will be invoked with a JSON string containing the ciphertext as the value of "raw" key. :rtype: twisted.internet.defer.Deferred @@ -159,7 +159,7 @@ class SoledadCrypto(object): the decrypted cleartext content from the encrypted document. :param doc: the document to be decrypted. - :type doc: SoledadDocument + :type doc: Document :return: The decrypted cleartext content of the document. :rtype: str """ @@ -225,7 +225,7 @@ def decrypt_sym(data, key, iv, method=ENC_METHOD.aes_256_gcm): class BlobEncryptor(object): """ Produces encrypted data from the cleartext data associated with a given - SoledadDocument using AES-256 cipher in GCM mode. + Document using AES-256 cipher in GCM mode. The production happens using a Twisted's FileBodyProducer, which uses a Cooperator to schedule calls and can be paused/resumed. Each call takes at diff --git a/client/src/leap/soledad/client/_database/__init__.py b/client/src/leap/soledad/client/_database/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/client/src/leap/soledad/client/_database/__init__.py diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/_database/adbapi.py index b002055e..5c28d108 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/_database/adbapi.py @@ -30,8 +30,9 @@ from zope.proxy import ProxyBase, setProxiedObject from leap.soledad.common.log import getLogger from leap.soledad.common.errors import DatabaseAccessError -from leap.soledad.client import sqlcipher as soledad_sqlcipher -from leap.soledad.client.pragmas import set_init_pragmas + +from . import sqlcipher +from . import pragmas if sys.version_info[0] < 3: from pysqlcipher import dbapi2 @@ -73,7 +74,7 @@ def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): :rtype: U1DBConnectionPool """ if openfun is None and driver == "pysqlcipher": - openfun = partial(set_init_pragmas, opts=opts) + openfun = partial(pragmas.set_init_pragmas, opts=opts) return U1DBConnectionPool( opts, # the following params are relayed "as is" to twisted's @@ -87,7 +88,7 @@ class U1DBConnection(adbapi.Connection): A wrapper for a U1DB connection instance. """ - u1db_wrapper = soledad_sqlcipher.SoledadSQLCipherWrapper + u1db_wrapper = sqlcipher.SoledadSQLCipherWrapper """ The U1DB wrapper to use. """ diff --git a/client/src/leap/soledad/client/_blobs.py b/client/src/leap/soledad/client/_database/blobs.py index 1475b302..79404bf3 100644 --- a/client/src/leap/soledad/client/_blobs.py +++ b/client/src/leap/soledad/client/_database/blobs.py @@ -20,8 +20,9 @@ Clientside BlobBackend Storage. from urlparse import urljoin +import binascii +import errno import os -import uuid import base64 from io import BytesIO @@ -34,13 +35,18 @@ from twisted.web.client import FileBodyProducer import treq -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client import pragmas -from leap.soledad.client._pipes import TruncatedTailPipe, PreamblePipe from leap.soledad.common.errors import SoledadError -from _crypto import DocInfo, BlobEncryptor, BlobDecryptor -from _http import HTTPClient +from .._document import BlobDoc +from .._crypto import DocInfo +from .._crypto import BlobEncryptor +from .._crypto import BlobDecryptor +from .._http import HTTPClient +from .._pipes import TruncatedTailPipe +from .._pipes import PreamblePipe + +from . import pragmas +from . import sqlcipher logger = Logger() @@ -129,6 +135,16 @@ class DecrypterBuffer(object): return self.decrypter._end_stream(), real_size +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + class BlobManager(object): """ Ideally, the decrypting flow goes like this: @@ -149,6 +165,7 @@ class BlobManager(object): self, local_path, remote, key, secret, user, token=None, cert_file=None): if local_path: + mkdir_p(os.path.dirname(local_path)) self.local = SQLiteBlobBackend(local_path, key) self.remote = remote self.secret = secret @@ -280,10 +297,13 @@ class SQLiteBlobBackend(object): def __init__(self, path, key=None): self.path = os.path.abspath( os.path.join(path, 'soledad_blob.db')) + mkdir_p(os.path.dirname(self.path)) if not key: raise ValueError('key cannot be None') backend = 'pysqlcipher.dbapi2' - opts = SQLCipherOptions('/tmp/ignored', key) + opts = sqlcipher.SQLCipherOptions( + '/tmp/ignored', binascii.b2a_hex(key), + is_raw_key=True, create=True) pragmafun = partial(pragmas.set_init_pragmas, opts=opts) openfun = _sqlcipherInitFactory(pragmafun) @@ -292,7 +312,11 @@ class SQLiteBlobBackend(object): cp_openfun=openfun, cp_min=1, cp_max=2, cp_name='blob_pool') def close(self): - return self.dbpool.close() + from twisted._threads import AlreadyQuit + try: + self.dbpool.close() + except AlreadyQuit: + pass @defer.inlineCallbacks def put(self, blob_id, blob_fd, size=None): @@ -352,20 +376,6 @@ def _sqlcipherInitFactory(fun): return _initialize -class BlobDoc(object): - - # TODO probably not needed, but convenient for testing for now. - - def __init__(self, content, blob_id): - - self.blob_id = blob_id - self.is_blob = True - self.blob_fd = content - if blob_id is None: - blob_id = uuid.uuid4().get_hex() - self.blob_id = blob_id - - # # testing facilities # @@ -431,8 +441,7 @@ def testit(reactor): # TODO convert these into proper unittests def _manager(): - if not os.path.isdir(args.path): - os.makedirs(args.path) + mkdir_p(os.path.dirname(args.path)) manager = BlobManager( args.path, args.url, 'A' * 32, args.secret, diff --git a/common/src/leap/soledad/common/l2db/backends/dbschema.sql b/client/src/leap/soledad/client/_database/dbschema.sql index ae027fc5..ae027fc5 100644 --- a/common/src/leap/soledad/common/l2db/backends/dbschema.sql +++ b/client/src/leap/soledad/client/_database/dbschema.sql diff --git a/client/src/leap/soledad/client/pragmas.py b/client/src/leap/soledad/client/_database/pragmas.py index 870ed63e..870ed63e 100644 --- a/client/src/leap/soledad/client/pragmas.py +++ b/client/src/leap/soledad/client/_database/pragmas.py diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/_database/sqlcipher.py index 2c995d5a..d22017bd 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/_database/sqlcipher.py @@ -50,16 +50,16 @@ from twisted.internet import reactor from twisted.internet import defer from twisted.enterprise import adbapi -from leap.soledad.common.document import SoledadDocument from leap.soledad.common.log import getLogger from leap.soledad.common.l2db import errors as u1db_errors -from leap.soledad.common.l2db import Document -from leap.soledad.common.l2db.backends import sqlite_backend from leap.soledad.common.errors import DatabaseAccessError from leap.soledad.client.http_target import SoledadHTTPSyncTarget from leap.soledad.client.sync import SoledadSynchronizer -from leap.soledad.client import pragmas + +from .._document import Document +from . import sqlite +from . import pragmas if sys.version_info[0] < 3: from pysqlcipher import dbapi2 as sqlcipher_dbapi2 @@ -69,8 +69,8 @@ else: logger = getLogger(__name__) -# Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2 -sqlite_backend.dbapi2 = sqlcipher_dbapi2 +# Monkey-patch u1db.backends.sqlite with pysqlcipher.dbapi2 +sqlite.dbapi2 = sqlcipher_dbapi2 # we may want to collect statistics from the sync process @@ -193,7 +193,7 @@ class SQLCipherOptions(object): # The SQLCipher database # -class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): +class SQLCipherDatabase(sqlite.SQLitePartialExpandDatabase): """ A U1DB implementation that uses SQLCipher as its persistence layer. """ @@ -232,7 +232,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): # --------------------------------------------------------- self._ensure_schema() - self.set_document_factory(soledad_doc_factory) + self.set_document_factory(doc_factory) self._prime_replica_uid() def _prime_replica_uid(self): @@ -341,7 +341,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :param doc: The new version of the document. :type doc: u1db.Document """ - sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes( + sqlite.SQLitePartialExpandDatabase._put_and_update_indexes( self, old_doc, doc) c = self._db_handle.cursor() c.execute('UPDATE document SET syncable=? WHERE doc_id=?', @@ -361,7 +361,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :return: a Document object. :type: u1db.Document """ - doc = sqlite_backend.SQLitePartialExpandDatabase._get_doc( + doc = sqlite.SQLitePartialExpandDatabase._get_doc( self, doc_id, check_for_conflicts) if doc: c = self._db_handle.cursor() @@ -437,7 +437,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase): self._opts, check_same_thread=False) self._real_replica_uid = None self._ensure_schema() - self.set_document_factory(soledad_doc_factory) + self.set_document_factory(doc_factory) except sqlcipher_dbapi2.DatabaseError as e: raise DatabaseAccessError(str(e)) @@ -505,7 +505,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase): return self._get_generation() -class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase): +class U1DBSQLiteBackend(sqlite.SQLitePartialExpandDatabase): """ A very simple wrapper for u1db around sqlcipher backend. @@ -537,7 +537,7 @@ class SoledadSQLCipherWrapper(SQLCipherDatabase): self._db_handle = conn self._real_replica_uid = None self._ensure_schema() - self.set_document_factory(soledad_doc_factory) + self.set_document_factory(doc_factory) self._prime_replica_uid() @@ -562,7 +562,7 @@ def _assert_db_is_encrypted(opts): # If the regular backend succeeds, then we need to stop because # the database was not properly initialized. try: - sqlite_backend.SQLitePartialExpandDatabase(opts.path) + sqlite.SQLitePartialExpandDatabase(opts.path) except sqlcipher_dbapi2.DatabaseError: # assert that we can access it using SQLCipher with the given # key @@ -583,17 +583,17 @@ class DatabaseIsNotEncrypted(Exception): pass -def soledad_doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False, - syncable=True): +def doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False, + syncable=True): """ Return a default Soledad Document. Used in the initialization for SQLCipherDatabase """ - return SoledadDocument(doc_id=doc_id, rev=rev, json=json, - has_conflicts=has_conflicts, syncable=syncable) + return Document(doc_id=doc_id, rev=rev, json=json, + has_conflicts=has_conflicts, syncable=syncable) -sqlite_backend.SQLiteDatabase.register_implementation(SQLCipherDatabase) +sqlite.SQLiteDatabase.register_implementation(SQLCipherDatabase) # diff --git a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py b/client/src/leap/soledad/client/_database/sqlite.py index 4f7b1259..4f7b1259 100644 --- a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py +++ b/client/src/leap/soledad/client/_database/sqlite.py diff --git a/client/src/leap/soledad/client/_document.py b/client/src/leap/soledad/client/_document.py new file mode 100644 index 00000000..9ba08e93 --- /dev/null +++ b/client/src/leap/soledad/client/_document.py @@ -0,0 +1,253 @@ +# -*- coding: utf-8 -*- +# _document.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Everything related to documents. +""" + +import enum +import weakref +import uuid + +from twisted.internet import defer + +from zope.interface import Interface +from zope.interface import implementer + +from leap.soledad.common.document import SoledadDocument + + +class IDocumentWithAttachment(Interface): + """ + A document that can have an attachment. + """ + + def set_store(self, store): + """ + Set the store used by this file to manage attachments. + + :param store: The store used to manage attachments. + :type store: Soledad + """ + + def put_attachment(self, fd): + """ + Attach data to this document. + + Add the attachment to local storage, enqueue for upload. + + The document content will be updated with a pointer to the attachment, + but the document has to be manually put in the database to reflect + modifications. + + :param fd: A file-like object whose content will be attached to this + document. + :type fd: file-like + + :return: A deferred which fires when the attachment has been added to + local storage. + :rtype: Deferred + """ + + def get_attachment(self): + """ + Return the data attached to this document. + + If document content contains a pointer to the attachment, try to get + the attachment from local storage and, if not found, from remote + storage. + + :return: A deferred which fires with a file like-object whose content + is the attachment of this document, or None if nothing is + attached. + :rtype: Deferred + """ + + def delete_attachment(self): + """ + Delete the attachment of this document. + + The pointer to the attachment will be removed from the document + content, but the document has to be manually put in the database to + reflect modifications. + + :return: A deferred which fires when the attachment has been deleted + from local storage. + :rtype: Deferred + """ + + def attachment_state(self): + """ + Return the state of the attachment of this document. + + The state is a member of AttachmentStates and is of one of NONE, + LOCAL, REMOTE or SYNCED. + + :return: A deferred which fires with The state of the attachment of + this document. + :rtype: Deferred + """ + + def is_dirty(self): + """ + Return wether this document's content differs from the contents stored + in local database. + + :return: Whether this document is dirty or not. + :rtype: bool + """ + + def upload_attachment(self): + """ + Upload this document's attachment. + + :return: A deferred which fires with the state of the attachment after + it's been uploaded, or NONE if there's no attachment for this + document. + :rtype: Deferred + """ + + def download_attachment(self): + """ + Download this document's attachment. + + :return: A deferred which fires with the state of the attachment after + it's been downloaded, or NONE if there's no attachment for + this document. + :rtype: Deferred + """ + + +class BlobDoc(object): + + # TODO probably not needed, but convenient for testing for now. + + def __init__(self, content, blob_id): + + self.blob_id = blob_id + self.is_blob = True + self.blob_fd = content + if blob_id is None: + blob_id = uuid.uuid4().get_hex() + self.blob_id = blob_id + + +class AttachmentStates(enum.IntEnum): + NONE = 0 + LOCAL = 1 + REMOTE = 2 + SYNCED = 4 + + +@implementer(IDocumentWithAttachment) +class Document(SoledadDocument): + + def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False, + syncable=True, store=None): + SoledadDocument.__init__(self, doc_id=doc_id, rev=rev, json=json, + has_conflicts=has_conflicts, + syncable=syncable) + self.set_store(store) + + # + # properties + # + + @property + def _manager(self): + if not self.store or not hasattr(self.store, 'blobmanager'): + raise Exception('No blob manager found to manage attachments.') + return self.store.blobmanager + + @property + def _blob_id(self): + if self.content and 'blob_id' in self.content: + return self.content['blob_id'] + return None + + def get_store(self): + return self._store() if self._store else None + + def set_store(self, store): + self._store = weakref.ref(store) if store else None + + store = property(get_store, set_store) + + # + # attachment api + # + + def put_attachment(self, fd): + # add pointer to content + blob_id = self._blob_id or str(uuid.uuid4()) + if not self.content: + self.content = {} + self.content['blob_id'] = blob_id + # put using manager + blob = BlobDoc(fd, blob_id) + fd.seek(0, 2) + size = fd.tell() + fd.seek(0) + return self._manager.put(blob, size) + + def get_attachment(self): + if not self._blob_id: + return defer.succeed(None) + return self._manager.get(self._blob_id) + + def delete_attachment(self): + raise NotImplementedError + + @defer.inlineCallbacks + def attachment_state(self): + state = AttachmentStates.NONE + + if not self._blob_id: + defer.returnValue(state) + + local_list = yield self._manager.local_list() + if self._blob_id in local_list: + state |= AttachmentStates.LOCAL + + remote_list = yield self._manager.remote_list() + if self._blob_id in remote_list: + state |= AttachmentStates.REMOTE + + defer.returnValue(state) + + @defer.inlineCallbacks + def is_dirty(self): + stored = yield self.store.get_doc(self.doc_id) + if stored.content != self.content: + defer.returnValue(True) + defer.returnValue(False) + + @defer.inlineCallbacks + def upload_attachment(self): + if not self._blob_id: + defer.returnValue(AttachmentStates.NONE) + + fd = yield self._manager.get_blob(self._blob_id) + # TODO: turn following method into a public one + yield self._manager._encrypt_and_upload(self._blob_id, fd) + defer.returnValue(self.attachment_state()) + + @defer.inlineCallbacks + def download_attachment(self): + if not self._blob_id: + defer.returnValue(None) + yield self.get_attachment() + defer.returnValue(self.attachment_state()) diff --git a/client/src/leap/soledad/client/_secrets/storage.py b/client/src/leap/soledad/client/_secrets/storage.py index 6ea89900..85713a48 100644 --- a/client/src/leap/soledad/client/_secrets/storage.py +++ b/client/src/leap/soledad/client/_secrets/storage.py @@ -23,8 +23,8 @@ from hashlib import sha256 from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common.log import getLogger -from leap.soledad.common.document import SoledadDocument from leap.soledad.client.shared_db import SoledadSharedDatabase +from leap.soledad.client._document import Document from leap.soledad.client._secrets.util import emit, UserDataMixin @@ -111,7 +111,7 @@ class SecretsStorage(UserDataMixin): def save_remote(self, encrypted): doc = self._remote_doc if not doc: - doc = SoledadDocument(doc_id=self._remote_doc_id()) + doc = Document(doc_id=self._remote_doc_id()) doc.content = encrypted db = self._shared_db if not db: diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 18fb35d7..3dd99227 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -51,13 +51,14 @@ from leap.soledad.common.l2db.remote import http_client from leap.soledad.common.l2db.remote.ssl_match_hostname import match_hostname from leap.soledad.common.errors import DatabaseAccessError -from leap.soledad.client import adbapi -from leap.soledad.client import events as soledad_events -from leap.soledad.client import interfaces as soledad_interfaces -from leap.soledad.client import sqlcipher -from leap.soledad.client._recovery_code import RecoveryCode -from leap.soledad.client._secrets import Secrets -from leap.soledad.client._crypto import SoledadCrypto +from . import events as soledad_events +from . import interfaces as soledad_interfaces +from ._crypto import SoledadCrypto +from ._database import adbapi +from ._database import blobs +from ._database import sqlcipher +from ._recovery_code import RecoveryCode +from ._secrets import Secrets logger = getLogger(__name__) @@ -171,6 +172,7 @@ class Soledad(object): self._recovery_code = RecoveryCode() self._secrets = Secrets(self) self._crypto = SoledadCrypto(self._secrets.remote_secret) + self._init_blobmanager() try: # initialize database access, trap any problems so we can shutdown @@ -262,6 +264,13 @@ class Soledad(object): sync_exchange_phase = _p return sync_phase, sync_exchange_phase + def _init_blobmanager(self): + path = os.path.join(os.path.dirname(self._local_db_path), 'blobs') + url = urlparse.urljoin(self.server_url, 'blobs/%s' % uuid) + key = self._secrets.local_key + self.blobmanager = blobs.BlobManager(path, url, key, self.uuid, + self.token, SOLEDAD_CERT) + # # Closing methods # @@ -272,6 +281,7 @@ class Soledad(object): """ logger.debug("closing soledad") self._dbpool.close() + self.blobmanager.close() if getattr(self, '_dbsyncer', None): self._dbsyncer.close() @@ -306,7 +316,7 @@ class Soledad(object): ============================== WARNING ============================== :param doc: A document with new content. - :type doc: leap.soledad.common.document.SoledadDocument + :type doc: leap.soledad.common.document.Document :return: A deferred whose callback will be invoked with the new revision identifier for the document. The document object will also be updated. @@ -323,7 +333,7 @@ class Soledad(object): This will also set doc.content to None. :param doc: A document to be deleted. - :type doc: leap.soledad.common.document.SoledadDocument + :type doc: leap.soledad.common.document.Document :return: A deferred. :rtype: twisted.internet.defer.Deferred """ @@ -387,6 +397,7 @@ class Soledad(object): """ return self._defer("get_all_docs", include_deleted) + @defer.inlineCallbacks def create_doc(self, content, doc_id=None): """ Create a new document. @@ -408,8 +419,9 @@ class Soledad(object): # create_doc (and probably to put_doc too). There are cases (mail # payloads for example) in which we already have the encoding in the # headers, so we don't need to guess it. - d = self._defer("create_doc", content, doc_id=doc_id) - return d + doc = yield self._defer("create_doc", content, doc_id=doc_id) + doc.set_store(self) + defer.returnValue(doc) def create_doc_from_json(self, json, doc_id=None): """ @@ -590,7 +602,7 @@ class Soledad(object): the time you GET_DOC_CONFLICTS until the point where you RESOLVE) :param doc: A Document with the new content to be inserted. - :type doc: SoledadDocument + :type doc: Document :param conflicted_doc_revs: A list of revisions that the new content supersedes. :type conflicted_doc_revs: list(str) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 09e90171..4795846c 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -167,7 +167,7 @@ class SoledadCrypto(object): Wrapper around encrypt_docstr that accepts the document as argument. :param doc: the document. - :type doc: SoledadDocument + :type doc: Document """ key = self.doc_passphrase(doc.doc_id) @@ -179,7 +179,7 @@ class SoledadCrypto(object): Wrapper around decrypt_doc_dict that accepts the document as argument. :param doc: the document. - :type doc: SoledadDocument + :type doc: Document :return: json string with the decrypted document :rtype: str @@ -194,7 +194,7 @@ class SoledadCrypto(object): # -# Crypto utilities for a SoledadDocument. +# Crypto utilities for a Document. # def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv, @@ -439,7 +439,7 @@ def is_symmetrically_encrypted(doc): Return True if the document was symmetrically encrypted. :param doc: The document to check. - :type doc: SoledadDocument + :type doc: Document :rtype: bool """ diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py index 92bc85d6..276b1200 100644 --- a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py @@ -28,7 +28,7 @@ from twisted.internet import defer, reactor from leap.soledad.common import l2db from leap.soledad.client import adbapi -from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client._database.sqlcipher import SQLCipherOptions folder = os.environ.get("TMPDIR", "tmp") diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py index 429566c7..d288582b 100644 --- a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py @@ -27,7 +27,7 @@ import sys from twisted.internet import defer, reactor from leap.soledad.client import adbapi -from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client._database.sqlcipher import SQLCipherOptions from leap.soledad.common import l2db diff --git a/client/src/leap/soledad/client/examples/use_adbapi.py b/client/src/leap/soledad/client/examples/use_adbapi.py index 39301b41..25b32307 100644 --- a/client/src/leap/soledad/client/examples/use_adbapi.py +++ b/client/src/leap/soledad/client/examples/use_adbapi.py @@ -24,7 +24,7 @@ import os from twisted.internet import defer, reactor from leap.soledad.client import adbapi -from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client._database.sqlcipher import SQLCipherOptions from leap.soledad.common import l2db diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py index cf4984d1..9d456830 100644 --- a/client/src/leap/soledad/client/http_target/fetch.py +++ b/client/src/leap/soledad/client/http_target/fetch.py @@ -23,10 +23,10 @@ from leap.soledad.client.events import emit_async from leap.soledad.client.http_target.support import RequestBody from leap.soledad.common.log import getLogger from leap.soledad.client._crypto import is_symmetrically_encrypted -from leap.soledad.common.document import SoledadDocument from leap.soledad.common.l2db import errors from leap.soledad.client import crypto as old_crypto +from .._document import Document from . import fetch_protocol logger = getLogger(__name__) @@ -113,7 +113,7 @@ class HTTPDocFetcher(object): @defer.inlineCallbacks def __atomic_doc_parse(self, doc_info, content, total): - doc = SoledadDocument(doc_info['id'], doc_info['rev'], content) + doc = Document(doc_info['id'], doc_info['rev'], content) if is_symmetrically_encrypted(content): content = (yield self._crypto.decrypt_doc(doc)).getvalue() elif old_crypto.is_symmetrically_encrypted(doc): diff --git a/client/src/leap/soledad/client/interfaces.py b/client/src/leap/soledad/client/interfaces.py index 1be47df7..0600449f 100644 --- a/client/src/leap/soledad/client/interfaces.py +++ b/client/src/leap/soledad/client/interfaces.py @@ -69,7 +69,7 @@ class ILocalStorage(Interface): Update a document in the local encrypted database. :param doc: the document to update - :type doc: SoledadDocument + :type doc: Document :return: a deferred that will fire with the new revision identifier for @@ -82,7 +82,7 @@ class ILocalStorage(Interface): Delete a document from the local encrypted database. :param doc: the document to delete - :type doc: SoledadDocument + :type doc: Document :return: a deferred that will fire with ... @@ -102,7 +102,7 @@ class ILocalStorage(Interface): :return: A deferred that will fire with the document object, containing a - SoledadDocument, or None if it could not be found + Document, or None if it could not be found :rtype: Deferred """ @@ -147,7 +147,7 @@ class ILocalStorage(Interface): :type doc_id: str :return: - A deferred tht will fire with the new document (SoledadDocument + A deferred tht will fire with the new document (Document instance). :rtype: Deferred """ @@ -167,7 +167,7 @@ class ILocalStorage(Interface): :param doc_id: An optional identifier specifying the document id. :type doc_id: :return: - A deferred that will fire with the new document (A SoledadDocument + A deferred that will fire with the new document (A Document instance) :rtype: Deferred """ @@ -304,7 +304,7 @@ class ILocalStorage(Interface): Mark a document as no longer conflicted. :param doc: a document with the new content to be inserted. - :type doc: SoledadDocument + :type doc: Document :param conflicted_doc_revs: A deferred that will fire with a list of revisions that the new content supersedes. diff --git a/common/src/leap/soledad/common/l2db/__init__.py b/common/src/leap/soledad/common/l2db/__init__.py index 568897c4..ebbf546a 100644 --- a/common/src/leap/soledad/common/l2db/__init__.py +++ b/common/src/leap/soledad/common/l2db/__init__.py @@ -37,8 +37,8 @@ def open(path, create, document_factory=None): parameters as Document.__init__. :return: An instance of Database. """ - from leap.soledad.common.l2db.backends import sqlite_backend - return sqlite_backend.SQLiteDatabase.open_database( + from leap.soledad.client._database import sqlite + return sqlite.SQLiteDatabase.open_database( path, create=create, document_factory=document_factory) diff --git a/common/src/leap/soledad/common/l2db/remote/server_state.py b/common/src/leap/soledad/common/l2db/remote/server_state.py index e20b4679..89ac5742 100644 --- a/common/src/leap/soledad/common/l2db/remote/server_state.py +++ b/common/src/leap/soledad/common/l2db/remote/server_state.py @@ -42,10 +42,9 @@ class ServerState(object): def open_database(self, path): """Open a database at the given location.""" - from u1db.backends import sqlite_backend + from leap.soledad.client._database import sqlite full_path = self._relpath(path) - return sqlite_backend.SQLiteDatabase.open_database(full_path, - create=False) + return sqlite.SQLiteDatabase.open_database(full_path, create=False) def check_database(self, path): """Check if the database at the given location exists. @@ -57,14 +56,13 @@ class ServerState(object): def ensure_database(self, path): """Ensure database at the given location.""" - from u1db.backends import sqlite_backend + from leap.soledad.client._database import sqlite full_path = self._relpath(path) - db = sqlite_backend.SQLiteDatabase.open_database(full_path, - create=True) + db = sqlite.SQLiteDatabase.open_database(full_path, create=True) return db, db._replica_uid def delete_database(self, path): """Delete database at the given location.""" - from u1db.backends import sqlite_backend + from leap.soledad.client._database import sqlite full_path = self._relpath(path) - sqlite_backend.SQLiteDatabase.delete_database(full_path) + sqlite.SQLiteDatabase.delete_database(full_path) diff --git a/testing/test_soledad/u1db_tests/__init__.py b/testing/test_soledad/u1db_tests/__init__.py index 1575b859..ccbb6ca6 100644 --- a/testing/test_soledad/u1db_tests/__init__.py +++ b/testing/test_soledad/u1db_tests/__init__.py @@ -37,11 +37,12 @@ from twisted.internet import reactor from leap.soledad.common.l2db import errors from leap.soledad.common.l2db import Document from leap.soledad.common.l2db.backends import inmemory -from leap.soledad.common.l2db.backends import sqlite_backend from leap.soledad.common.l2db.remote import server_state from leap.soledad.common.l2db.remote import http_app from leap.soledad.common.l2db.remote import http_target +from leap.soledad.client._database import sqlite + if sys.version_info[0] < 3: from pysqlcipher import dbapi2 else: @@ -140,7 +141,7 @@ def copy_memory_database_for_test(test, db): def make_sqlite_partial_expanded_for_test(test, replica_uid): - db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + db = sqlite.SQLitePartialExpandDatabase(':memory:') db._set_replica_uid(replica_uid) return db @@ -151,7 +152,7 @@ def copy_sqlite_partial_expanded_for_test(test, db): # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR # HOUSE. - new_db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + new_db = sqlite.SQLitePartialExpandDatabase(':memory:') tmpfile = StringIO() for line in db._db_handle.iterdump(): if 'sqlite_sequence' not in line: # work around bug in iterdump diff --git a/testing/test_soledad/u1db_tests/test_open.py b/testing/test_soledad/u1db_tests/test_open.py index b572fba0..eaaee72f 100644 --- a/testing/test_soledad/u1db_tests/test_open.py +++ b/testing/test_soledad/u1db_tests/test_open.py @@ -27,7 +27,8 @@ from test_soledad.u1db_tests.test_backends import TestAlternativeDocument from leap.soledad.common.l2db import errors from leap.soledad.common.l2db import open as u1db_open -from leap.soledad.common.l2db.backends import sqlite_backend + +from leap.soledad.client._database import sqlite @skip("Skiping tests imported from U1DB.") @@ -47,7 +48,7 @@ class TestU1DBOpen(tests.TestCase): db = u1db_open(self.db_path, create=True) self.addCleanup(db.close) self.assertTrue(os.path.exists(self.db_path)) - self.assertIsInstance(db, sqlite_backend.SQLiteDatabase) + self.assertIsInstance(db, sqlite.SQLiteDatabase) def test_open_with_factory(self): db = u1db_open(self.db_path, create=True, @@ -56,7 +57,7 @@ class TestU1DBOpen(tests.TestCase): self.assertEqual(TestAlternativeDocument, db._factory) def test_open_existing(self): - db = sqlite_backend.SQLitePartialExpandDatabase(self.db_path) + db = sqlite.SQLitePartialExpandDatabase(self.db_path) self.addCleanup(db.close) doc = db.create_doc_from_json(tests.simple_doc) # Even though create=True, we shouldn't wipe the db @@ -66,8 +67,8 @@ class TestU1DBOpen(tests.TestCase): self.assertEqual(doc, doc2) def test_open_existing_no_create(self): - db = sqlite_backend.SQLitePartialExpandDatabase(self.db_path) + db = sqlite.SQLitePartialExpandDatabase(self.db_path) self.addCleanup(db.close) db2 = u1db_open(self.db_path, create=False) self.addCleanup(db2.close) - self.assertIsInstance(db2, sqlite_backend.SQLitePartialExpandDatabase) + self.assertIsInstance(db2, sqlite.SQLitePartialExpandDatabase) diff --git a/testing/test_soledad/util.py b/testing/test_soledad/util.py index 6ffb60b6..0335d544 100644 --- a/testing/test_soledad/util.py +++ b/testing/test_soledad/util.py @@ -46,9 +46,9 @@ from leap.soledad.common.couch.state import CouchServerState from leap.soledad.client import Soledad from leap.soledad.client import http_target from leap.soledad.client import auth -from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.client.sqlcipher import SQLCipherOptions from leap.soledad.client._crypto import is_symmetrically_encrypted +from leap.soledad.client._database.sqlcipher import SQLCipherDatabase +from leap.soledad.client._database.sqlcipher import SQLCipherOptions from leap.soledad.server import SoledadApp diff --git a/testing/tests/blobs/test_blob_manager.py b/testing/tests/blobs/test_blob_manager.py index 7609936d..69a272c8 100644 --- a/testing/tests/blobs/test_blob_manager.py +++ b/testing/tests/blobs/test_blob_manager.py @@ -19,8 +19,8 @@ Tests for BlobManager. """ from twisted.trial import unittest from twisted.internet import defer -from leap.soledad.client._blobs import BlobManager, BlobDoc, FIXED_REV -from leap.soledad.client._blobs import BlobAlreadyExistsError +from leap.soledad.client._database.blobs import BlobManager, BlobDoc, FIXED_REV +from leap.soledad.client._database.blobs import BlobAlreadyExistsError from io import BytesIO from mock import Mock import pytest diff --git a/testing/tests/blobs/test_decrypter_buffer.py b/testing/tests/blobs/test_decrypter_buffer.py index 288c9975..edaa66e2 100644 --- a/testing/tests/blobs/test_decrypter_buffer.py +++ b/testing/tests/blobs/test_decrypter_buffer.py @@ -18,12 +18,16 @@ Tests for blobs decrypter buffer. A component which is used as a decryption sink during blob stream download. """ +from io import BytesIO +from mock import Mock + from twisted.trial import unittest from twisted.internet import defer -from leap.soledad.client._blobs import DecrypterBuffer, BlobManager, FIXED_REV + +from leap.soledad.client._database.blobs import DecrypterBuffer +from leap.soledad.client._database.blobs import BlobManager +from leap.soledad.client._database.blobs import FIXED_REV from leap.soledad.client import _crypto -from io import BytesIO -from mock import Mock class DecrypterBufferCase(unittest.TestCase): diff --git a/testing/tests/blobs/test_sqlcipher_client_backend.py b/testing/tests/blobs/test_sqlcipher_client_backend.py index 951bac22..865a64e1 100644 --- a/testing/tests/blobs/test_sqlcipher_client_backend.py +++ b/testing/tests/blobs/test_sqlcipher_client_backend.py @@ -19,7 +19,7 @@ Tests for sqlcipher backend on blobs client. """ from twisted.trial import unittest from twisted.internet import defer -from leap.soledad.client._blobs import SQLiteBlobBackend +from leap.soledad.client._database.blobs import SQLiteBlobBackend from io import BytesIO import pytest diff --git a/testing/tests/client/test_attachments.py b/testing/tests/client/test_attachments.py new file mode 100644 index 00000000..ad4595e9 --- /dev/null +++ b/testing/tests/client/test_attachments.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# test_attachments.py +# Copyright (C) 2017 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Tests for document attachments. +""" + +import pytest + +from io import BytesIO +from mock import Mock + +from twisted.internet import defer +from test_soledad.util import BaseSoledadTest + + +from leap.soledad.client import AttachmentStates + + +def mock_response(doc): + doc._manager._client.get = Mock( + return_value=defer.succeed(Mock(code=200, json=lambda: []))) + doc._manager._client.put = Mock( + return_value=defer.succeed(Mock(code=200))) + + +@pytest.mark.usefixture('method_tmpdir') +class AttachmentTests(BaseSoledadTest): + + @defer.inlineCallbacks + def test_create_doc_saves_store(self): + doc = yield self._soledad.create_doc({}) + self.assertEqual(self._soledad, doc.store) + + @defer.inlineCallbacks + def test_put_attachment(self): + doc = yield self._soledad.create_doc({}) + mock_response(doc) + yield doc.put_attachment(BytesIO('test')) + local_list = yield doc._manager.local_list() + self.assertIn(doc._blob_id, local_list) + + @defer.inlineCallbacks + def test_get_attachment(self): + doc = yield self._soledad.create_doc({}) + mock_response(doc) + yield doc.put_attachment(BytesIO('test')) + fd = yield doc.get_attachment() + self.assertEqual('test', fd.read()) + + @defer.inlineCallbacks + def test_attachment_state(self): + doc = yield self._soledad.create_doc({}) + state = yield doc.attachment_state() + self.assertEqual(AttachmentStates.NONE, state) + mock_response(doc) + yield doc.put_attachment(BytesIO('test')) + state = yield doc.attachment_state() + self.assertEqual(AttachmentStates.LOCAL, state) + + @defer.inlineCallbacks + def test_is_dirty(self): + doc = yield self._soledad.create_doc({}) + dirty = yield doc.is_dirty() + self.assertFalse(dirty) + doc.content = {'test': True} + dirty = yield doc.is_dirty() + self.assertTrue(dirty) diff --git a/testing/tests/client/test_aux_methods.py b/testing/tests/client/test_aux_methods.py index 729aa28a..da33255c 100644 --- a/testing/tests/client/test_aux_methods.py +++ b/testing/tests/client/test_aux_methods.py @@ -22,7 +22,7 @@ import os from pytest import inlineCallbacks from leap.soledad.client import Soledad -from leap.soledad.client.adbapi import U1DBConnectionPool +from leap.soledad.client._database.adbapi import U1DBConnectionPool from leap.soledad.client._secrets.util import SecretsError from test_soledad.util import BaseSoledadTest diff --git a/testing/tests/server/test_blobs_server.py b/testing/tests/server/test_blobs_server.py index 2e5af01f..cd39833f 100644 --- a/testing/tests/server/test_blobs_server.py +++ b/testing/tests/server/test_blobs_server.py @@ -24,8 +24,10 @@ from twisted.web.server import Site from twisted.internet import reactor from twisted.internet import defer from treq._utils import set_global_pool + from leap.soledad.server import _blobs as server_blobs -from leap.soledad.client._blobs import BlobManager, BlobAlreadyExistsError +from leap.soledad.client._database.blobs import BlobManager +from leap.soledad.client._database.blobs import BlobAlreadyExistsError class BlobServerTestCase(unittest.TestCase): diff --git a/testing/tests/sqlcipher/test_async.py b/testing/tests/sqlcipher/test_async.py index 42c315fe..dac6c6b9 100644 --- a/testing/tests/sqlcipher/test_async.py +++ b/testing/tests/sqlcipher/test_async.py @@ -20,8 +20,8 @@ import hashlib from twisted.internet import defer from test_soledad.util import BaseSoledadTest -from leap.soledad.client import adbapi -from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client._database import adbapi +from leap.soledad.client._database import sqlcipher class ASyncSQLCipherRetryTestCase(BaseSoledadTest): @@ -42,7 +42,7 @@ class ASyncSQLCipherRetryTestCase(BaseSoledadTest): def _get_dbpool(self): tmpdb = os.path.join(self.tempdir, "test.soledad") - opts = SQLCipherOptions(tmpdb, "secret", create=True) + opts = sqlcipher.SQLCipherOptions(tmpdb, "secret", create=True) return adbapi.getConnectionPool(opts) def _get_sample(self): diff --git a/testing/tests/sqlcipher/test_backend.py b/testing/tests/sqlcipher/test_backend.py index 6e9595db..4f614fb3 100644 --- a/testing/tests/sqlcipher/test_backend.py +++ b/testing/tests/sqlcipher/test_backend.py @@ -26,14 +26,13 @@ import sys # l2db stuff. from leap.soledad.common.l2db import errors from leap.soledad.common.l2db import query_parser -from leap.soledad.common.l2db.backends.sqlite_backend \ - import SQLitePartialExpandDatabase # soledad stuff. from leap.soledad.common.document import SoledadDocument -from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import DatabaseIsNotEncrypted +from leap.soledad.client._database.sqlite import SQLitePartialExpandDatabase +from leap.soledad.client._database.sqlcipher import SQLCipherDatabase +from leap.soledad.client._database.sqlcipher import SQLCipherOptions +from leap.soledad.client._database.sqlcipher import DatabaseIsNotEncrypted # u1db tests stuff. from test_soledad import u1db_tests as tests diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py index a54a02ee..e1bd7de1 100644 --- a/testing/tests/sync/test_sync_target.py +++ b/testing/tests/sync/test_sync_target.py @@ -32,9 +32,9 @@ from twisted.internet import defer from leap.soledad.client import http_target as target from leap.soledad.client.http_target.fetch_protocol import DocStreamReceiver -from leap.soledad.client.sqlcipher import SQLCipherU1DBSync -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client._database.sqlcipher import SQLCipherU1DBSync +from leap.soledad.client._database.sqlcipher import SQLCipherOptions +from leap.soledad.client._database.sqlcipher import SQLCipherDatabase from leap.soledad.client import _crypto from leap.soledad.common import l2db |