[feat] add attachments api

11 files changed, 320 insertions, 35 deletions

diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip
index 8983b6b5..5a61a7b5 100644
--- a/client/pkg/requirements.pip
+++ b/client/pkg/requirements.pip
@@ -5,3 +5,5 @@ cryptography
 pysqlcipher;python_version=="2.7"
 pysqlcipher3;python_version=="3.4"
 treq
+weakref
+enum34
diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py
index 3a114021..bcad78db 100644
--- a/client/src/leap/soledad/client/__init__.py
+++ b/client/src/leap/soledad/client/__init__.py
@@ -17,12 +17,14 @@
 """
 Soledad - Synchronization Of Locally Encrypted Data Among Devices.
 """
-from leap.soledad.client.api import Soledad
 from leap.soledad.common import soledad_assert
 
+from .api import Soledad
+from ._document import Document, AttachmentStates
 from ._version import get_versions
 
 __version__ = get_versions()['version']
 del get_versions
 
-__all__ = ['soledad_assert', 'Soledad', '__version__']
+__all__ = ['soledad_assert', 'Soledad', 'Document', 'AttachmentStates',
+           '__version__']
diff --git a/client/src/leap/soledad/client/_blobs.py b/client/src/leap/soledad/client/_blobs.py
index 6f692f6b..90007427 100644
--- a/client/src/leap/soledad/client/_blobs.py
+++ b/client/src/leap/soledad/client/_blobs.py
@@ -20,6 +20,7 @@ Clientside BlobBackend Storage.
 
 from urlparse import urljoin
 
+import errno
 import os
 import uuid
 import base64
@@ -129,6 +130,16 @@ class DecrypterBuffer(object):
         return self.decrypter._end_stream(), real_size
 
 
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
+
 class BlobManager(object):
     """
     Ideally, the decrypting flow goes like this:
@@ -149,6 +160,7 @@ class BlobManager(object):
             self, local_path, remote, key, secret, user, token=None,
             cert_file=None):
         if local_path:
+            mkdir_p(os.path.dirname(local_path))
             self.local = SQLiteBlobBackend(local_path, key)
         self.remote = remote
         self.secret = secret
@@ -277,6 +289,7 @@ class SQLiteBlobBackend(object):
     def __init__(self, path, key=None):
         self.path = os.path.abspath(
             os.path.join(path, 'soledad_blob.db'))
+        mkdir_p(os.path.dirname(self.path))
         if not key:
             raise ValueError('key cannot be None')
         backend = 'pysqlcipher.dbapi2'
@@ -289,7 +302,11 @@ class SQLiteBlobBackend(object):
             cp_openfun=openfun, cp_min=1, cp_max=2, cp_name='blob_pool')
 
     def close(self):
-        return self.dbpool.close()
+        from twisted._threads import AlreadyQuit
+        try:
+            self.dbpool.close()
+        except AlreadyQuit:
+            pass
 
     @defer.inlineCallbacks
     def put(self, blob_id, blob_fd, size=None):
@@ -423,7 +440,7 @@ def testit(reactor):
 
     def _manager():
         if not os.path.isdir(args.path):
-            os.makedirs(args.path)
+            mkdir_p(args.path)
         manager = BlobManager(
             args.path, args.url,
             'A' * 32, args.secret,
diff --git a/client/src/leap/soledad/client/_crypto.py b/client/src/leap/soledad/client/_crypto.py
index 7a1d508e..e66cc600 100644
--- a/client/src/leap/soledad/client/_crypto.py
+++ b/client/src/leap/soledad/client/_crypto.py
@@ -135,7 +135,7 @@ class SoledadCrypto(object):
         and wrapping the result as a simple JSON string with a "raw" key.
 
         :param doc: the document to be encrypted.
-        :type doc: SoledadDocument
+        :type doc: Document
         :return: A deferred whose callback will be invoked with a JSON string
             containing the ciphertext as the value of "raw" key.
         :rtype: twisted.internet.defer.Deferred
@@ -159,7 +159,7 @@ class SoledadCrypto(object):
         the decrypted cleartext content from the encrypted document.
 
         :param doc: the document to be decrypted.
-        :type doc: SoledadDocument
+        :type doc: Document
         :return: The decrypted cleartext content of the document.
         :rtype: str
         """
@@ -225,7 +225,7 @@ def decrypt_sym(data, key, iv, method=ENC_METHOD.aes_256_gcm):
 class BlobEncryptor(object):
     """
     Produces encrypted data from the cleartext data associated with a given
-    SoledadDocument using AES-256 cipher in GCM mode.
+    Document using AES-256 cipher in GCM mode.
 
     The production happens using a Twisted's FileBodyProducer, which uses a
     Cooperator to schedule calls and can be paused/resumed. Each call takes at
diff --git a/client/src/leap/soledad/client/_document.py b/client/src/leap/soledad/client/_document.py
new file mode 100644
index 00000000..9ba08e93
--- /dev/null
+++ b/client/src/leap/soledad/client/_document.py
@@ -0,0 +1,253 @@
+# -*- coding: utf-8 -*-
+# _document.py
+# Copyright (C) 2017 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Everything related to documents.
+"""
+
+import enum
+import weakref
+import uuid
+
+from twisted.internet import defer
+
+from zope.interface import Interface
+from zope.interface import implementer
+
+from leap.soledad.common.document import SoledadDocument
+
+
+class IDocumentWithAttachment(Interface):
+    """
+    A document that can have an attachment.
+    """
+
+    def set_store(self, store):
+        """
+        Set the store used by this file to manage attachments.
+
+        :param store: The store used to manage attachments.
+        :type store: Soledad
+        """
+
+    def put_attachment(self, fd):
+        """
+        Attach data to this document.
+
+        Add the attachment to local storage, enqueue for upload.
+
+        The document content will be updated with a pointer to the attachment,
+        but the document has to be manually put in the database to reflect
+        modifications.
+
+        :param fd: A file-like object whose content will be attached to this
+                   document.
+        :type fd: file-like
+
+        :return: A deferred which fires when the attachment has been added to
+                 local storage.
+        :rtype: Deferred
+        """
+
+    def get_attachment(self):
+        """
+        Return the data attached to this document.
+
+        If document content contains a pointer to the attachment, try to get
+        the attachment from local storage and, if not found, from remote
+        storage.
+
+        :return: A deferred which fires with a file like-object whose content
+                 is the attachment of this document, or None if nothing is
+                 attached.
+        :rtype: Deferred
+        """
+
+    def delete_attachment(self):
+        """
+        Delete the attachment of this document.
+
+        The pointer to the attachment will be removed from the document
+        content, but the document has to be manually put in the database to
+        reflect modifications.
+
+        :return: A deferred which fires when the attachment has been deleted
+                 from local storage.
+        :rtype: Deferred
+        """
+
+    def attachment_state(self):
+        """
+        Return the state of the attachment of this document.
+
+        The state is a member of AttachmentStates and is of one of NONE,
+        LOCAL, REMOTE or SYNCED.
+
+        :return: A deferred which fires with The state of the attachment of
+                 this document.
+        :rtype: Deferred
+        """
+
+    def is_dirty(self):
+        """
+        Return wether this document's content differs from the contents stored
+        in local database.
+
+        :return: Whether this document is dirty or not.
+        :rtype: bool
+        """
+
+    def upload_attachment(self):
+        """
+        Upload this document's attachment.
+
+        :return: A deferred which fires with the state of the attachment after
+                 it's been uploaded, or NONE if there's no attachment for this
+                 document.
+        :rtype: Deferred
+        """
+
+    def download_attachment(self):
+        """
+        Download this document's attachment.
+
+        :return: A deferred which fires with the state of the attachment after
+                 it's been downloaded, or NONE if there's no attachment for
+                 this document.
+        :rtype: Deferred
+        """
+
+
+class BlobDoc(object):
+
+    # TODO probably not needed, but convenient for testing for now.
+
+    def __init__(self, content, blob_id):
+
+        self.blob_id = blob_id
+        self.is_blob = True
+        self.blob_fd = content
+        if blob_id is None:
+            blob_id = uuid.uuid4().get_hex()
+        self.blob_id = blob_id
+
+
+class AttachmentStates(enum.IntEnum):
+    NONE = 0
+    LOCAL = 1
+    REMOTE = 2
+    SYNCED = 4
+
+
+@implementer(IDocumentWithAttachment)
+class Document(SoledadDocument):
+
+    def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False,
+                 syncable=True, store=None):
+        SoledadDocument.__init__(self, doc_id=doc_id, rev=rev, json=json,
+                                 has_conflicts=has_conflicts,
+                                 syncable=syncable)
+        self.set_store(store)
+
+    #
+    # properties
+    #
+
+    @property
+    def _manager(self):
+        if not self.store or not hasattr(self.store, 'blobmanager'):
+            raise Exception('No blob manager found to manage attachments.')
+        return self.store.blobmanager
+
+    @property
+    def _blob_id(self):
+        if self.content and 'blob_id' in self.content:
+            return self.content['blob_id']
+        return None
+
+    def get_store(self):
+        return self._store() if self._store else None
+
+    def set_store(self, store):
+        self._store = weakref.ref(store) if store else None
+
+    store = property(get_store, set_store)
+
+    #
+    # attachment api
+    #
+
+    def put_attachment(self, fd):
+        # add pointer to content
+        blob_id = self._blob_id or str(uuid.uuid4())
+        if not self.content:
+            self.content = {}
+        self.content['blob_id'] = blob_id
+        # put using manager
+        blob = BlobDoc(fd, blob_id)
+        fd.seek(0, 2)
+        size = fd.tell()
+        fd.seek(0)
+        return self._manager.put(blob, size)
+
+    def get_attachment(self):
+        if not self._blob_id:
+            return defer.succeed(None)
+        return self._manager.get(self._blob_id)
+
+    def delete_attachment(self):
+        raise NotImplementedError
+
+    @defer.inlineCallbacks
+    def attachment_state(self):
+        state = AttachmentStates.NONE
+
+        if not self._blob_id:
+            defer.returnValue(state)
+
+        local_list = yield self._manager.local_list()
+        if self._blob_id in local_list:
+            state |= AttachmentStates.LOCAL
+
+        remote_list = yield self._manager.remote_list()
+        if self._blob_id in remote_list:
+            state |= AttachmentStates.REMOTE
+
+        defer.returnValue(state)
+
+    @defer.inlineCallbacks
+    def is_dirty(self):
+        stored = yield self.store.get_doc(self.doc_id)
+        if stored.content != self.content:
+            defer.returnValue(True)
+        defer.returnValue(False)
+
+    @defer.inlineCallbacks
+    def upload_attachment(self):
+        if not self._blob_id:
+            defer.returnValue(AttachmentStates.NONE)
+
+        fd = yield self._manager.get_blob(self._blob_id)
+        # TODO: turn following method into a public one
+        yield self._manager._encrypt_and_upload(self._blob_id, fd)
+        defer.returnValue(self.attachment_state())
+
+    @defer.inlineCallbacks
+    def download_attachment(self):
+        if not self._blob_id:
+            defer.returnValue(None)
+        yield self.get_attachment()
+        defer.returnValue(self.attachment_state())
diff --git a/client/src/leap/soledad/client/_secrets/storage.py b/client/src/leap/soledad/client/_secrets/storage.py
index 6ea89900..85713a48 100644
--- a/client/src/leap/soledad/client/_secrets/storage.py
+++ b/client/src/leap/soledad/client/_secrets/storage.py
@@ -23,8 +23,8 @@ from hashlib import sha256
 from leap.soledad.common import SHARED_DB_NAME
 from leap.soledad.common.log import getLogger
 
-from leap.soledad.common.document import SoledadDocument
 from leap.soledad.client.shared_db import SoledadSharedDatabase
+from leap.soledad.client._document import Document
 from leap.soledad.client._secrets.util import emit, UserDataMixin
 
 
@@ -111,7 +111,7 @@ class SecretsStorage(UserDataMixin):
     def save_remote(self, encrypted):
         doc = self._remote_doc
         if not doc:
-            doc = SoledadDocument(doc_id=self._remote_doc_id())
+            doc = Document(doc_id=self._remote_doc_id())
         doc.content = encrypted
         db = self._shared_db
         if not db:
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 18fb35d7..fee9e160 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -58,6 +58,7 @@ from leap.soledad.client import sqlcipher
 from leap.soledad.client._recovery_code import RecoveryCode
 from leap.soledad.client._secrets import Secrets
 from leap.soledad.client._crypto import SoledadCrypto
+from ._blobs import BlobManager
 
 
 logger = getLogger(__name__)
@@ -171,6 +172,7 @@ class Soledad(object):
         self._recovery_code = RecoveryCode()
         self._secrets = Secrets(self)
         self._crypto = SoledadCrypto(self._secrets.remote_secret)
+        self._init_blobmanager()
 
         try:
             # initialize database access, trap any problems so we can shutdown
@@ -262,6 +264,13 @@ class Soledad(object):
                 sync_exchange_phase = _p
         return sync_phase, sync_exchange_phase
 
+    def _init_blobmanager(self):
+        path = os.path.join(os.path.dirname(self._local_db_path), 'blobs')
+        url = urlparse.urljoin(self.server_url, 'blobs/%s' % uuid)
+        key = self._secrets.local_key
+        self.blobmanager = BlobManager(path, url, key, self.uuid, self.token,
+                                       SOLEDAD_CERT)
+
     #
     # Closing methods
     #
@@ -272,6 +281,7 @@ class Soledad(object):
         """
         logger.debug("closing soledad")
         self._dbpool.close()
+        self.blobmanager.close()
         if getattr(self, '_dbsyncer', None):
             self._dbsyncer.close()
 
@@ -306,7 +316,7 @@ class Soledad(object):
         ============================== WARNING ==============================
 
         :param doc: A document with new content.
-        :type doc: leap.soledad.common.document.SoledadDocument
+        :type doc: leap.soledad.common.document.Document
         :return: A deferred whose callback will be invoked with the new
             revision identifier for the document. The document object will
             also be updated.
@@ -323,7 +333,7 @@ class Soledad(object):
         This will also set doc.content to None.
 
         :param doc: A document to be deleted.
-        :type doc: leap.soledad.common.document.SoledadDocument
+        :type doc: leap.soledad.common.document.Document
         :return: A deferred.
         :rtype: twisted.internet.defer.Deferred
         """
@@ -387,6 +397,7 @@ class Soledad(object):
         """
         return self._defer("get_all_docs", include_deleted)
 
+    @defer.inlineCallbacks
     def create_doc(self, content, doc_id=None):
         """
         Create a new document.
@@ -408,8 +419,9 @@ class Soledad(object):
         # create_doc (and probably to put_doc too). There are cases (mail
         # payloads for example) in which we already have the encoding in the
         # headers, so we don't need to guess it.
-        d = self._defer("create_doc", content, doc_id=doc_id)
-        return d
+        doc = yield self._defer("create_doc", content, doc_id=doc_id)
+        doc.set_store(self)
+        defer.returnValue(doc)
 
     def create_doc_from_json(self, json, doc_id=None):
         """
@@ -590,7 +602,7 @@ class Soledad(object):
         the time you GET_DOC_CONFLICTS until the point where you RESOLVE)
 
         :param doc: A Document with the new content to be inserted.
-        :type doc: SoledadDocument
+        :type doc: Document
         :param conflicted_doc_revs: A list of revisions that the new content
             supersedes.
         :type conflicted_doc_revs: list(str)
diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 09e90171..4795846c 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -167,7 +167,7 @@ class SoledadCrypto(object):
         Wrapper around encrypt_docstr that accepts the document as argument.
 
         :param doc: the document.
-        :type doc: SoledadDocument
+        :type doc: Document
         """
         key = self.doc_passphrase(doc.doc_id)
 
@@ -179,7 +179,7 @@ class SoledadCrypto(object):
         Wrapper around decrypt_doc_dict that accepts the document as argument.
 
         :param doc: the document.
-        :type doc: SoledadDocument
+        :type doc: Document
 
         :return: json string with the decrypted document
         :rtype: str
@@ -194,7 +194,7 @@ class SoledadCrypto(object):
 
 
 #
-# Crypto utilities for a SoledadDocument.
+# Crypto utilities for a Document.
 #
 
 def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv,
@@ -439,7 +439,7 @@ def is_symmetrically_encrypted(doc):
     Return True if the document was symmetrically encrypted.
 
     :param doc: The document to check.
-    :type doc: SoledadDocument
+    :type doc: Document
 
     :rtype: bool
     """
diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py
index cf4984d1..c85c5bab 100644
--- a/client/src/leap/soledad/client/http_target/fetch.py
+++ b/client/src/leap/soledad/client/http_target/fetch.py
@@ -23,7 +23,7 @@ from leap.soledad.client.events import emit_async
 from leap.soledad.client.http_target.support import RequestBody
 from leap.soledad.common.log import getLogger
 from leap.soledad.client._crypto import is_symmetrically_encrypted
-from leap.soledad.common.document import SoledadDocument
+from leap.soledad.common.document import Document
 from leap.soledad.common.l2db import errors
 from leap.soledad.client import crypto as old_crypto
 
@@ -113,7 +113,7 @@ class HTTPDocFetcher(object):
 
     @defer.inlineCallbacks
     def __atomic_doc_parse(self, doc_info, content, total):
-        doc = SoledadDocument(doc_info['id'], doc_info['rev'], content)
+        doc = Document(doc_info['id'], doc_info['rev'], content)
         if is_symmetrically_encrypted(content):
             content = (yield self._crypto.decrypt_doc(doc)).getvalue()
         elif old_crypto.is_symmetrically_encrypted(doc):
diff --git a/client/src/leap/soledad/client/interfaces.py b/client/src/leap/soledad/client/interfaces.py
index 1be47df7..0600449f 100644
--- a/client/src/leap/soledad/client/interfaces.py
+++ b/client/src/leap/soledad/client/interfaces.py
@@ -69,7 +69,7 @@ class ILocalStorage(Interface):
         Update a document in the local encrypted database.
 
         :param doc: the document to update
-        :type doc: SoledadDocument
+        :type doc: Document
 
         :return:
             a deferred that will fire with the new revision identifier for
@@ -82,7 +82,7 @@ class ILocalStorage(Interface):
         Delete a document from the local encrypted database.
 
         :param doc: the document to delete
-        :type doc: SoledadDocument
+        :type doc: Document
 
         :return:
             a deferred that will fire with ...
@@ -102,7 +102,7 @@ class ILocalStorage(Interface):
 
         :return:
             A deferred that will fire with the document object, containing a
-            SoledadDocument, or None if it could not be found
+            Document, or None if it could not be found
         :rtype: Deferred
         """
 
@@ -147,7 +147,7 @@ class ILocalStorage(Interface):
         :type doc_id: str
 
         :return:
-            A deferred tht will fire with the new document (SoledadDocument
+            A deferred tht will fire with the new document (Document
             instance).
         :rtype: Deferred
         """
@@ -167,7 +167,7 @@ class ILocalStorage(Interface):
         :param doc_id: An optional identifier specifying the document id.
         :type doc_id:
         :return:
-            A deferred that will fire with the new document (A SoledadDocument
+            A deferred that will fire with the new document (A Document
             instance)
         :rtype: Deferred
         """
@@ -304,7 +304,7 @@ class ILocalStorage(Interface):
         Mark a document as no longer conflicted.
 
         :param doc: a document with the new content to be inserted.
-        :type doc: SoledadDocument
+        :type doc: Document
         :param conflicted_doc_revs:
             A deferred that will fire with a list of revisions that the new
             content supersedes.
diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py
index 2c995d5a..14fecd3b 100644
--- a/client/src/leap/soledad/client/sqlcipher.py
+++ b/client/src/leap/soledad/client/sqlcipher.py
@@ -50,16 +50,15 @@ from twisted.internet import reactor
 from twisted.internet import defer
 from twisted.enterprise import adbapi
 
-from leap.soledad.common.document import SoledadDocument
 from leap.soledad.common.log import getLogger
 from leap.soledad.common.l2db import errors as u1db_errors
-from leap.soledad.common.l2db import Document
 from leap.soledad.common.l2db.backends import sqlite_backend
 from leap.soledad.common.errors import DatabaseAccessError
 
 from leap.soledad.client.http_target import SoledadHTTPSyncTarget
 from leap.soledad.client.sync import SoledadSynchronizer
 from leap.soledad.client import pragmas
+from leap.soledad.client._document import Document
 
 if sys.version_info[0] < 3:
     from pysqlcipher import dbapi2 as sqlcipher_dbapi2
@@ -232,7 +231,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
         # ---------------------------------------------------------
 
         self._ensure_schema()
-        self.set_document_factory(soledad_doc_factory)
+        self.set_document_factory(doc_factory)
         self._prime_replica_uid()
 
     def _prime_replica_uid(self):
@@ -437,7 +436,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase):
                 self._opts, check_same_thread=False)
             self._real_replica_uid = None
             self._ensure_schema()
-            self.set_document_factory(soledad_doc_factory)
+            self.set_document_factory(doc_factory)
         except sqlcipher_dbapi2.DatabaseError as e:
             raise DatabaseAccessError(str(e))
 
@@ -537,7 +536,7 @@ class SoledadSQLCipherWrapper(SQLCipherDatabase):
         self._db_handle = conn
         self._real_replica_uid = None
         self._ensure_schema()
-        self.set_document_factory(soledad_doc_factory)
+        self.set_document_factory(doc_factory)
         self._prime_replica_uid()
 
 
@@ -583,14 +582,14 @@ class DatabaseIsNotEncrypted(Exception):
     pass
 
 
-def soledad_doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False,
-                        syncable=True):
+def doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False,
+                syncable=True):
     """
     Return a default Soledad Document.
     Used in the initialization for SQLCipherDatabase
     """
-    return SoledadDocument(doc_id=doc_id, rev=rev, json=json,
-                           has_conflicts=has_conflicts, syncable=syncable)
+    return Document(doc_id=doc_id, rev=rev, json=json,
+                    has_conflicts=has_conflicts, syncable=syncable)
 
 
 sqlite_backend.SQLiteDatabase.register_implementation(SQLCipherDatabase)