summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordrebs <drebs@leap.se>2017-02-23 12:28:59 -0300
committerdrebs <drebs@leap.se>2017-04-04 18:27:32 +0200
commit943fbba160f49971e33d7696f2999f97734eb291 (patch)
treefb285d9528adb7f5859a7c874317f9b1384dbf51
parent59d3136226e8f87a906f92e315f00d7705292c89 (diff)
[bug] fix blobs put
-rw-r--r--client/src/leap/soledad/client/_blobs.py108
-rw-r--r--server/src/leap/soledad/server/_blobs.py2
2 files changed, 59 insertions, 51 deletions
diff --git a/client/src/leap/soledad/client/_blobs.py b/client/src/leap/soledad/client/_blobs.py
index 3de67569..336fe9af 100644
--- a/client/src/leap/soledad/client/_blobs.py
+++ b/client/src/leap/soledad/client/_blobs.py
@@ -19,7 +19,10 @@ Clientside BlobBackend Storage.
"""
from copy import copy
+from urlparse import urljoin
+
import os.path
+import uuid
from io import BytesIO
from functools import partial
@@ -143,7 +146,7 @@ class BlobManager(object):
# concurrently. not sure if we'd gain something.
yield self.local.put(doc.blob_id, fd)
fd.seek(0)
- yield self._encrypt_and_upload(doc.blob_id, fd, up)
+ yield self._encrypt_and_upload(doc.blob_id, doc.doc_id, doc.rev, fd)
@defer.inlineCallbacks
def get(self, blob_id, doc_id, rev):
@@ -152,7 +155,11 @@ class BlobManager(object):
logger.info("Found blob in local database: %s" % blob_id)
defer.returnValue(local_blob)
- blob, size = yield self._download_and_decrypt(blob_id, doc_id, rev)
+ result = yield self._download_and_decrypt(blob_id, doc_id, rev)
+
+ if not result:
+ defer.returnValue(None)
+ blob, size = result
if blob:
logger.info("Got decrypted blob of type: %s" % type(blob))
@@ -169,7 +176,7 @@ class BlobManager(object):
logger.error('sorry, dunno what happened')
@defer.inlineCallbacks
- def _encrypt_and_upload(self, blob_id, doc_id, rev, payload):
+ def _encrypt_and_upload(self, blob_id, doc_id, rev, fd):
# TODO ------------------------------------------
# this is wrong, is doing 2 stages.
# the crypto producer can be passed to
@@ -177,12 +184,14 @@ class BlobManager(object):
# try to rewrite as a tube: pass the fd to aes and let aes writer
# produce data to the treq request fd.
# ------------------------------------------------
+ logger.info("Staring upload of blob: %s" % blob_id)
doc_info = DocInfo(doc_id, rev)
- uri = self.remote + '/' + self.user + '/' + blob_id
- crypter = BlobEncryptor(doc_info, payload, secret=self.secret,
+ uri = urljoin(self.remote, self.user + "/" + blob_id)
+ crypter = BlobEncryptor(doc_info, fd, secret=self.secret,
armor=True)
- result = yield crypter.encrypt()
- yield treq.put(uri, data=result)
+ fd = yield crypter.encrypt()
+ yield treq.put(uri, data=fd)
+ logger.info("Finished upload: %s" % (blob_id,))
@defer.inlineCallbacks
def _download_and_decrypt(self, blob_id, doc_id, rev):
@@ -262,21 +271,19 @@ def _sqlcipherInitFactory(fun):
return _initialize
-# --------------------8<----------------------------------------------
-# class BlobDoc(object):
-#
-# # TODO probably not needed, but convenient for testing for now.
-#
-# def __init__(self, doc_id, rev, content, blob_id=None):
-#
-# self.doc_id = doc_id
-# self.rev = rev
-# self.is_blob = True
-# self.blob_fd = content
-# if blob_id is None:
-# blob_id = uuid4().get_hex()
-# self.blob_id = blob_id
-# --------------------8<----------------------------------------------
+class BlobDoc(object):
+
+ # TODO probably not needed, but convenient for testing for now.
+
+ def __init__(self, doc_id, rev, content, blob_id=None):
+
+ self.doc_id = doc_id
+ self.rev = rev
+ self.is_blob = True
+ self.blob_fd = content
+ if blob_id is None:
+ blob_id = uuid.uuid4().get_hex()
+ self.blob_id = blob_id
#
@@ -293,7 +300,7 @@ def testit(reactor):
# parse command line arguments
import argparse
- usage = "\n cd server/src/leap/soledad/server/ && python _blobs.py" \
+ usage = "\n mkdir /tmp/blobs/user && cd server/src/leap/soledad/server/ && python _blobs.py" \
"\n python _blobs.py upload /path/to/file blob_id" \
"\n python _blobs.py download blob_id"
@@ -327,47 +334,49 @@ def testit(reactor):
# TODO convert these into proper unittests
+ def _manager():
+ manager = BlobManager(
+ '/tmp/blobs', 'http://localhost:9000/',
+ 'A' * 32, 'secret', 'user')
+ return manager
+
@defer.inlineCallbacks
def _upload(blob_id, payload):
- logger.info(":: Starting upload only...")
- doc_info = DocInfo('mydoc', '1')
- logger.info(str(doc_info))
- # use BlobEncryptor intead of BlobManager to only upload and not put
- # blob on local db
- crypter = BlobEncryptor(
- doc_info, open(payload, 'r'), 'A' * 32, armor=True)
- logger.info(":: Uploading with encryptor")
- result = yield crypter.encrypt()
- yield treq.put('http://localhost:9000/user/' + blob_id, data=result)
- logger.info(":: Finished upload only.")
+ logger.info(":: Starting upload only: %s" % str((blob_id, payload)))
+ manager = _manager()
+ with open(payload, 'r') as fd:
+ yield manager._encrypt_and_upload(blob_id, 'mydoc', '1', fd)
+ logger.info(":: Finished upload only: %s" % str((blob_id, payload)))
@defer.inlineCallbacks
def _download(blob_id):
logger.info(":: Starting download only: %s" % blob_id)
- manager = BlobManager(
- '/tmp/blobs', 'http://localhost:9000/',
- 'A' * 32, 'secret', 'user')
+ manager = _manager()
result = yield manager._download_and_decrypt(blob_id, 'mydoc', '1')
logger.info(":: Result of download: %s" % str(result))
- if not result:
- logger.info(":: Download failed for: %s" % blob_id)
- else:
+ if result:
fd, _ = result
logger.info(":: Content of blob %s: %s" % (blob_id, fd.getvalue()))
logger.info(":: Finished download only: %s" % blob_id)
+ @defer.inlineCallbacks
def _put(blob_id, payload):
- pass
+ logger.info(":: Starting full put: %s" % blob_id)
+ manager = _manager()
+ with open(payload) as fd:
+ doc = BlobDoc('mydoc', '1', fd, blob_id=blob_id)
+ result = yield manager.put(doc)
+ logger.info(":: Result of put: %s" % str(result))
+ logger.info(":: Finished full put: %s" % blob_id)
@defer.inlineCallbacks
def _get(blob_id):
- logger.info(":: Starting full get...")
- manager = BlobManager(
- '/tmp/blobs', 'http://localhost:9000/',
- 'A' * 32, 'secret', 'user')
- payload = yield manager.get(blob_id, 'mydoc', '1')
- logger.info(":: Finished full get.")
- defer.returnValue(payload)
+ logger.info(":: Starting full get: %s" % blob_id)
+ manager = _manager()
+ fd = yield manager.get(blob_id, 'mydoc', '1')
+ if fd:
+ logger.info(":: Result of get: " + fd.getvalue())
+ logger.info(":: Finished full get: %s" % blob_id)
if args.action == 'upload':
yield _upload(args.blob_id, args.payload)
@@ -376,8 +385,7 @@ def testit(reactor):
elif args.action == 'put':
yield _put(args.blob_id, args.payload)
elif args.action == 'get':
- result = yield _get(args.blob_id)
- logger.info(":: Result of get: " + result.getvalue())
+ yield _get(args.blob_id)
if __name__ == '__main__':
diff --git a/server/src/leap/soledad/server/_blobs.py b/server/src/leap/soledad/server/_blobs.py
index 847ee755..b0d065d9 100644
--- a/server/src/leap/soledad/server/_blobs.py
+++ b/server/src/leap/soledad/server/_blobs.py
@@ -188,7 +188,7 @@ _path = _config['blobs_path']
blobs_resource = BlobsResource(_path)
if __name__ == '__main__':
# A dummy blob server
- # curl -X PUT --data-binary @/tmp/book.pdf localhost:9000/user/somerandomstring
+ # curl -X PUT --data-binary @/tmp/book.pdf localhost:9000/user/someid
# curl -X GET -o /dev/null localhost:9000/user/somerandomstring
from twisted.web.server import Site