diff options
| -rw-r--r-- | docs/api.rst | 3 | ||||
| -rw-r--r-- | docs/api/attachments.rst | 4 | ||||
| -rw-r--r-- | docs/api/blobs-client.rst | 13 | ||||
| -rw-r--r-- | docs/api/blobs-server.rst | 63 | ||||
| -rw-r--r-- | docs/api/blobs.rst | 11 | ||||
| -rw-r--r-- | docs/api/blobs/blobmanager.rst | 15 | ||||
| -rw-r--r-- | docs/api/blobs/errors.rst | 6 | ||||
| -rw-r--r-- | docs/api/blobs/sync.rst | 12 | ||||
| -rw-r--r-- | docs/reference/attachments.rst | 2 | ||||
| -rw-r--r-- | docs/reference/blobs/client.rst | 49 | ||||
| -rw-r--r-- | docs/reference/blobs/server.rst | 63 | ||||
| -rw-r--r-- | docs/reference/blobs/sync.rst | 18 | ||||
| -rw-r--r-- | src/leap/soledad/client/_db/blobs/errors.py | 24 |
13 files changed, 188 insertions, 95 deletions
diff --git a/docs/api.rst b/docs/api.rst index 362c632c..a184c9f7 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -5,5 +5,6 @@ API Reference :maxdepth: 1 api/client - api/blobs api/attachments + api/blobs-client + api/blobs-server diff --git a/docs/api/attachments.rst b/docs/api/attachments.rst index 80b81d67..434a7526 100644 --- a/docs/api/attachments.rst +++ b/docs/api/attachments.rst @@ -1,5 +1,5 @@ -Client-side Attachments API -=========================== +Document Attachments API +======================== Soledad Documents implement the ``IDocumentWithAttachment`` API that associates :ref:`blobs` with documents. diff --git a/docs/api/blobs-client.rst b/docs/api/blobs-client.rst new file mode 100644 index 00000000..b5811cc6 --- /dev/null +++ b/docs/api/blobs-client.rst @@ -0,0 +1,13 @@ +Blobs Client-side Python API +============================ + +Each Soledad Client has a property called ``blobmanager`` which is an instance +of the ``BlobManager`` class and handles Blobs creation, retrieval, deletion +and synchronization. + +.. toctree:: + :maxdepth: 1 + + blobs/blobmanager + blobs/sync + blobs/errors diff --git a/docs/api/blobs-server.rst b/docs/api/blobs-server.rst new file mode 100644 index 00000000..3efe86bd --- /dev/null +++ b/docs/api/blobs-server.rst @@ -0,0 +1,63 @@ +Blobs Server-side HTTP API +========================== + +Soledad Server provides two different REST APIs for interacting with blobs: + +* A *public* :ref:`blobs-http-api`, providing the *Blobs* service for Soledad + Client (i.e. actual users of the infrastructure). + +* A *local* :ref:`incoming-http-api`, providing the delivery part of the + :ref:`incoming-box` service, currently used for the MX mail delivery. + +Authentication is handled differently for each of the endpoints, see +:ref:`authentication` for more details. + +.. _blobs-http-api: + +Public Blobs HTTP API +--------------------- + +The *public endpoint* provides the following REST API for interacting with the +*Blobs* service: + +=========================== ========== ================================= ============================================ +path method action accepted query string fields +=========================== ========== ================================= ============================================ +``/blobs/{uuid}`` ``GET`` Get a list of blobs. filtered by ``namespace``, ``filter_flag``, ``order_by`` + a flag. +``/blobs/{uuid}/{blob_id}`` ``GET`` Get the contents of a blob. ``namespace`` +``/blobs/{uuid}/{blob_id}`` ``PUT`` Create a blob. The content of the ``namespace`` + blob should be sent in the body + of the request. +``/blobs/{uuid}/{blob_id}`` ``POST`` Set the flags for a blob. A list ``namespace`` + of flags should be sent in the + body of the request. +``/blobs/{uuid}/{blob_id}`` ``DELETE`` Delete a blob. ``namespace`` +=========================== ========== ================================= ============================================ + +The Blobs service supports *namespaces*. All requests can be modified by the +``namespace`` query string parameter, and the results will be restricted to +a certain namespace. When no namespace explicitelly given, the ``default`` +namespace is used. + +When listing blobs, the results can be filtered by flag and/or ordered by date +using the ``filter_flag`` and ``order_by`` query string parameters. The +possible values for ``order_by`` are ``date`` or ``+date`` for increasing +order, or ``-date`` for decreasing order. + +.. _incoming-http-api: + +Local Incoming Box HTTP API +--------------------------- + +The *local endpoint* provides the following REST API for interacting with the +:ref:`incoming-box` service. + +============================== ========== ================================= +path method action +============================== ========== ================================= +``/incoming/{uuid}/{blob_id}`` ``PUT`` Create an incoming blob. The content of the blob should be sent in the body of the request. +============================== ========== ================================= + +All blobs created using this API are inserted under the namespace ``MX`` and +flagged as ``PENDING``. diff --git a/docs/api/blobs.rst b/docs/api/blobs.rst deleted file mode 100644 index fbf31e1c..00000000 --- a/docs/api/blobs.rst +++ /dev/null @@ -1,11 +0,0 @@ -Client-side Blobs API -===================== - -The Soledad Client object has a ``blobmanager`` property which is responsible -for handling blobs. - -.. autoclass:: leap.soledad.client._db.blobs.BlobManager - :members: - :special-members: __init__ - :undoc-members: - :show-inheritance: diff --git a/docs/api/blobs/blobmanager.rst b/docs/api/blobs/blobmanager.rst new file mode 100644 index 00000000..04165a57 --- /dev/null +++ b/docs/api/blobs/blobmanager.rst @@ -0,0 +1,15 @@ +.. _blobmanager-api: + +Blobs creation, retrieval, deletion and flagging +================================================ + +The ``BlobManager`` class is responsible for blobs creation, retrieval, +deletion, flagging and synchronizing. For better code organization, the methods +related to synchronization are implemented separatelly in a superclass (see +:ref:`blobs-sync-api`). + +.. autoclass:: leap.soledad.client._db.blobs.BlobManager + :members: + :special-members: __init__ + :undoc-members: + :show-inheritance: diff --git a/docs/api/blobs/errors.rst b/docs/api/blobs/errors.rst new file mode 100644 index 00000000..66454631 --- /dev/null +++ b/docs/api/blobs/errors.rst @@ -0,0 +1,6 @@ +Blobs Errors +============ + +.. automodule:: leap.soledad.client._db.blobs.errors + :members: + :undoc-members: diff --git a/docs/api/blobs/sync.rst b/docs/api/blobs/sync.rst new file mode 100644 index 00000000..53740d64 --- /dev/null +++ b/docs/api/blobs/sync.rst @@ -0,0 +1,12 @@ +.. _blobs-sync-api: + +Blobs Synchronization +===================== + +The synchronization part of the ``BlobManager`` class is implemented in the +``BlobsSynchronizer`` class, whose API can be seen below. + +.. autoclass:: leap.soledad.client._db.blobs.sync.BlobsSynchronizer + :members: + :special-members: __init__ + :undoc-members: diff --git a/docs/reference/attachments.rst b/docs/reference/attachments.rst index 29387b36..4f3740f1 100644 --- a/docs/reference/attachments.rst +++ b/docs/reference/attachments.rst @@ -26,7 +26,7 @@ Server-side ----------- In the server, attachments are stored as :ref:`blobs`. See -:ref:`http-blobs-api` for more information on how to interact with the server +:ref:`blobs-http-api` for more information on how to interact with the server using HTTP. The :ref:`IBlobsBackend <i-blobs-backend>` interface is provided, so in the diff --git a/docs/reference/blobs/client.rst b/docs/reference/blobs/client.rst index 75c3d997..734866fe 100644 --- a/docs/reference/blobs/client.rst +++ b/docs/reference/blobs/client.rst @@ -1,20 +1,47 @@ Client-side blobs ================= -On the client-side, blobs can be managed using the BlobManager API. The -BlobManager is responsible for managing storage of blobs both in local and -remote storages +Data storage +------------ + +On the client-side, blobs can be managed using the ``BlobManager`` API, which +is responsible for managing storage of blobs both in local and remote storages. +See :ref:`blobmanager-api` and :ref:`blobs-sync-api` for information on the +client-side API. All data is stored locally in the ``blobs`` table of a SQLCipher database called ``{uuid}_blobs.db`` that lies in the same directory as the Soledad -Client's JSON documents database. Both databases are encrypted with the same -symmetric secret. All actions performed locally are mirrored remotelly using -the :ref:`http-blobs-api`. - -The BlobManager supports *namespaces* and *flags* and can list local and remote -blobs, possibly filtering by flags and ordering by date (increasingly or -decreasingly). It has helper methods to send or fetch all missing blobs, thus -aiding in synchronization of local and remote data. +Client's JSON documents database (see :ref:`client-databases`). All actions +performed locally are mirrored remotelly using the :ref:`blobs-http-api`. + +Namespaces +---------- + +The Blobs API supports **namespaces** so that applications can store and fetch +blobs without interfering in each another. Namespaces are also used to +implement the server-side :ref:`incoming-http-api`, used for mail delivery. All +methods that deal with blobs storage, transfer and flagging provide +a `namespace` parameter. If no namespace is given, the value `default` is used. + +Remote flags +------------ + +In order to allow clients to control the processing of blobs that are delivered +by external applications, the Blobs API has the concept of **remote flags**. +The client can get and set the following flags for Blobs that reside in the +server: ``PENDING``, ``PROCESSING``, ``PROCESSED``, and ``FAILED``. + +Remote listing +-------------- + +The client can obtain a list of blobs in the server side so it can compare with +its own local list and queue up blobs for download and upload. The remote +listing can be ordered by *upload date* and filtered by *namespace* and *flag*. +The listing can also only return the number of matches instead of the whole +content. + +Client-side encryption +---------------------- When uploading, the content of the blob is encrypted with a symmetric secret prior to being sent to the server. When downloading, the content of the blob is diff --git a/docs/reference/blobs/server.rst b/docs/reference/blobs/server.rst index 532fccd7..cda0b95c 100644 --- a/docs/reference/blobs/server.rst +++ b/docs/reference/blobs/server.rst @@ -1,68 +1,23 @@ Server-side blobs ================= +The server-side implementation of blobs provides HTTP APIs for data storage +using a filesystem backend. + +HTTP APIs +--------- + Soledad Server provides two different REST APIs for interacting with blobs: -* A public **HTTP Blobs API**, providing the *Blobs* service for Soledad Client - (i.e. actual users of the infrastructure). +* A *public* :ref:`blobs-http-api`, providing the *Blobs* service for Soledad + Client (i.e. actual users of the infrastructure). -* A local **HTTP Incoming Box API**, providing the delivery part of the +* A *local* :ref:`incoming-http-api`, providing the delivery part of the :ref:`incoming-box` service, currently used for the MX mail delivery. Authentication is handled differently for each of the endpoints, see :ref:`authentication` for more details. - -.. _http-blobs-api: - -HTTP Blobs API --------------- - -The *public endpoint* provides the following REST API for interacting with the -*Blobs* service: - -=========================== ========== ================================= ============================================ -path method action accepted query string fields -=========================== ========== ================================= ============================================ -``/blobs/{uuid}`` ``GET`` Get a list of blobs. filtered by ``namespace``, ``filter_flag``, ``order_by`` - a flag. -``/blobs/{uuid}/{blob_id}`` ``GET`` Get the contents of a blob. ``namespace`` -``/blobs/{uuid}/{blob_id}`` ``PUT`` Create a blob. The content of the ``namespace`` - blob should be sent in the body - of the request. -``/blobs/{uuid}/{blob_id}`` ``POST`` Set the flags for a blob. A list ``namespace`` - of flags should be sent in the - body of the request. -``/blobs/{uuid}/{blob_id}`` ``DELETE`` Delete a blob. ``namespace`` -=========================== ========== ================================= ============================================ - -The Blobs service supports *namespaces*. All requests can be modified by the -``namespace`` query string parameter, and the results will be restricted to -a certain namespace. When no namespace explicitelly given, the ``default`` -namespace is used. - -When listing blobs, the results can be filtered by flag and/or ordered by date -using the ``filter_flag`` and ``order_by`` query string parameters. The -possible values for ``order_by`` are ``date`` or ``+date`` for increasing -order, or ``-date`` for decreasing order. - - -HTTP Incoming Box API ---------------------- - -The *local endpoint* provides the following REST API for interacting with the -:ref:`incoming-box` service. - -============================== ========== ================================= -path method action -============================== ========== ================================= -``/incoming/{uuid}/{blob_id}`` ``PUT`` Create an incoming blob. The content of the blob should be sent in the body of the request. -============================== ========== ================================= - -All blobs created using this API are inserted under the namespace ``MX`` and -flagged as ``PENDING``. - - .. _filesystem-backend: Filesystem backend diff --git a/docs/reference/blobs/sync.rst b/docs/reference/blobs/sync.rst index 729bb0ab..bdfad0cf 100644 --- a/docs/reference/blobs/sync.rst +++ b/docs/reference/blobs/sync.rst @@ -19,10 +19,10 @@ Synchronization status In the client-side, each has an associated synchronization status, which can be one of: -- `SYNCED`: The blob exists both in this client and in the server. -- `PENDING_UPLOAD`: The blob was inserted locally, but has not yet been uploaded. -- `PENDING_DOWNLOAD`: The blob exists in the server, but has not yet been downloaded. -- `FAILED_DOWNLOAD`: A download attempt has been made but the content is corrupted for some reason. +- ``SYNCED``: The blob exists both in this client and in the server. +- ``PENDING_UPLOAD``: The blob was inserted locally, but has not yet been uploaded. +- ``PENDING_DOWNLOAD``: The blob exists in the server, but has not yet been downloaded. +- ``FAILED_DOWNLOAD``: A download attempt has been made but the content is corrupted for some reason. Concurrency limits ------------------ @@ -41,8 +41,8 @@ the transfer has been successful or until there has been an unrecoverable transfer error. Currently, the only unrecoverable transfer error is a failed verification of the blob tag (i.e. a failed MAC verification). -Successive transfer attempts have an increasing delay between them, to minimize -competition for resources used by other concurrent transfer attempts. The delay -starts with 10 seconds and increases to 20, 30, 40, 50, and finally 60 seconds -on each new failed attempt. The delay for a new retry then stays at 60 seconds -for new attempts. +Successive failed transfer attempts of the same blob are separated by an +increasing time interval to minimize competition for resources used by other +concurrent transfer attempts. The interaval starts at 10 seconds and increases +to 20, 30, 40, 50, and finally 60 seconds. All further retries will be +separated by a 60 seconds time interval. diff --git a/src/leap/soledad/client/_db/blobs/errors.py b/src/leap/soledad/client/_db/blobs/errors.py index 90de9553..c6e70de7 100644 --- a/src/leap/soledad/client/_db/blobs/errors.py +++ b/src/leap/soledad/client/_db/blobs/errors.py @@ -15,26 +15,38 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ -Blobs exceptions +This module contains the different errors that can happen when dealing with +blobs. """ from leap.soledad.common.errors import SoledadError class BlobAlreadyExistsError(SoledadError): - pass + """ + Raised on attempts to put local or remote blobs that already exist in + storage. + """ class BlobNotFoundError(SoledadError): - pass + """ + Raised on attemtps to get remote blobs that do not exist in storage. + """ class InvalidFlagsError(SoledadError): - pass + """ + Raised on attempts to set invalid flags for remotelly stored blobs. + """ class RetriableTransferError(Exception): - pass + """ + Raised for any blob transfer error that is considered retriable. + """ class MaximumRetriesError(Exception): - pass + """ + Raised when the maximum number of transfer retries has been reached. + """ |