summaryrefslogtreecommitdiff
path: root/src/leap/soledad/backends
diff options
context:
space:
mode:
authordrebs <drebs@leap.se>2013-06-16 21:45:16 -0300
committerdrebs <drebs@leap.se>2013-06-20 15:19:50 -0300
commitdbe5e37ef742617c93c7975a612582a77c7724a8 (patch)
tree30efb7b027cda6992753df27833b6bb6f2d091b9 /src/leap/soledad/backends
parent0deaa2070a88404779ae9a8dab88ee11521c5032 (diff)
Split client and server in two different packages and refactor.
Diffstat (limited to 'src/leap/soledad/backends')
-rw-r--r--src/leap/soledad/backends/__init__.py21
-rw-r--r--src/leap/soledad/backends/couch.py484
-rw-r--r--src/leap/soledad/backends/leap_backend.py544
-rw-r--r--src/leap/soledad/backends/objectstore.py296
-rw-r--r--src/leap/soledad/backends/sqlcipher.py653
5 files changed, 0 insertions, 1998 deletions
diff --git a/src/leap/soledad/backends/__init__.py b/src/leap/soledad/backends/__init__.py
deleted file mode 100644
index 737c08e6..00000000
--- a/src/leap/soledad/backends/__init__.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# -*- coding: utf-8 -*-
-# __init__.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-Backends that extend U1DB functionality.
-"""
diff --git a/src/leap/soledad/backends/couch.py b/src/leap/soledad/backends/couch.py
deleted file mode 100644
index 57885012..00000000
--- a/src/leap/soledad/backends/couch.py
+++ /dev/null
@@ -1,484 +0,0 @@
-# -*- coding: utf-8 -*-
-# couch.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""A U1DB backend that uses CouchDB as its persistence layer."""
-
-# general imports
-import uuid
-import re
-import simplejson as json
-
-
-from base64 import b64encode, b64decode
-from u1db import errors
-from u1db.sync import Synchronizer
-from u1db.backends.inmemory import InMemoryIndex
-from u1db.remote.server_state import ServerState
-from u1db.errors import DatabaseDoesNotExist
-from couchdb.client import Server, Document as CouchDocument
-from couchdb.http import ResourceNotFound
-
-
-from leap.soledad.backends.objectstore import (
- ObjectStoreDatabase,
- ObjectStoreSyncTarget,
-)
-from leap.soledad.backends.leap_backend import LeapDocument
-
-
-class InvalidURLError(Exception):
- """
- Exception raised when Soledad encounters a malformed URL.
- """
-
-
-class CouchDatabase(ObjectStoreDatabase):
- """
- A U1DB backend that uses Couch as its persistence layer.
- """
-
- U1DB_TRANSACTION_LOG_KEY = 'transaction_log'
- U1DB_CONFLICTS_KEY = 'conflicts'
- U1DB_OTHER_GENERATIONS_KEY = 'other_generations'
- U1DB_INDEXES_KEY = 'indexes'
- U1DB_REPLICA_UID_KEY = 'replica_uid'
-
- COUCH_ID_KEY = '_id'
- COUCH_REV_KEY = '_rev'
- COUCH_U1DB_ATTACHMENT_KEY = 'u1db_json'
- COUCH_U1DB_REV_KEY = 'u1db_rev'
-
- @classmethod
- def open_database(cls, url, create):
- """
- Open a U1DB database using CouchDB as backend.
-
- @param url: the url of the database replica
- @type url: str
- @param create: should the replica be created if it does not exist?
- @type create: bool
-
- @return: the database instance
- @rtype: CouchDatabase
- """
- # get database from url
- m = re.match('(^https?://[^/]+)/(.+)$', url)
- if not m:
- raise InvalidURLError
- url = m.group(1)
- dbname = m.group(2)
- server = Server(url=url)
- try:
- server[dbname]
- except ResourceNotFound:
- if not create:
- raise DatabaseDoesNotExist()
- return cls(url, dbname)
-
- def __init__(self, url, dbname, replica_uid=None, full_commit=True,
- session=None):
- """
- Create a new Couch data container.
-
- @param url: the url of the couch database
- @type url: str
- @param dbname: the database name
- @type dbname: str
- @param replica_uid: an optional unique replica identifier
- @type replica_uid: str
- @param full_commit: turn on the X-Couch-Full-Commit header
- @type full_commit: bool
- @param session: an http.Session instance or None for a default session
- @type session: http.Session
- """
- self._url = url
- self._full_commit = full_commit
- self._session = session
- self._server = Server(url=self._url,
- full_commit=self._full_commit,
- session=self._session)
- self._dbname = dbname
- # this will ensure that transaction and sync logs exist and are
- # up-to-date.
- try:
- self._database = self._server[self._dbname]
- except ResourceNotFound:
- self._server.create(self._dbname)
- self._database = self._server[self._dbname]
- ObjectStoreDatabase.__init__(self, replica_uid=replica_uid,
- # TODO: move the factory choice
- # away
- document_factory=LeapDocument)
-
- #-------------------------------------------------------------------------
- # methods from Database
- #-------------------------------------------------------------------------
-
- def _get_doc(self, doc_id, check_for_conflicts=False):
- """
- Get just the document content, without fancy handling.
-
- @param doc_id: The unique document identifier
- @type doc_id: str
- @param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise asking for a deleted
- document will return None.
- @type include_deleted: bool
-
- @return: a Document object.
- @type: u1db.Document
- """
- cdoc = self._database.get(doc_id)
- if cdoc is None:
- return None
- has_conflicts = False
- if check_for_conflicts:
- has_conflicts = self._has_conflicts(doc_id)
- doc = self._factory(
- doc_id=doc_id,
- rev=cdoc[self.COUCH_U1DB_REV_KEY],
- has_conflicts=has_conflicts)
- contents = self._database.get_attachment(
- cdoc,
- self.COUCH_U1DB_ATTACHMENT_KEY)
- if contents:
- doc.content = json.loads(contents.read())
- else:
- doc.make_tombstone()
- return doc
-
- def get_all_docs(self, include_deleted=False):
- """
- Get the JSON content for all documents in the database.
-
- @param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise deleted documents will not
- be included in the results.
- @type include_deleted: bool
-
- @return: (generation, [Document])
- The current generation of the database, followed by a list of all
- the documents in the database.
- @rtype: tuple
- """
- generation = self._get_generation()
- results = []
- for doc_id in self._database:
- if doc_id == self.U1DB_DATA_DOC_ID:
- continue
- doc = self._get_doc(doc_id, check_for_conflicts=True)
- if doc.content is None and not include_deleted:
- continue
- results.append(doc)
- return (generation, results)
-
- def _put_doc(self, doc):
- """
- Update a document.
-
- This is called everytime we just want to do a raw put on the db (i.e.
- without index updates, document constraint checks, and conflict
- checks).
-
- @param doc: The document to update.
- @type doc: u1db.Document
-
- @return: The new revision identifier for the document.
- @rtype: str
- """
- # prepare couch's Document
- cdoc = CouchDocument()
- cdoc[self.COUCH_ID_KEY] = doc.doc_id
- # we have to guarantee that couch's _rev is consistent
- old_cdoc = self._database.get(doc.doc_id)
- if old_cdoc is not None:
- cdoc[self.COUCH_REV_KEY] = old_cdoc[self.COUCH_REV_KEY]
- # store u1db's rev
- cdoc[self.COUCH_U1DB_REV_KEY] = doc.rev
- # save doc in db
- self._database.save(cdoc)
- # store u1db's content as json string
- if not doc.is_tombstone():
- self._database.put_attachment(
- cdoc, doc.get_json(),
- filename=self.COUCH_U1DB_ATTACHMENT_KEY)
- else:
- self._database.delete_attachment(
- cdoc,
- self.COUCH_U1DB_ATTACHMENT_KEY)
-
- def get_sync_target(self):
- """
- Return a SyncTarget object, for another u1db to synchronize with.
-
- @return: The sync target.
- @rtype: CouchSyncTarget
- """
- return CouchSyncTarget(self)
-
- def create_index(self, index_name, *index_expressions):
- """
- Create a named index, which can then be queried for future lookups.
-
- @param index_name: A unique name which can be used as a key prefix.
- @param index_expressions: Index expressions defining the index
- information.
- """
- if index_name in self._indexes:
- if self._indexes[index_name]._definition == list(
- index_expressions):
- return
- raise errors.IndexNameTakenError
- index = InMemoryIndex(index_name, list(index_expressions))
- for doc_id in self._database:
- if doc_id == self.U1DB_DATA_DOC_ID: # skip special file
- continue
- doc = self._get_doc(doc_id)
- if doc.content is not None:
- index.add_json(doc_id, doc.get_json())
- self._indexes[index_name] = index
- # save data in object store
- self._store_u1db_data()
-
- def close(self):
- """
- Release any resources associated with this database.
-
- @return: True if db was succesfully closed.
- @rtype: bool
- """
- # TODO: fix this method so the connection is properly closed and
- # test_close (+tearDown, which deletes the db) works without problems.
- self._url = None
- self._full_commit = None
- self._session = None
- #self._server = None
- self._database = None
- return True
-
- def sync(self, url, creds=None, autocreate=True):
- """
- Synchronize documents with remote replica exposed at url.
-
- @param url: The url of the target replica to sync with.
- @type url: str
- @param creds: optional dictionary giving credentials.
- to authorize the operation with the server.
- @type creds: dict
- @param autocreate: Ask the target to create the db if non-existent.
- @type autocreate: bool
-
- @return: The local generation before the synchronisation was performed.
- @rtype: int
- """
- return Synchronizer(self, CouchSyncTarget(url, creds=creds)).sync(
- autocreate=autocreate)
-
- #-------------------------------------------------------------------------
- # methods from ObjectStoreDatabase
- #-------------------------------------------------------------------------
-
- def _init_u1db_data(self):
- """
- Initialize U1DB info data structure in the couch db.
-
- A U1DB database needs to keep track of all database transactions,
- document conflicts, the generation of other replicas it has seen,
- indexes created by users and so on.
-
- In this implementation, all this information is stored in a special
- document stored in the couch db with id equals to
- CouchDatabse.U1DB_DATA_DOC_ID.
-
- This method initializes the document that will hold such information.
- """
- if self._replica_uid is None:
- self._replica_uid = uuid.uuid4().hex
- # TODO: prevent user from overwriting a document with the same doc_id
- # as this one.
- doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
- doc.content = {
- self.U1DB_TRANSACTION_LOG_KEY: b64encode(json.dumps([])),
- self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})),
- self.U1DB_OTHER_GENERATIONS_KEY: b64encode(json.dumps({})),
- self.U1DB_INDEXES_KEY: b64encode(json.dumps({})),
- self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
- }
- self._put_doc(doc)
-
- def _fetch_u1db_data(self):
- """
- Fetch U1DB info from the couch db.
-
- See C{_init_u1db_data} documentation.
- """
- # retrieve u1db data from couch db
- cdoc = self._database.get(self.U1DB_DATA_DOC_ID)
- jsonstr = self._database.get_attachment(
- cdoc, self.COUCH_U1DB_ATTACHMENT_KEY).read()
- content = json.loads(jsonstr)
- # set u1db database info
- self._transaction_log = json.loads(
- b64decode(content[self.U1DB_TRANSACTION_LOG_KEY]))
- self._conflicts = json.loads(
- b64decode(content[self.U1DB_CONFLICTS_KEY]))
- self._other_generations = json.loads(
- b64decode(content[self.U1DB_OTHER_GENERATIONS_KEY]))
- self._indexes = self._load_indexes_from_json(
- b64decode(content[self.U1DB_INDEXES_KEY]))
- self._replica_uid = b64decode(content[self.U1DB_REPLICA_UID_KEY])
- # save couch _rev
- self._couch_rev = cdoc[self.COUCH_REV_KEY]
-
- def _store_u1db_data(self):
- """
- Store U1DB info in the couch db.
-
- See C{_init_u1db_data} documentation.
- """
- doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
- doc.content = {
- # Here, the b64 encode ensures that document content
- # does not cause strange behaviour in couchdb because
- # of encoding.
- self.U1DB_TRANSACTION_LOG_KEY:
- b64encode(json.dumps(self._transaction_log)),
- self.U1DB_CONFLICTS_KEY: b64encode(json.dumps(self._conflicts)),
- self.U1DB_OTHER_GENERATIONS_KEY:
- b64encode(json.dumps(self._other_generations)),
- self.U1DB_INDEXES_KEY: b64encode(self._dump_indexes_as_json()),
- self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
- self.COUCH_REV_KEY: self._couch_rev}
- self._put_doc(doc)
-
- #-------------------------------------------------------------------------
- # Couch specific methods
- #-------------------------------------------------------------------------
-
- INDEX_NAME_KEY = 'name'
- INDEX_DEFINITION_KEY = 'definition'
- INDEX_VALUES_KEY = 'values'
-
- def delete_database(self):
- """
- Delete a U1DB CouchDB database.
- """
- del(self._server[self._dbname])
-
- def _dump_indexes_as_json(self):
- """
- Dump index definitions as JSON string.
- """
- indexes = {}
- for name, idx in self._indexes.iteritems():
- indexes[name] = {}
- for attr in [self.INDEX_NAME_KEY, self.INDEX_DEFINITION_KEY,
- self.INDEX_VALUES_KEY]:
- indexes[name][attr] = getattr(idx, '_' + attr)
- return json.dumps(indexes)
-
- def _load_indexes_from_json(self, indexes):
- """
- Load index definitions from JSON string.
-
- @param indexes: A JSON serialization of a list of [('index-name',
- ['field', 'field2'])].
- @type indexes: str
-
- @return: A dictionary with the index definitions.
- @rtype: dict
- """
- dict = {}
- for name, idx_dict in json.loads(indexes).iteritems():
- idx = InMemoryIndex(name, idx_dict[self.INDEX_DEFINITION_KEY])
- idx._values = idx_dict[self.INDEX_VALUES_KEY]
- dict[name] = idx
- return dict
-
-
-class CouchSyncTarget(ObjectStoreSyncTarget):
- """
- Functionality for using a CouchDatabase as a synchronization target.
- """
-
-
-class CouchServerState(ServerState):
- """
- Inteface of the WSGI server with the CouchDB backend.
- """
-
- def __init__(self, couch_url):
- self._couch_url = couch_url
-
- def open_database(self, dbname):
- """
- Open a couch database.
-
- @param dbname: The name of the database to open.
- @type dbname: str
-
- @return: The CouchDatabase object.
- @rtype: CouchDatabase
- """
- # TODO: open couch
- return CouchDatabase.open_database(
- self._couch_url + '/' + dbname,
- create=False)
-
- def ensure_database(self, dbname):
- """
- Ensure couch database exists.
-
- @param dbname: The name of the database to ensure.
- @type dbname: str
-
- @return: The CouchDatabase object and the replica uid.
- @rtype: (CouchDatabase, str)
- """
- db = CouchDatabase.open_database(
- self._couch_url + '/' + dbname,
- create=True)
- return db, db._replica_uid
-
- def delete_database(self, dbname):
- """
- Delete couch database.
-
- @param dbname: The name of the database to delete.
- @type dbname: str
- """
- CouchDatabase.delete_database(self._couch_url + '/' + dbname)
-
- def _set_couch_url(self, url):
- """
- Set the couchdb URL
-
- @param url: CouchDB URL
- @type url: str
- """
- self._couch_url = url
-
- def _get_couch_url(self):
- """
- Return CouchDB URL
-
- @rtype: str
- """
- return self._couch_url
-
- couch_url = property(_get_couch_url, _set_couch_url, doc='CouchDB URL')
diff --git a/src/leap/soledad/backends/leap_backend.py b/src/leap/soledad/backends/leap_backend.py
deleted file mode 100644
index 4d92db37..00000000
--- a/src/leap/soledad/backends/leap_backend.py
+++ /dev/null
@@ -1,544 +0,0 @@
-# -*- coding: utf-8 -*-
-# leap_backend.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-A U1DB backend for encrypting data before sending to server and decrypting
-after receiving.
-"""
-
-import simplejson as json
-import hashlib
-import hmac
-import binascii
-
-
-from u1db import Document
-from u1db.remote import utils
-from u1db.errors import BrokenSyncStream
-from u1db.remote.http_target import HTTPSyncTarget
-
-
-from leap.soledad import soledad_assert
-from leap.soledad.crypto import (
- EncryptionMethods,
- UnknownEncryptionMethod,
-)
-from leap.soledad.auth import TokenBasedAuth
-
-
-#
-# Exceptions
-#
-
-class DocumentNotEncrypted(Exception):
- """
- Raised for failures in document encryption.
- """
- pass
-
-
-class UnknownEncryptionScheme(Exception):
- """
- Raised when trying to decrypt from unknown encryption schemes.
- """
- pass
-
-
-class UnknownMacMethod(Exception):
- """
- Raised when trying to authenticate document's content with unknown MAC
- mehtod.
- """
- pass
-
-
-class WrongMac(Exception):
- """
- Raised when failing to authenticate document's contents based on MAC.
- """
-
-
-#
-# Encryption schemes used for encryption.
-#
-
-class EncryptionSchemes(object):
- """
- Representation of encryption schemes used to encrypt documents.
- """
-
- NONE = 'none'
- SYMKEY = 'symkey'
- PUBKEY = 'pubkey'
-
-
-class MacMethods(object):
- """
- Representation of MAC methods used to authenticate document's contents.
- """
-
- HMAC = 'hmac'
-
-
-#
-# Crypto utilities for a LeapDocument.
-#
-
-ENC_JSON_KEY = '_enc_json'
-ENC_SCHEME_KEY = '_enc_scheme'
-ENC_METHOD_KEY = '_enc_method'
-ENC_IV_KEY = '_enc_iv'
-MAC_KEY = '_mac'
-MAC_METHOD_KEY = '_mac_method'
-
-
-def mac_doc(crypto, doc_id, doc_rev, ciphertext, mac_method):
- """
- Calculate a MAC for C{doc} using C{ciphertext}.
-
- Current MAC method used is HMAC, with the following parameters:
-
- * key: sha256(storage_secret, doc_id)
- * msg: doc_id + doc_rev + ciphertext
- * digestmod: sha256
-
- @param crypto: A SoledadCryto instance used to perform the encryption.
- @type crypto: leap.soledad.crypto.SoledadCrypto
- @param doc_id: The id of the document.
- @type doc_id: str
- @param doc_rev: The revision of the document.
- @type doc_rev: str
- @param ciphertext: The content of the document.
- @type ciphertext: str
- @param mac_method: The MAC method to use.
- @type mac_method: str
-
- @return: The calculated MAC.
- @rtype: str
- """
- if mac_method == MacMethods.HMAC:
- return hmac.new(
- crypto.doc_mac_key(doc_id),
- str(doc_id) + str(doc_rev) + ciphertext,
- hashlib.sha256).digest()
- # raise if we do not know how to handle this MAC method
- raise UnknownMacMethod('Unknown MAC method: %s.' % mac_method)
-
-
-def encrypt_doc(crypto, doc):
- """
- Encrypt C{doc}'s content.
-
- Encrypt doc's contents using AES-256 CTR mode and return a valid JSON
- string representing the following:
-
- {
- ENC_JSON_KEY: '<encrypted doc JSON string>',
- ENC_SCHEME_KEY: 'symkey',
- ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR,
- ENC_IV_KEY: '<the initial value used to encrypt>',
- MAC_KEY: '<mac>'
- MAC_METHOD_KEY: 'hmac'
- }
-
- @param crypto: A SoledadCryto instance used to perform the encryption.
- @type crypto: leap.soledad.crypto.SoledadCrypto
- @param doc: The document with contents to be encrypted.
- @type doc: LeapDocument
-
- @return: The JSON serialization of the dict representing the encrypted
- content.
- @rtype: str
- """
- soledad_assert(doc.is_tombstone() is False)
- # encrypt content using AES-256 CTR mode
- iv, ciphertext = crypto.encrypt_sym(
- doc.get_json(),
- crypto.doc_passphrase(doc.doc_id),
- method=EncryptionMethods.AES_256_CTR)
- # Return a representation for the encrypted content. In the following, we
- # convert binary data to hexadecimal representation so the JSON
- # serialization does not complain about what it tries to serialize.
- hex_ciphertext = binascii.b2a_hex(ciphertext)
- return json.dumps({
- ENC_JSON_KEY: hex_ciphertext,
- ENC_SCHEME_KEY: EncryptionSchemes.SYMKEY,
- ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR,
- ENC_IV_KEY: iv,
- MAC_KEY: binascii.b2a_hex(mac_doc( # store the mac as hex.
- crypto, doc.doc_id, doc.rev,
- ciphertext,
- MacMethods.HMAC)),
- MAC_METHOD_KEY: MacMethods.HMAC,
- })
-
-
-def decrypt_doc(crypto, doc):
- """
- Decrypt C{doc}'s content.
-
- Return the JSON string representation of the document's decrypted content.
-
- The content of the document should have the following structure:
-
- {
- ENC_JSON_KEY: '<enc_blob>',
- ENC_SCHEME_KEY: '<enc_scheme>',
- ENC_METHOD_KEY: '<enc_method>',
- ENC_IV_KEY: '<initial value used to encrypt>', # (optional)
- MAC_KEY: '<mac>'
- MAC_METHOD_KEY: 'hmac'
- }
-
- C{enc_blob} is the encryption of the JSON serialization of the document's
- content. For now Soledad just deals with documents whose C{enc_scheme} is
- EncryptionSchemes.SYMKEY and C{enc_method} is
- EncryptionMethods.AES_256_CTR.
-
- @param crypto: A SoledadCryto instance to perform the encryption.
- @type crypto: leap.soledad.crypto.SoledadCrypto
- @param doc: The document to be decrypted.
- @type doc: LeapDocument
-
- @return: The JSON serialization of the decrypted content.
- @rtype: str
- """
- soledad_assert(doc.is_tombstone() is False)
- soledad_assert(ENC_JSON_KEY in doc.content)
- soledad_assert(ENC_SCHEME_KEY in doc.content)
- soledad_assert(ENC_METHOD_KEY in doc.content)
- soledad_assert(MAC_KEY in doc.content)
- soledad_assert(MAC_METHOD_KEY in doc.content)
- # verify MAC
- ciphertext = binascii.a2b_hex( # content is stored as hex.
- doc.content[ENC_JSON_KEY])
- mac = mac_doc(
- crypto, doc.doc_id, doc.rev,
- ciphertext,
- doc.content[MAC_METHOD_KEY])
- if binascii.a2b_hex(doc.content[MAC_KEY]) != mac: # mac is stored as hex.
- raise WrongMac('Could not authenticate document\'s contents.')
- # decrypt doc's content
- enc_scheme = doc.content[ENC_SCHEME_KEY]
- plainjson = None
- if enc_scheme == EncryptionSchemes.SYMKEY:
- enc_method = doc.content[ENC_METHOD_KEY]
- if enc_method == EncryptionMethods.AES_256_CTR:
- soledad_assert(ENC_IV_KEY in doc.content)
- plainjson = crypto.decrypt_sym(
- ciphertext,
- crypto.doc_passphrase(doc.doc_id),
- method=enc_method,
- iv=doc.content[ENC_IV_KEY])
- else:
- raise UnknownEncryptionMethod(enc_method)
- else:
- raise UnknownEncryptionScheme(enc_scheme)
- return plainjson
-
-
-class LeapDocument(Document):
- """
- Encryptable and syncable document.
-
- LEAP Documents can be flagged as syncable or not, so the replicas
- might not sync every document.
- """
-
- def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False,
- syncable=True):
- """
- Container for handling an encryptable document.
-
- @param doc_id: The unique document identifier.
- @type doc_id: str
- @param rev: The revision identifier of the document.
- @type rev: str
- @param json: The JSON string for this document.
- @type json: str
- @param has_conflicts: Boolean indicating if this document has conflicts
- @type has_conflicts: bool
- @param syncable: Should this document be synced with remote replicas?
- @type syncable: bool
- """
- Document.__init__(self, doc_id, rev, json, has_conflicts)
- self._syncable = syncable
-
- def _get_syncable(self):
- """
- Return whether this document is syncable.
-
- @return: Is this document syncable?
- @rtype: bool
- """
- return self._syncable
-
- def _set_syncable(self, syncable=True):
- """
- Determine if this document should be synced with remote replicas.
-
- @param syncable: Should this document be synced with remote replicas?
- @type syncable: bool
- """
- self._syncable = syncable
-
- syncable = property(
- _get_syncable,
- _set_syncable,
- doc="Determine if document should be synced with server."
- )
-
- def _get_rev(self):
- """
- Get the document revision.
-
- Returning the revision as string solves the following exception in
- Twisted web:
- exceptions.TypeError: Can only pass-through bytes on Python 2
-
- @return: The document revision.
- @rtype: str
- """
- if self._rev is None:
- return None
- return str(self._rev)
-
- def _set_rev(self, rev):
- """
- Set document revision.
-
- @param rev: The new document revision.
- @type rev: bytes
- """
- self._rev = rev
-
- rev = property(
- _get_rev,
- _set_rev,
- doc="Wrapper to ensure `doc.rev` is always returned as bytes.")
-
-
-#
-# LeapSyncTarget
-#
-
-class LeapSyncTarget(HTTPSyncTarget, TokenBasedAuth):
- """
- A SyncTarget that encrypts data before sending and decrypts data after
- receiving.
- """
-
- #
- # Token auth methods.
- #
-
- def set_token_credentials(self, uuid, token):
- """
- Store given credentials so we can sign the request later.
-
- @param uuid: The user's uuid.
- @type uuid: str
- @param token: The authentication token.
- @type token: str
- """
- TokenBasedAuth.set_token_credentials(self, uuid, token)
-
- def _sign_request(self, method, url_query, params):
- """
- Return an authorization header to be included in the HTTP request.
-
- @param method: The HTTP method.
- @type method: str
- @param url_query: The URL query string.
- @type url_query: str
- @param params: A list with encoded query parameters.
- @type param: list
-
- @return: The Authorization header.
- @rtype: list of tuple
- """
- return TokenBasedAuth._sign_request(self, method, url_query, params)
-
- #
- # Modified HTTPSyncTarget methods.
- #
-
- @staticmethod
- def connect(url, crypto=None):
- return LeapSyncTarget(url, crypto=crypto)
-
- def __init__(self, url, creds=None, crypto=None):
- """
- Initialize the LeapSyncTarget.
-
- @param url: The url of the target replica to sync with.
- @type url: str
- @param creds: optional dictionary giving credentials.
- to authorize the operation with the server.
- @type creds: dict
- @param soledad: An instance of Soledad so we can encrypt/decrypt
- document contents when syncing.
- @type soledad: soledad.Soledad
- """
- HTTPSyncTarget.__init__(self, url, creds)
- self._crypto = crypto
-
- def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None):
- """
- Parse incoming synchronization stream and insert documents in the
- local database.
-
- If an incoming document's encryption scheme is equal to
- EncryptionSchemes.SYMKEY, then this method will decrypt it with
- Soledad's symmetric key.
-
- @param data: The body of the HTTP response.
- @type data: str
- @param return_doc_cb: A callback to insert docs from target.
- @type return_doc_cb: function
- @param ensure_callback: A callback to ensure we have the correct
- target_replica_uid, if it was just created.
- @type ensure_callback: function
-
- @raise BrokenSyncStream: If C{data} is malformed.
-
- @return: A dictionary representing the first line of the response got
- from remote replica.
- @rtype: list of str
- """
- parts = data.splitlines() # one at a time
- if not parts or parts[0] != '[':
- raise BrokenSyncStream
- data = parts[1:-1]
- comma = False
- if data:
- line, comma = utils.check_and_strip_comma(data[0])
- res = json.loads(line)
- if ensure_callback and 'replica_uid' in res:
- ensure_callback(res['replica_uid'])
- for entry in data[1:]:
- if not comma: # missing in between comma
- raise BrokenSyncStream
- line, comma = utils.check_and_strip_comma(entry)
- entry = json.loads(line)
- #-------------------------------------------------------------
- # symmetric decryption of document's contents
- #-------------------------------------------------------------
- # if arriving content was symmetrically encrypted, we decrypt
- # it.
- doc = LeapDocument(entry['id'], entry['rev'], entry['content'])
- if doc.content and ENC_SCHEME_KEY in doc.content:
- if doc.content[ENC_SCHEME_KEY] == \
- EncryptionSchemes.SYMKEY:
- doc.set_json(decrypt_doc(self._crypto, doc))
- #-------------------------------------------------------------
- # end of symmetric decryption
- #-------------------------------------------------------------
- return_doc_cb(doc, entry['gen'], entry['trans_id'])
- if parts[-1] != ']':
- try:
- partdic = json.loads(parts[-1])
- except ValueError:
- pass
- else:
- if isinstance(partdic, dict):
- self._error(partdic)
- raise BrokenSyncStream
- if not data or comma: # no entries or bad extra comma
- raise BrokenSyncStream
- return res
-
- def sync_exchange(self, docs_by_generations, source_replica_uid,
- last_known_generation, last_known_trans_id,
- return_doc_cb, ensure_callback=None):
- """
- Find out which documents the remote database does not know about,
- encrypt and send them.
-
- This does the same as the parent's method but encrypts content before
- syncing.
-
- @param docs_by_generations: A list of (doc_id, generation, trans_id)
- of local documents that were changed since the last local
- generation the remote replica knows about.
- @type docs_by_generations: list of tuples
- @param source_replica_uid: The uid of the source replica.
- @type source_replica_uid: str
- @param last_known_generation: Target's last known generation.
- @type last_known_generation: int
- @param last_known_trans_id: Target's last known transaction id.
- @type last_known_trans_id: str
- @param return_doc_cb: A callback for inserting received documents from
- target.
- @type return_doc_cb: function
- @param ensure_callback: A callback that ensures we know the target
- replica uid if the target replica was just created.
- @type ensure_callback: function
-
- @return: The new generation and transaction id of the target replica.
- @rtype: tuple
- """
- self._ensure_connection()
- if self._trace_hook: # for tests
- self._trace_hook('sync_exchange')
- url = '%s/sync-from/%s' % (self._url.path, source_replica_uid)
- self._conn.putrequest('POST', url)
- self._conn.putheader('content-type', 'application/x-u1db-sync-stream')
- for header_name, header_value in self._sign_request('POST', url, {}):
- self._conn.putheader(header_name, header_value)
- entries = ['[']
- size = 1
-
- def prepare(**dic):
- entry = comma + '\r\n' + json.dumps(dic)
- entries.append(entry)
- return len(entry)
-
- comma = ''
- size += prepare(
- last_known_generation=last_known_generation,
- last_known_trans_id=last_known_trans_id,
- ensure=ensure_callback is not None)
- comma = ','
- for doc, gen, trans_id in docs_by_generations:
- # skip non-syncable docs
- if isinstance(doc, LeapDocument) and not doc.syncable:
- continue
- #-------------------------------------------------------------
- # symmetric encryption of document's contents
- #-------------------------------------------------------------
- doc_json = doc.get_json()
- if not doc.is_tombstone():
- doc_json = encrypt_doc(self._crypto, doc)
- #-------------------------------------------------------------
- # end of symmetric encryption
- #-------------------------------------------------------------
- size += prepare(id=doc.doc_id, rev=doc.rev,
- content=doc_json,
- gen=gen, trans_id=trans_id)
- entries.append('\r\n]')
- size += len(entries[-1])
- self._conn.putheader('content-length', str(size))
- self._conn.endheaders()
- for entry in entries:
- self._conn.send(entry)
- entries = None
- data, _ = self._response()
- res = self._parse_sync_stream(data, return_doc_cb, ensure_callback)
- data = None
- return res['new_generation'], res['new_transaction_id']
diff --git a/src/leap/soledad/backends/objectstore.py b/src/leap/soledad/backends/objectstore.py
deleted file mode 100644
index 8afac3ec..00000000
--- a/src/leap/soledad/backends/objectstore.py
+++ /dev/null
@@ -1,296 +0,0 @@
-# -*- coding: utf-8 -*-
-# objectstore.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-Abstract U1DB backend to handle storage using object stores (like CouchDB, for
-example).
-
-Right now, this is only used by CouchDatabase backend, but can also be
-extended to implement OpenStack or Amazon S3 storage, for example.
-
-See U1DB documentation for more information on how to use databases.
-"""
-
-from u1db.backends.inmemory import (
- InMemoryDatabase,
- InMemorySyncTarget,
-)
-from u1db import errors
-
-
-class ObjectStoreDatabase(InMemoryDatabase):
- """
- A backend for storing u1db data in an object store.
- """
-
- @classmethod
- def open_database(cls, url, create, document_factory=None):
- """
- Open a U1DB database using an object store as backend.
-
- @param url: the url of the database replica
- @type url: str
- @param create: should the replica be created if it does not exist?
- @type create: bool
- @param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- @type document_factory: callable
-
- @return: the database instance
- @rtype: CouchDatabase
- """
- raise NotImplementedError(cls.open_database)
-
- def __init__(self, replica_uid=None, document_factory=None):
- """
- Initialize the object store database.
-
- @param replica_uid: an optional unique replica identifier
- @type replica_uid: str
- @param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- @type document_factory: callable
- """
- InMemoryDatabase.__init__(
- self,
- replica_uid,
- document_factory=document_factory)
- # sync data in memory with data in object store
- if not self._get_doc(self.U1DB_DATA_DOC_ID):
- self._init_u1db_data()
- self._fetch_u1db_data()
-
- #-------------------------------------------------------------------------
- # methods from Database
- #-------------------------------------------------------------------------
-
- def _set_replica_uid(self, replica_uid):
- """
- Force the replica_uid to be set.
-
- @param replica_uid: The uid of the replica.
- @type replica_uid: str
- """
- InMemoryDatabase._set_replica_uid(self, replica_uid)
- self._store_u1db_data()
-
- def _put_doc(self, doc):
- """
- Update a document.
-
- This is called everytime we just want to do a raw put on the db (i.e.
- without index updates, document constraint checks, and conflict
- checks).
-
- @param doc: The document to update.
- @type doc: u1db.Document
-
- @return: The new revision identifier for the document.
- @rtype: str
- """
- raise NotImplementedError(self._put_doc)
-
- def _get_doc(self, doc_id):
- """
- Get just the document content, without fancy handling.
-
- @param doc_id: The unique document identifier
- @type doc_id: str
- @param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise asking for a deleted
- document will return None.
- @type include_deleted: bool
-
- @return: a Document object.
- @type: u1db.Document
- """
- raise NotImplementedError(self._get_doc)
-
- def get_all_docs(self, include_deleted=False):
- """
- Get the JSON content for all documents in the database.
-
- @param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise deleted documents will not
- be included in the results.
- @type include_deleted: bool
-
- @return: (generation, [Document])
- The current generation of the database, followed by a list of all
- the documents in the database.
- @rtype: tuple
- """
- raise NotImplementedError(self.get_all_docs)
-
- def delete_doc(self, doc):
- """
- Mark a document as deleted.
-
- @param doc: The document to mark as deleted.
- @type doc: u1db.Document
-
- @return: The new revision id of the document.
- @type: str
- """
- old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
- if old_doc is None:
- raise errors.DocumentDoesNotExist
- if old_doc.rev != doc.rev:
- raise errors.RevisionConflict()
- if old_doc.is_tombstone():
- raise errors.DocumentAlreadyDeleted
- if old_doc.has_conflicts:
- raise errors.ConflictedDoc()
- new_rev = self._allocate_doc_rev(doc.rev)
- doc.rev = new_rev
- doc.make_tombstone()
- self._put_and_update_indexes(old_doc, doc)
- return new_rev
-
- # index-related methods
-
- def create_index(self, index_name, *index_expressions):
- """
- Create a named index, which can then be queried for future lookups.
-
- See U1DB documentation for more information.
-
- @param index_name: A unique name which can be used as a key prefix.
- @param index_expressions: Index expressions defining the index
- information.
- """
- raise NotImplementedError(self.create_index)
-
- def delete_index(self, index_name):
- """
- Remove a named index.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param index_name: The name of the index we are removing.
- @type index_name: str
- """
- InMemoryDatabase.delete_index(self, index_name)
- self._store_u1db_data()
-
- def _replace_conflicts(self, doc, conflicts):
- """
- Set new conflicts for a document.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param doc: The document with a new set of conflicts.
- @param conflicts: The new set of conflicts.
- @type conflicts: list
- """
- InMemoryDatabase._replace_conflicts(self, doc, conflicts)
- self._store_u1db_data()
-
- def _do_set_replica_gen_and_trans_id(self, other_replica_uid,
- other_generation,
- other_transaction_id):
- """
- Set the last-known generation and transaction id for the other
- database replica.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param other_replica_uid: The U1DB identifier for the other replica.
- @type other_replica_uid: str
- @param other_generation: The generation number for the other replica.
- @type other_generation: int
- @param other_transaction_id: The transaction id associated with the
- generation.
- @type other_transaction_id: str
- """
- InMemoryDatabase._do_set_replica_gen_and_trans_id(
- self,
- other_replica_uid,
- other_generation,
- other_transaction_id)
- self._store_u1db_data()
-
- #-------------------------------------------------------------------------
- # implemented methods from CommonBackend
- #-------------------------------------------------------------------------
-
- def _put_and_update_indexes(self, old_doc, doc):
- """
- Update a document and all indexes related to it.
-
- @param old_doc: The old version of the document.
- @type old_doc: u1db.Document
- @param doc: The new version of the document.
- @type doc: u1db.Document
- """
- for index in self._indexes.itervalues():
- if old_doc is not None and not old_doc.is_tombstone():
- index.remove_json(old_doc.doc_id, old_doc.get_json())
- if not doc.is_tombstone():
- index.add_json(doc.doc_id, doc.get_json())
- trans_id = self._allocate_transaction_id()
- self._put_doc(doc)
- self._transaction_log.append((doc.doc_id, trans_id))
- self._store_u1db_data()
-
- #-------------------------------------------------------------------------
- # methods specific for object stores
- #-------------------------------------------------------------------------
-
- U1DB_DATA_DOC_ID = 'u1db_data'
-
- def _fetch_u1db_data(self):
- """
- Fetch u1db configuration data from backend storage.
-
- See C{_init_u1db_data} documentation.
- """
- NotImplementedError(self._fetch_u1db_data)
-
- def _store_u1db_data(self):
- """
- Store u1db configuration data on backend storage.
-
- See C{_init_u1db_data} documentation.
- """
- NotImplementedError(self._store_u1db_data)
-
- def _init_u1db_data(self):
- """
- Initialize u1db configuration data on backend storage.
-
- A U1DB database needs to keep track of all database transactions,
- document conflicts, the generation of other replicas it has seen,
- indexes created by users and so on.
-
- In this implementation, all this information is stored in a special
- document stored in the couch db with id equals to
- CouchDatabse.U1DB_DATA_DOC_ID.
-
- This method initializes the document that will hold such information.
- """
- NotImplementedError(self._init_u1db_data)
-
-
-class ObjectStoreSyncTarget(InMemorySyncTarget):
- """
- Functionality for using an ObjectStore as a synchronization target.
- """
diff --git a/src/leap/soledad/backends/sqlcipher.py b/src/leap/soledad/backends/sqlcipher.py
deleted file mode 100644
index d6d62f21..00000000
--- a/src/leap/soledad/backends/sqlcipher.py
+++ /dev/null
@@ -1,653 +0,0 @@
-# -*- coding: utf-8 -*-
-# sqlcipher.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-A U1DB backend that uses SQLCipher as its persistence layer.
-
-The SQLCipher API (http://sqlcipher.net/sqlcipher-api/) is fully implemented,
-with the exception of the following statements:
-
- * PRAGMA cipher_use_hmac
- * PRAGMA cipher_default_use_mac
-
-SQLCipher 2.0 introduced a per-page HMAC to validate that the page data has
-not be tampered with. By default, when creating or opening a database using
-SQLCipher 2, SQLCipher will attempt to use an HMAC check. This change in
-database format means that SQLCipher 2 can't operate on version 1.1.x
-databases by default. Thus, in order to provide backward compatibility with
-SQLCipher 1.1.x, PRAGMA cipher_use_hmac can be used to disable the HMAC
-functionality on specific databases.
-
-In some very specific cases, it is not possible to call PRAGMA cipher_use_hmac
-as one of the first operations on a database. An example of this is when
-trying to ATTACH a 1.1.x database to the main database. In these cases PRAGMA
-cipher_default_use_hmac can be used to globally alter the default use of HMAC
-when opening a database.
-
-So, as the statements above were introduced for backwards compatibility with
-SLCipher 1.1 databases, we do not implement them as all SQLCipher databases
-handled by Soledad should be created by SQLCipher >= 2.0.
-"""
-
-import os
-import time
-import string
-
-
-from u1db.backends import sqlite_backend
-from pysqlcipher import dbapi2
-from u1db import (
- errors,
-)
-from leap.soledad.backends.leap_backend import LeapDocument
-
-
-# Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2
-sqlite_backend.dbapi2 = dbapi2
-
-
-def open(path, password, create=True, document_factory=None, crypto=None,
- raw_key=False, cipher='aes-256-cbc', kdf_iter=4000,
- cipher_page_size=1024):
- """Open a database at the given location.
-
- Will raise u1db.errors.DatabaseDoesNotExist if create=False and the
- database does not already exist.
-
- @param path: The filesystem path for the database to open.
- @param type: str
- @param create: True/False, should the database be created if it doesn't
- already exist?
- @param type: bool
- @param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- @type document_factory: callable
- @param crypto: An instance of SoledadCrypto so we can encrypt/decrypt
- document contents when syncing.
- @type crypto: soledad.crypto.SoledadCrypto
- @param raw_key: Whether C{password} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
- @type raw_key: bool
- @param cipher: The cipher and mode to use.
- @type cipher: str
- @param kdf_iter: The number of iterations to use.
- @type kdf_iter: int
- @param cipher_page_size: The page size.
- @type cipher_page_size: int
-
- @return: An instance of Database.
- @rtype SQLCipherDatabase
- """
- return SQLCipherDatabase.open_database(
- path, password, create=create, document_factory=document_factory,
- crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter,
- cipher_page_size=cipher_page_size)
-
-
-#
-# Exceptions
-#
-
-class DatabaseIsNotEncrypted(Exception):
- """
- Exception raised when trying to open non-encrypted databases.
- """
- pass
-
-
-class NotAnHexString(Exception):
- """
- Raised when trying to (raw) key the database with a non-hex string.
- """
- pass
-
-
-#
-# The SQLCipher database
-#
-
-class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase):
- """A U1DB implementation that uses SQLCipher as its persistence layer."""
-
- _index_storage_value = 'expand referenced encrypted'
-
- def __init__(self, sqlcipher_file, password, document_factory=None,
- crypto=None, raw_key=False, cipher='aes-256-cbc',
- kdf_iter=4000, cipher_page_size=1024):
- """
- Create a new sqlcipher file.
-
- @param sqlcipher_file: The path for the SQLCipher file.
- @type sqlcipher_file: str
- @param password: The password that protects the SQLCipher db.
- @type password: str
- @param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- @type document_factory: callable
- @param crypto: An instance of SoledadCrypto so we can encrypt/decrypt
- document contents when syncing.
- @type crypto: soledad.crypto.SoledadCrypto
- @param raw_key: Whether C{password} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
- @type raw_key: bool
- @param cipher: The cipher and mode to use.
- @type cipher: str
- @param kdf_iter: The number of iterations to use.
- @type kdf_iter: int
- @param cipher_page_size: The page size.
- @type cipher_page_size: int
- """
- # ensure the db is encrypted if the file already exists
- if os.path.exists(sqlcipher_file):
- self.assert_db_is_encrypted(
- sqlcipher_file, password, raw_key, cipher, kdf_iter,
- cipher_page_size)
- # connect to the database
- self._db_handle = dbapi2.connect(sqlcipher_file)
- # set SQLCipher cryptographic parameters
- self._set_crypto_pragmas(
- self._db_handle, password, raw_key, cipher, kdf_iter,
- cipher_page_size)
- self._real_replica_uid = None
- self._ensure_schema()
- self._crypto = crypto
-
- def factory(doc_id=None, rev=None, json='{}', has_conflicts=False,
- syncable=True):
- return LeapDocument(doc_id=doc_id, rev=rev, json=json,
- has_conflicts=has_conflicts,
- syncable=syncable)
- self.set_document_factory(factory)
-
- @classmethod
- def _open_database(cls, sqlcipher_file, password, document_factory=None,
- crypto=None, raw_key=False, cipher='aes-256-cbc',
- kdf_iter=4000, cipher_page_size=1024):
- """
- Open a SQLCipher database.
-
- @param sqlcipher_file: The path for the SQLCipher file.
- @type sqlcipher_file: str
- @param password: The password that protects the SQLCipher db.
- @type password: str
- @param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- @type document_factory: callable
- @param crypto: An instance of SoledadCrypto so we can encrypt/decrypt
- document contents when syncing.
- @type crypto: soledad.crypto.SoledadCrypto
- @param raw_key: Whether C{password} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
- @type raw_key: bool
- @param cipher: The cipher and mode to use.
- @type cipher: str
- @param kdf_iter: The number of iterations to use.
- @type kdf_iter: int
- @param cipher_page_size: The page size.
- @type cipher_page_size: int
-
- @return: The database object.
- @rtype: SQLCipherDatabase
- """
- if not os.path.isfile(sqlcipher_file):
- raise errors.DatabaseDoesNotExist()
- tries = 2
- while True:
- # Note: There seems to be a bug in sqlite 3.5.9 (with python2.6)
- # where without re-opening the database on Windows, it
- # doesn't see the transaction that was just committed
- db_handle = dbapi2.connect(sqlcipher_file)
- # set cryptographic params
- cls._set_crypto_pragmas(
- db_handle, password, raw_key, cipher, kdf_iter,
- cipher_page_size)
- c = db_handle.cursor()
- v, err = cls._which_index_storage(c)
- db_handle.close()
- if v is not None:
- break
- # possibly another process is initializing it, wait for it to be
- # done
- if tries == 0:
- raise err # go for the richest error?
- tries -= 1
- time.sleep(cls.WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL)
- return SQLCipherDatabase._sqlite_registry[v](
- sqlcipher_file, password, document_factory=document_factory,
- crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter,
- cipher_page_size=cipher_page_size)
-
- @classmethod
- def open_database(cls, sqlcipher_file, password, create, backend_cls=None,
- document_factory=None, crypto=None, raw_key=False,
- cipher='aes-256-cbc', kdf_iter=4000,
- cipher_page_size=1024):
- """
- Open a SQLCipher database.
-
- @param sqlcipher_file: The path for the SQLCipher file.
- @type sqlcipher_file: str
- @param password: The password that protects the SQLCipher db.
- @type password: str
- @param create: Should the datbase be created if it does not already
- exist?
- @type: bool
- @param backend_cls: A class to use as backend.
- @type backend_cls: type
- @param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- @type document_factory: callable
- @param crypto: An instance of SoledadCrypto so we can encrypt/decrypt
- document contents when syncing.
- @type crypto: soledad.crypto.SoledadCrypto
- @param raw_key: Whether C{password} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
- @type raw_key: bool
- @param cipher: The cipher and mode to use.
- @type cipher: str
- @param kdf_iter: The number of iterations to use.
- @type kdf_iter: int
- @param cipher_page_size: The page size.
- @type cipher_page_size: int
-
- @return: The database object.
- @rtype: SQLCipherDatabase
- """
- try:
- return cls._open_database(
- sqlcipher_file, password, document_factory=document_factory,
- crypto=crypto, raw_key=raw_key, cipher=cipher,
- kdf_iter=kdf_iter, cipher_page_size=cipher_page_size)
- except errors.DatabaseDoesNotExist:
- if not create:
- raise
- # TODO: remove backend class from here.
- if backend_cls is None:
- # default is SQLCipherPartialExpandDatabase
- backend_cls = SQLCipherDatabase
- return backend_cls(
- sqlcipher_file, password, document_factory=document_factory,
- crypto=crypto, raw_key=raw_key, cipher=cipher,
- kdf_iter=kdf_iter, cipher_page_size=cipher_page_size)
-
- def sync(self, url, creds=None, autocreate=True):
- """
- Synchronize documents with remote replica exposed at url.
-
- @param url: The url of the target replica to sync with.
- @type url: str
- @param creds: optional dictionary giving credentials.
- to authorize the operation with the server.
- @type creds: dict
- @param autocreate: Ask the target to create the db if non-existent.
- @type autocreate: bool
-
- @return: The local generation before the synchronisation was performed.
- @rtype: int
- """
- from u1db.sync import Synchronizer
- from leap.soledad.backends.leap_backend import LeapSyncTarget
- return Synchronizer(
- self,
- LeapSyncTarget(url,
- creds=creds,
- crypto=self._crypto)).sync(autocreate=autocreate)
-
- def _extra_schema_init(self, c):
- """
- Add any extra fields, etc to the basic table definitions.
-
- @param c: The cursor for querying the database.
- @type c: dbapi2.cursor
- """
- c.execute(
- 'ALTER TABLE document '
- 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE')
-
- def _put_and_update_indexes(self, old_doc, doc):
- """
- Update a document and all indexes related to it.
-
- @param old_doc: The old version of the document.
- @type old_doc: u1db.Document
- @param doc: The new version of the document.
- @type doc: u1db.Document
- """
- sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes(
- self, old_doc, doc)
- c = self._db_handle.cursor()
- c.execute('UPDATE document SET syncable=? '
- 'WHERE doc_id=?',
- (doc.syncable, doc.doc_id))
-
- def _get_doc(self, doc_id, check_for_conflicts=False):
- """
- Get just the document content, without fancy handling.
-
- @param doc_id: The unique document identifier
- @type doc_id: str
- @param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise asking for a deleted
- document will return None.
- @type include_deleted: bool
-
- @return: a Document object.
- @type: u1db.Document
- """
- doc = sqlite_backend.SQLitePartialExpandDatabase._get_doc(
- self, doc_id, check_for_conflicts)
- if doc:
- c = self._db_handle.cursor()
- c.execute('SELECT syncable FROM document '
- 'WHERE doc_id=?',
- (doc.doc_id,))
- result = c.fetchone()
- doc.syncable = bool(result[0])
- return doc
-
- #
- # SQLCipher API methods
- #
-
- @classmethod
- def assert_db_is_encrypted(cls, sqlcipher_file, key, raw_key, cipher,
- kdf_iter, cipher_page_size):
- """
- Assert that C{sqlcipher_file} contains an encrypted database.
-
- When opening an existing database, PRAGMA key will not immediately
- throw an error if the key provided is incorrect. To test that the
- database can be successfully opened with the provided key, it is
- necessary to perform some operation on the database (i.e. read from
- it) and confirm it is success.
-
- The easiest way to do this is select off the sqlite_master table,
- which will attempt to read the first page of the database and will
- parse the schema.
-
- @param sqlcipher_file: The path for the SQLCipher file.
- @type sqlcipher_file: str
- @param key: The key that protects the SQLCipher db.
- @type key: str
- @param raw_key: Whether C{key} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
- @type raw_key: bool
- @param cipher: The cipher and mode to use.
- @type cipher: str
- @param kdf_iter: The number of iterations to use.
- @type kdf_iter: int
- @param cipher_page_size: The page size.
- @type cipher_page_size: int
- """
- try:
- # try to open an encrypted database with the regular u1db
- # backend should raise a DatabaseError exception.
- sqlite_backend.SQLitePartialExpandDatabase(sqlcipher_file)
- raise DatabaseIsNotEncrypted()
- except dbapi2.DatabaseError:
- # assert that we can access it using SQLCipher with the given
- # key
- db_handle = dbapi2.connect(sqlcipher_file)
- cls._set_crypto_pragmas(
- db_handle, key, raw_key, cipher, kdf_iter, cipher_page_size)
- db_handle.cursor().execute('SELECT count(*) FROM sqlite_master')
-
- @classmethod
- def _set_crypto_pragmas(cls, db_handle, key, raw_key, cipher, kdf_iter,
- cipher_page_size):
- """
- Set cryptographic params (key, cipher, KDF number of iterations and
- cipher page size).
- """
- cls._pragma_key(db_handle, key, raw_key)
- cls._pragma_cipher(db_handle, cipher)
- cls._pragma_kdf_iter(db_handle, kdf_iter)
- cls._pragma_cipher_page_size(db_handle, cipher_page_size)
-
- @classmethod
- def _pragma_key(cls, db_handle, key, raw_key):
- """
- Set the C{key} for use with the database.
-
- The process of creating a new, encrypted database is called 'keying'
- the database. SQLCipher uses just-in-time key derivation at the point
- it is first needed for an operation. This means that the key (and any
- options) must be set before the first operation on the database. As
- soon as the database is touched (e.g. SELECT, CREATE TABLE, UPDATE,
- etc.) and pages need to be read or written, the key is prepared for
- use.
-
- Implementation Notes:
-
- * PRAGMA key should generally be called as the first operation on a
- database.
-
- @param key: The key for use with the database.
- @type key: str
- @param raw_key: Whether C{key} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
- @type raw_key: bool
- """
- if raw_key:
- cls._pragma_key_raw(db_handle, key)
- else:
- cls._pragma_key_passphrase(db_handle, key)
-
- @classmethod
- def _pragma_key_passphrase(cls, db_handle, passphrase):
- """
- Set a passphrase for encryption key derivation.
-
- The key itself can be a passphrase, which is converted to a key using
- PBKDF2 key derivation. The result is used as the encryption key for
- the database. By using this method, there is no way to alter the KDF;
- if you want to do so you should use a raw key instead and derive the
- key using your own KDF.
-
- @param db_handle: A handle to the SQLCipher database.
- @type db_handle: pysqlcipher.Connection
- @param passphrase: The passphrase used to derive the encryption key.
- @type passphrase: str
- """
- db_handle.cursor().execute("PRAGMA key = '%s'" % passphrase)
-
- @classmethod
- def _pragma_key_raw(cls, db_handle, key):
- """
- Set a raw hexadecimal encryption key.
-
- It is possible to specify an exact byte sequence using a blob literal.
- With this method, it is the calling application's responsibility to
- ensure that the data provided is a 64 character hex string, which will
- be converted directly to 32 bytes (256 bits) of key data.
-
- @param db_handle: A handle to the SQLCipher database.
- @type db_handle: pysqlcipher.Connection
- @param key: A 64 character hex string.
- @type key: str
- """
- if not all(c in string.hexdigits for c in key):
- raise NotAnHexString(key)
- db_handle.cursor().execute('PRAGMA key = "x\'%s"' % key)
-
- @classmethod
- def _pragma_cipher(cls, db_handle, cipher='aes-256-cbc'):
- """
- Set the cipher and mode to use for symmetric encryption.
-
- SQLCipher uses aes-256-cbc as the default cipher and mode of
- operation. It is possible to change this, though not generally
- recommended, using PRAGMA cipher.
-
- SQLCipher makes direct use of libssl, so all cipher options available
- to libssl are also available for use with SQLCipher. See `man enc` for
- OpenSSL's supported ciphers.
-
- Implementation Notes:
-
- * PRAGMA cipher must be called after PRAGMA key and before the first
- actual database operation or it will have no effect.
-
- * If a non-default value is used PRAGMA cipher to create a database,
- it must also be called every time that database is opened.
-
- * SQLCipher does not implement its own encryption. Instead it uses the
- widely available and peer-reviewed OpenSSL libcrypto for all
- cryptographic functions.
-
- @param db_handle: A handle to the SQLCipher database.
- @type db_handle: pysqlcipher.Connection
- @param cipher: The cipher and mode to use.
- @type cipher: str
- """
- db_handle.cursor().execute("PRAGMA cipher = '%s'" % cipher)
-
- @classmethod
- def _pragma_kdf_iter(cls, db_handle, kdf_iter=4000):
- """
- Set the number of iterations for the key derivation function.
-
- SQLCipher uses PBKDF2 key derivation to strengthen the key and make it
- resistent to brute force and dictionary attacks. The default
- configuration uses 4000 PBKDF2 iterations (effectively 16,000 SHA1
- operations). PRAGMA kdf_iter can be used to increase or decrease the
- number of iterations used.
-
- Implementation Notes:
-
- * PRAGMA kdf_iter must be called after PRAGMA key and before the first
- actual database operation or it will have no effect.
-
- * If a non-default value is used PRAGMA kdf_iter to create a database,
- it must also be called every time that database is opened.
-
- * It is not recommended to reduce the number of iterations if a
- passphrase is in use.
-
- @param db_handle: A handle to the SQLCipher database.
- @type db_handle: pysqlcipher.Connection
- @param kdf_iter: The number of iterations to use.
- @type kdf_iter: int
- """
- db_handle.cursor().execute("PRAGMA kdf_iter = '%d'" % kdf_iter)
-
- @classmethod
- def _pragma_cipher_page_size(cls, db_handle, cipher_page_size=1024):
- """
- Set the page size of the encrypted database.
-
- SQLCipher 2 introduced the new PRAGMA cipher_page_size that can be
- used to adjust the page size for the encrypted database. The default
- page size is 1024 bytes, but it can be desirable for some applications
- to use a larger page size for increased performance. For instance,
- some recent testing shows that increasing the page size can noticeably
- improve performance (5-30%) for certain queries that manipulate a
- large number of pages (e.g. selects without an index, large inserts in
- a transaction, big deletes).
-
- To adjust the page size, call the pragma immediately after setting the
- key for the first time and each subsequent time that you open the
- database.
-
- Implementation Notes:
-
- * PRAGMA cipher_page_size must be called after PRAGMA key and before
- the first actual database operation or it will have no effect.
-
- * If a non-default value is used PRAGMA cipher_page_size to create a
- database, it must also be called every time that database is opened.
-
- @param db_handle: A handle to the SQLCipher database.
- @type db_handle: pysqlcipher.Connection
- @param cipher_page_size: The page size.
- @type cipher_page_size: int
- """
- db_handle.cursor().execute(
- "PRAGMA cipher_page_size = '%d'" % cipher_page_size)
-
- @classmethod
- def _pragma_rekey(cls, db_handle, new_key, raw_key):
- """
- Change the key of an existing encrypted database.
-
- To change the key on an existing encrypted database, it must first be
- unlocked with the current encryption key. Once the database is
- readable and writeable, PRAGMA rekey can be used to re-encrypt every
- page in the database with a new key.
-
- * PRAGMA rekey must be called after PRAGMA key. It can be called at any
- time once the database is readable.
-
- * PRAGMA rekey can not be used to encrypted a standard SQLite
- database! It is only useful for changing the key on an existing
- database.
-
- * Previous versions of SQLCipher provided a PRAGMA rekey_cipher and
- code>PRAGMA rekey_kdf_iter. These are deprecated and should not be
- used. Instead, use sqlcipher_export().
-
- @param db_handle: A handle to the SQLCipher database.
- @type db_handle: pysqlcipher.Connection
- @param new_key: The new key.
- @type new_key: str
- @param raw_key: Whether C{password} is a raw 64-char hex string or a
- passphrase that should be hashed to obtain the encyrption key.
- @type raw_key: bool
- """
- if raw_key:
- cls._pragma_rekey_raw(db_handle, key)
- else:
- cls._pragma_rekey_passphrase(db_handle, key)
-
- @classmethod
- def _pragma_rekey_passphrase(cls, db_handle, passphrase):
- """
- Change the passphrase for encryption key derivation.
-
- The key itself can be a passphrase, which is converted to a key using
- PBKDF2 key derivation. The result is used as the encryption key for
- the database.
-
- @param db_handle: A handle to the SQLCipher database.
- @type db_handle: pysqlcipher.Connection
- @param passphrase: The passphrase used to derive the encryption key.
- @type passphrase: str
- """
- db_handle.cursor().execute("PRAGMA rekey = '%s'" % passphrase)
-
- @classmethod
- def _pragma_rekey_raw(cls, db_handle, key):
- """
- Change the raw hexadecimal encryption key.
-
- It is possible to specify an exact byte sequence using a blob literal.
- With this method, it is the calling application's responsibility to
- ensure that the data provided is a 64 character hex string, which will
- be converted directly to 32 bytes (256 bits) of key data.
-
- @param db_handle: A handle to the SQLCipher database.
- @type db_handle: pysqlcipher.Connection
- @param key: A 64 character hex string.
- @type key: str
- """
- if not all(c in string.hexdigits for c in key):
- raise NotAnHexString(key)
- db_handle.cursor().execute('PRAGMA rekey = "x\'%s"' % passphrase)
-
-
-sqlite_backend.SQLiteDatabase.register_implementation(SQLCipherDatabase)