summaryrefslogtreecommitdiff
path: root/common/src
diff options
context:
space:
mode:
Diffstat (limited to 'common/src')
-rw-r--r--common/src/leap/soledad/common/couch.py43
-rw-r--r--common/src/leap/soledad/common/ddocs/__init__.py4
-rw-r--r--common/src/leap/soledad/common/tests/couchdb.ini.template4
-rw-r--r--common/src/leap/soledad/common/tests/test_couch.py240
4 files changed, 141 insertions, 150 deletions
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 4caaf48f..b9e699f3 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -18,35 +18,25 @@
"""A U1DB backend that uses CouchDB as its persistence layer."""
-import os
import simplejson as json
-import sys
-import time
+import re
import uuid
import logging
import binascii
import socket
+from couchdb.client import Server
+from couchdb.http import ResourceNotFound, Unauthorized
+from u1db import errors, query_parser, vectorclock
from u1db.backends import CommonBackend, CommonSyncTarget
from u1db.remote import http_app
from u1db.remote.server_state import ServerState
-from u1db import (
- errors,
- query_parser,
- vectorclock,
-)
-from couchdb.client import (
- Server,
- Document as CouchDocument,
- _doc_resource,
-)
-from couchdb.http import ResourceNotFound, Unauthorized
from leap.soledad.common import USER_DB_PREFIX
from leap.soledad.common.document import SoledadDocument
-from leap.soledad.common.ddocs import ensure_ddocs_on_remote_db
+from leap.soledad.common.ddocs import ensure_ddocs_on_db
logger = logging.getLogger(__name__)
@@ -193,11 +183,11 @@ class CouchDatabase(CommonBackend):
server[dbname]
except ResourceNotFound:
if not create:
- raise DatabaseDoesNotExist()
+ raise errors.DatabaseDoesNotExist()
return cls(url, dbname)
def __init__(self, url, dbname, replica_uid=None, full_commit=True,
- session=None):
+ session=None, ensure_ddocs=False):
"""
Create a new Couch data container.
@@ -230,7 +220,9 @@ class CouchDatabase(CommonBackend):
except ResourceNotFound:
self._server.create(self._dbname)
self._database = self._server[self._dbname]
- self._initialize(replica_uid or uuid.uuid4().hex)
+ self._set_replica_uid(replica_uid or uuid.uuid4().hex)
+ if ensure_ddocs:
+ ensure_ddocs_on_db(self._database)
def get_sync_target(self):
"""
@@ -270,25 +262,12 @@ class CouchDatabase(CommonBackend):
"""
try:
doc = self._database['u1db_config']
+ doc['replica_uid'] = replica_uid
except ResourceNotFound:
doc = {
'_id': 'u1db_config',
'replica_uid': replica_uid,
}
- self._database.save(doc)
-
- def _ensure_design_docs(self):
- """
- Ensure that the design docs have been created.
- """
- if self._is_initialized():
- return
- self._initialize()
-
- def _set_replica_uid(self, replica_uid):
- """Force the replica_uid to be set."""
- doc = self._database['u1db_config']
- doc['replica_uid'] = replica_uid
self._database.save(doc)
self._real_replica_uid = replica_uid
diff --git a/common/src/leap/soledad/common/ddocs/__init__.py b/common/src/leap/soledad/common/ddocs/__init__.py
index c2f78e18..389bdff9 100644
--- a/common/src/leap/soledad/common/ddocs/__init__.py
+++ b/common/src/leap/soledad/common/ddocs/__init__.py
@@ -37,7 +37,7 @@ logger = logging.getLogger(__name__)
prefix = dirname(realpath(__file__))
-def ensure_ddocs_on_remote_db(db, prefix=prefix):
+def ensure_ddocs_on_db(db, prefix=prefix):
"""
Ensure that the design documents in C{db} contain.
@@ -118,7 +118,7 @@ def build_ddocs(prefix=prefix):
except IOError:
pass
ddocs[ddoc]['views'][view] = {}
-
+
if mapfun is not None:
ddocs[ddoc]['views'][view]['map'] = mapfun
if reducefun is not None:
diff --git a/common/src/leap/soledad/common/tests/couchdb.ini.template b/common/src/leap/soledad/common/tests/couchdb.ini.template
index 7d0316f0..217ae201 100644
--- a/common/src/leap/soledad/common/tests/couchdb.ini.template
+++ b/common/src/leap/soledad/common/tests/couchdb.ini.template
@@ -74,6 +74,8 @@ use_users_db = false
[query_servers]
; javascript = %(tempdir)s/server/main.js
+javascript = /usr/bin/couchjs /usr/share/couchdb/server/main.js
+coffeescript = /usr/bin/couchjs /usr/share/couchdb/server/main-coffee.js
; Changing reduce_limit to false will disable reduce_limit.
@@ -219,4 +221,4 @@ min_file_size = 131072
;[admins]
;testuser = -hashed-f50a252c12615697c5ed24ec5cd56b05d66fe91e,b05471ba260132953930cf9f97f327f5
-; pass for above user is 'testpass' \ No newline at end of file
+; pass for above user is 'testpass'
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index bdef4e0d..a181c6cb 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -25,6 +25,9 @@ import copy
import shutil
from base64 import b64decode
+from couchdb.client import Server
+from u1db import errors
+
from leap.common.files import mkdir_p
from leap.soledad.common.tests import u1db_tests as tests
@@ -147,7 +150,7 @@ class TestCouchBackendImpl(CouchDBTestCase):
def test__allocate_doc_id(self):
db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
- 'u1db_tests')
+ 'u1db_tests', ensure_ddocs=True)
doc_id1 = db._allocate_doc_id()
self.assertTrue(doc_id1.startswith('D-'))
self.assertEqual(34, len(doc_id1))
@@ -162,32 +165,51 @@ class TestCouchBackendImpl(CouchDBTestCase):
def make_couch_database_for_test(test, replica_uid):
port = str(test.wrapper.port)
return couch.CouchDatabase('http://localhost:' + port, replica_uid,
- replica_uid=replica_uid or 'test')
+ replica_uid=replica_uid or 'test',
+ ensure_ddocs=True)
def copy_couch_database_for_test(test, db):
port = str(test.wrapper.port)
- new_db = couch.CouchDatabase('http://localhost:' + port,
- db._replica_uid + '_copy',
+ couch_url = 'http://localhost:' + port
+ new_dbname = db._replica_uid + '_copy'
+ new_db = couch.CouchDatabase(couch_url,
+ new_dbname,
replica_uid=db._replica_uid or 'test')
- gen, docs = db.get_all_docs(include_deleted=True)
- for doc in docs:
- new_db._put_doc(doc)
- new_db._transaction_log = copy.deepcopy(db._transaction_log)
- new_db._conflicts = copy.deepcopy(db._conflicts)
- new_db._other_generations = copy.deepcopy(db._other_generations)
- new_db._indexes = copy.deepcopy(db._indexes)
- # save u1db data on couch
- for key in new_db.U1DB_DATA_KEYS:
- doc_id = '%s%s' % (new_db.U1DB_DATA_DOC_ID_PREFIX, key)
- doc = new_db._get_doc(doc_id)
- doc.content = {'content': getattr(new_db, key)}
- new_db._put_doc(doc)
+ # copy all docs
+ old_couch_db = Server(couch_url)[db._replica_uid]
+ new_couch_db = Server(couch_url)[new_dbname]
+ for doc_id in old_couch_db:
+ doc = old_couch_db.get(doc_id)
+ # copy design docs
+ if ('u1db_rev' not in doc):
+ new_couch_db.save(doc)
+ # copy u1db docs
+ else:
+ new_doc = {
+ '_id': doc['_id'],
+ 'u1db_transactions': doc['u1db_transactions'],
+ 'u1db_rev': doc['u1db_rev']
+ }
+ attachments = []
+ if ('u1db_conflicts' in doc):
+ new_doc['u1db_conflicts'] = doc['u1db_conflicts']
+ for c_rev in doc['u1db_conflicts']:
+ attachments.append('u1db_conflict_%s' % c_rev)
+ new_couch_db.save(new_doc)
+ # save conflict data
+ attachments.append('u1db_content')
+ for att_name in attachments:
+ att = old_couch_db.get_attachment(doc_id, att_name)
+ if (att is not None):
+ new_couch_db.put_attachment(new_doc, att,
+ filename=att_name)
return new_db
def make_document_for_test(test, doc_id, rev, content, has_conflicts=False):
- return couch.CouchDocument(doc_id, rev, content, has_conflicts=has_conflicts)
+ return couch.CouchDocument(
+ doc_id, rev, content, has_conflicts=has_conflicts)
COUCH_SCENARIOS = [
@@ -245,9 +267,8 @@ class CouchWithConflictsTests(
test_backends.LocalDatabaseWithConflictsTests.tearDown(self)
-# Notice: the CouchDB backend is currently used for storing encrypted data in
-# the server, so indexing makes no sense. Thus, we ignore index testing for
-# now.
+# Notice: the CouchDB backend does not have indexing capabilities, but we
+# added in memory indexing for tests only.
class CouchIndexTests(test_backends.DatabaseIndexTests, CouchDBTestCase):
@@ -310,6 +331,89 @@ class CouchDatabaseSyncTargetTests(test_sync.DatabaseSyncTargetTests,
[(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]})
+# The following tests need that the database have an index, so we fake one.
+old_class = couch.CouchDatabase
+
+from u1db.backends.inmemory import InMemoryIndex
+
+
+class IndexedCouchDatabase(couch.CouchDatabase):
+
+ def __init__(self, url, dbname, replica_uid=None, full_commit=True,
+ session=None, ensure_ddocs=True):
+ old_class.__init__(self, url, dbname, replica_uid, full_commit,
+ session, ensure_ddocs=True)
+ self._indexes = {}
+
+ def _put_doc(self, old_doc, doc):
+ for index in self._indexes.itervalues():
+ if old_doc is not None and not old_doc.is_tombstone():
+ index.remove_json(old_doc.doc_id, old_doc.get_json())
+ if not doc.is_tombstone():
+ index.add_json(doc.doc_id, doc.get_json())
+ old_class._put_doc(self, old_doc, doc)
+
+ def create_index(self, index_name, *index_expressions):
+ if index_name in self._indexes:
+ if self._indexes[index_name]._definition == list(
+ index_expressions):
+ return
+ raise errors.IndexNameTakenError
+ index = InMemoryIndex(index_name, list(index_expressions))
+ _, all_docs = self.get_all_docs()
+ for doc in all_docs:
+ index.add_json(doc.doc_id, doc.get_json())
+ self._indexes[index_name] = index
+
+ def delete_index(self, index_name):
+ del self._indexes[index_name]
+
+ def list_indexes(self):
+ definitions = []
+ for idx in self._indexes.itervalues():
+ definitions.append((idx._name, idx._definition))
+ return definitions
+
+ def get_from_index(self, index_name, *key_values):
+ try:
+ index = self._indexes[index_name]
+ except KeyError:
+ raise errors.IndexDoesNotExist
+ doc_ids = index.lookup(key_values)
+ result = []
+ for doc_id in doc_ids:
+ result.append(self._get_doc(doc_id, check_for_conflicts=True))
+ return result
+
+ def get_range_from_index(self, index_name, start_value=None,
+ end_value=None):
+ """Return all documents with key values in the specified range."""
+ try:
+ index = self._indexes[index_name]
+ except KeyError:
+ raise errors.IndexDoesNotExist
+ if isinstance(start_value, basestring):
+ start_value = (start_value,)
+ if isinstance(end_value, basestring):
+ end_value = (end_value,)
+ doc_ids = index.lookup_range(start_value, end_value)
+ result = []
+ for doc_id in doc_ids:
+ result.append(self._get_doc(doc_id, check_for_conflicts=True))
+ return result
+
+ def get_index_keys(self, index_name):
+ try:
+ index = self._indexes[index_name]
+ except KeyError:
+ raise errors.IndexDoesNotExist
+ keys = index.keys()
+ # XXX inefficiency warning
+ return list(set([tuple(key.split('\x01')) for key in keys]))
+
+
+couch.CouchDatabase = IndexedCouchDatabase
+
sync_scenarios = []
for name, scenario in COUCH_SCENARIOS:
scenario = dict(scenario)
@@ -343,98 +447,4 @@ class CouchDatabaseSyncTests(test_sync.DatabaseSyncTests, CouchDBTestCase):
test_sync.DatabaseSyncTests.tearDown(self)
-#-----------------------------------------------------------------------------
-# The following tests test extra functionality introduced by our backends
-#-----------------------------------------------------------------------------
-
-class CouchDatabaseStorageTests(CouchDBTestCase):
-
- def _listify(self, l):
- if type(l) is dict:
- return {
- self._listify(a): self._listify(b) for a, b in l.iteritems()}
- if hasattr(l, '__iter__'):
- return [self._listify(i) for i in l]
- return l
-
- def _fetch_u1db_data(self, db, key):
- doc = db._get_doc("%s%s" % (db.U1DB_DATA_DOC_ID_PREFIX, key))
- return doc.content['content']
-
- def test_transaction_log_storage_after_put(self):
- db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
- 'u1db_tests')
- db.create_doc({'simple': 'doc'})
- content = self._fetch_u1db_data(db, db.U1DB_TRANSACTION_LOG_KEY)
- self.assertEqual(
- self._listify(db._transaction_log),
- self._listify(content))
-
- def test_conflict_log_storage_after_put_if_newer(self):
- db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
- 'u1db_tests')
- doc = db.create_doc({'simple': 'doc'})
- doc.set_json(nested_doc)
- doc.rev = db._replica_uid + ':2'
- db._force_doc_sync_conflict(doc)
- content = self._fetch_u1db_data(db, db.U1DB_CONFLICTS_KEY)
- self.assertEqual(
- self._listify(db._conflicts),
- self._listify(content))
-
- def test_other_gens_storage_after_set(self):
- db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
- 'u1db_tests')
- doc = db.create_doc({'simple': 'doc'})
- db._set_replica_gen_and_trans_id('a', 'b', 'c')
- content = self._fetch_u1db_data(db, db.U1DB_OTHER_GENERATIONS_KEY)
- self.assertEqual(
- self._listify(db._other_generations),
- self._listify(content))
-
- def test_index_storage_after_create(self):
- db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
- 'u1db_tests')
- doc = db.create_doc({'name': 'john'})
- db.create_index('myindex', 'name')
- content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY)
- myind = db._indexes['myindex']
- index = {
- 'myindex': {
- 'definition': myind._definition,
- 'name': myind._name,
- 'values': myind._values,
- }
- }
- self.assertEqual(
- self._listify(index),
- self._listify(content))
-
- def test_index_storage_after_delete(self):
- db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
- 'u1db_tests')
- doc = db.create_doc({'name': 'john'})
- db.create_index('myindex', 'name')
- db.create_index('myindex2', 'name')
- db.delete_index('myindex')
- content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY)
- myind = db._indexes['myindex2']
- index = {
- 'myindex2': {
- 'definition': myind._definition,
- 'name': myind._name,
- 'values': myind._values,
- }
- }
- self.assertEqual(
- self._listify(index),
- self._listify(content))
-
- def test_replica_uid_storage_after_db_creation(self):
- db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
- 'u1db_tests')
- content = self._fetch_u1db_data(db, db.U1DB_REPLICA_UID_KEY)
- self.assertEqual(db._replica_uid, content)
-
-
load_tests = tests.load_with_scenarios
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-01 04:43:48 +0000