Fix couch tests to reflect remodelling.

4 files changed, 141 insertions, 150 deletions

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 4caaf48f..b9e699f3 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -18,35 +18,25 @@
 
 """A U1DB backend that uses CouchDB as its persistence layer."""
 
-import os
 import simplejson as json
-import sys
-import time
+import re
 import uuid
 import logging
 import binascii
 import socket
 
 
+from couchdb.client import Server
+from couchdb.http import ResourceNotFound, Unauthorized
+from u1db import errors, query_parser, vectorclock
 from u1db.backends import CommonBackend, CommonSyncTarget
 from u1db.remote import http_app
 from u1db.remote.server_state import ServerState
-from u1db import (
-    errors,
-    query_parser,
-    vectorclock,
-)
-from couchdb.client import (
-    Server,
-    Document as CouchDocument,
-    _doc_resource,
-)
-from couchdb.http import ResourceNotFound, Unauthorized
 
 
 from leap.soledad.common import USER_DB_PREFIX
 from leap.soledad.common.document import SoledadDocument
-from leap.soledad.common.ddocs import ensure_ddocs_on_remote_db
+from leap.soledad.common.ddocs import ensure_ddocs_on_db
 
 
 logger = logging.getLogger(__name__)
@@ -193,11 +183,11 @@ class CouchDatabase(CommonBackend):
             server[dbname]
         except ResourceNotFound:
             if not create:
-                raise DatabaseDoesNotExist()
+                raise errors.DatabaseDoesNotExist()
         return cls(url, dbname)
 
     def __init__(self, url, dbname, replica_uid=None, full_commit=True,
-                 session=None):
+                 session=None, ensure_ddocs=False):
         """
         Create a new Couch data container.
 
@@ -230,7 +220,9 @@ class CouchDatabase(CommonBackend):
         except ResourceNotFound:
             self._server.create(self._dbname)
             self._database = self._server[self._dbname]
-        self._initialize(replica_uid or uuid.uuid4().hex)
+        self._set_replica_uid(replica_uid or uuid.uuid4().hex)
+        if ensure_ddocs:
+            ensure_ddocs_on_db(self._database)
 
     def get_sync_target(self):
         """
@@ -270,25 +262,12 @@ class CouchDatabase(CommonBackend):
         """
         try:
             doc = self._database['u1db_config']
+            doc['replica_uid'] = replica_uid
         except ResourceNotFound:
             doc = {
                 '_id': 'u1db_config',
                 'replica_uid': replica_uid,
             }
-            self._database.save(doc)
-
-    def _ensure_design_docs(self):
-        """
-        Ensure that the design docs have been created.
-        """
-        if self._is_initialized():
-            return
-        self._initialize()
-
-    def _set_replica_uid(self, replica_uid):
-        """Force the replica_uid to be set."""
-        doc = self._database['u1db_config']
-        doc['replica_uid'] = replica_uid
         self._database.save(doc)
         self._real_replica_uid = replica_uid
 
diff --git a/common/src/leap/soledad/common/ddocs/__init__.py b/common/src/leap/soledad/common/ddocs/__init__.py
index c2f78e18..389bdff9 100644
--- a/common/src/leap/soledad/common/ddocs/__init__.py
+++ b/common/src/leap/soledad/common/ddocs/__init__.py
@@ -37,7 +37,7 @@ logger = logging.getLogger(__name__)
 prefix = dirname(realpath(__file__))
 
 
-def ensure_ddocs_on_remote_db(db, prefix=prefix):
+def ensure_ddocs_on_db(db, prefix=prefix):
     """
     Ensure that the design documents in C{db} contain.
 
@@ -118,7 +118,7 @@ def build_ddocs(prefix=prefix):
                         except IOError:
                             pass
                         ddocs[ddoc]['views'][view] = {}
-                        
+
                         if mapfun is not None:
                             ddocs[ddoc]['views'][view]['map'] = mapfun
                         if reducefun is not None:
diff --git a/common/src/leap/soledad/common/tests/couchdb.ini.template b/common/src/leap/soledad/common/tests/couchdb.ini.template
index 7d0316f0..217ae201 100644
--- a/common/src/leap/soledad/common/tests/couchdb.ini.template
+++ b/common/src/leap/soledad/common/tests/couchdb.ini.template
@@ -74,6 +74,8 @@ use_users_db = false
 
 [query_servers]
 ; javascript = %(tempdir)s/server/main.js
+javascript = /usr/bin/couchjs /usr/share/couchdb/server/main.js
+coffeescript = /usr/bin/couchjs /usr/share/couchdb/server/main-coffee.js
 
 
 ; Changing reduce_limit to false will disable reduce_limit.
@@ -219,4 +221,4 @@ min_file_size = 131072
 
 ;[admins]
 ;testuser = -hashed-f50a252c12615697c5ed24ec5cd56b05d66fe91e,b05471ba260132953930cf9f97f327f5
-; pass for above user is 'testpass'
\ No newline at end of file
+; pass for above user is 'testpass'
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index bdef4e0d..a181c6cb 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -25,6 +25,9 @@ import copy
 import shutil
 from base64 import b64decode
 
+from couchdb.client import Server
+from u1db import errors
+
 from leap.common.files import mkdir_p
 
 from leap.soledad.common.tests import u1db_tests as tests
@@ -147,7 +150,7 @@ class TestCouchBackendImpl(CouchDBTestCase):
 
     def test__allocate_doc_id(self):
         db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
-                                 'u1db_tests')
+                                 'u1db_tests', ensure_ddocs=True)
         doc_id1 = db._allocate_doc_id()
         self.assertTrue(doc_id1.startswith('D-'))
         self.assertEqual(34, len(doc_id1))
@@ -162,32 +165,51 @@ class TestCouchBackendImpl(CouchDBTestCase):
 def make_couch_database_for_test(test, replica_uid):
     port = str(test.wrapper.port)
     return couch.CouchDatabase('http://localhost:' + port, replica_uid,
-                               replica_uid=replica_uid or 'test')
+                               replica_uid=replica_uid or 'test',
+                               ensure_ddocs=True)
 
 
 def copy_couch_database_for_test(test, db):
     port = str(test.wrapper.port)
-    new_db = couch.CouchDatabase('http://localhost:' + port,
-                                 db._replica_uid + '_copy',
+    couch_url = 'http://localhost:' + port
+    new_dbname = db._replica_uid + '_copy'
+    new_db = couch.CouchDatabase(couch_url,
+                                 new_dbname,
                                  replica_uid=db._replica_uid or 'test')
-    gen, docs = db.get_all_docs(include_deleted=True)
-    for doc in docs:
-        new_db._put_doc(doc)
-    new_db._transaction_log = copy.deepcopy(db._transaction_log)
-    new_db._conflicts = copy.deepcopy(db._conflicts)
-    new_db._other_generations = copy.deepcopy(db._other_generations)
-    new_db._indexes = copy.deepcopy(db._indexes)
-    # save u1db data on couch
-    for key in new_db.U1DB_DATA_KEYS:
-        doc_id = '%s%s' % (new_db.U1DB_DATA_DOC_ID_PREFIX, key)
-        doc = new_db._get_doc(doc_id)
-        doc.content = {'content': getattr(new_db, key)}
-        new_db._put_doc(doc)
+    # copy all docs
+    old_couch_db = Server(couch_url)[db._replica_uid]
+    new_couch_db = Server(couch_url)[new_dbname]
+    for doc_id in old_couch_db:
+        doc = old_couch_db.get(doc_id)
+        # copy design docs
+        if ('u1db_rev' not in doc):
+            new_couch_db.save(doc)
+        # copy u1db docs
+        else:
+            new_doc = {
+                '_id': doc['_id'],
+                'u1db_transactions': doc['u1db_transactions'],
+                'u1db_rev': doc['u1db_rev']
+            }
+            attachments = []
+            if ('u1db_conflicts' in doc):
+                new_doc['u1db_conflicts'] = doc['u1db_conflicts']
+                for c_rev in doc['u1db_conflicts']:
+                    attachments.append('u1db_conflict_%s' % c_rev)
+            new_couch_db.save(new_doc)
+            # save conflict data
+            attachments.append('u1db_content')
+            for att_name in attachments:
+                att = old_couch_db.get_attachment(doc_id, att_name)
+                if (att is not None):
+                    new_couch_db.put_attachment(new_doc, att,
+                                                filename=att_name)
     return new_db
 
 
 def make_document_for_test(test, doc_id, rev, content, has_conflicts=False):
-    return couch.CouchDocument(doc_id, rev, content, has_conflicts=has_conflicts)
+    return couch.CouchDocument(
+        doc_id, rev, content, has_conflicts=has_conflicts)
 
 
 COUCH_SCENARIOS = [
@@ -245,9 +267,8 @@ class CouchWithConflictsTests(
         test_backends.LocalDatabaseWithConflictsTests.tearDown(self)
 
 
-# Notice: the CouchDB backend is currently used for storing encrypted data in
-# the server, so indexing makes no sense. Thus, we ignore index testing for
-# now.
+# Notice: the CouchDB backend does not have indexing capabilities, but we
+# added in memory indexing for tests only.
 
 class CouchIndexTests(test_backends.DatabaseIndexTests, CouchDBTestCase):
 
@@ -310,6 +331,89 @@ class CouchDatabaseSyncTargetTests(test_sync.DatabaseSyncTargetTests,
                  [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]})
 
 
+# The following tests need that the database have an index, so we fake one.
+old_class = couch.CouchDatabase
+
+from u1db.backends.inmemory import InMemoryIndex
+
+
+class IndexedCouchDatabase(couch.CouchDatabase):
+
+    def __init__(self, url, dbname, replica_uid=None, full_commit=True,
+                     session=None, ensure_ddocs=True):
+        old_class.__init__(self, url, dbname, replica_uid, full_commit,
+                           session, ensure_ddocs=True)
+        self._indexes = {}
+
+    def _put_doc(self, old_doc, doc):
+        for index in self._indexes.itervalues():
+            if old_doc is not None and not old_doc.is_tombstone():
+                index.remove_json(old_doc.doc_id, old_doc.get_json())
+            if not doc.is_tombstone():
+                index.add_json(doc.doc_id, doc.get_json())
+        old_class._put_doc(self, old_doc, doc)
+
+    def create_index(self, index_name, *index_expressions):
+        if index_name in self._indexes:
+            if self._indexes[index_name]._definition == list(
+                    index_expressions):
+                return
+            raise errors.IndexNameTakenError
+        index = InMemoryIndex(index_name, list(index_expressions))
+        _, all_docs = self.get_all_docs()
+        for doc in all_docs:
+            index.add_json(doc.doc_id, doc.get_json())
+        self._indexes[index_name] = index
+
+    def delete_index(self, index_name):
+        del self._indexes[index_name]
+
+    def list_indexes(self):
+        definitions = []
+        for idx in self._indexes.itervalues():
+            definitions.append((idx._name, idx._definition))
+        return definitions
+
+    def get_from_index(self, index_name, *key_values):
+        try:
+            index = self._indexes[index_name]
+        except KeyError:
+            raise errors.IndexDoesNotExist
+        doc_ids = index.lookup(key_values)
+        result = []
+        for doc_id in doc_ids:
+            result.append(self._get_doc(doc_id, check_for_conflicts=True))
+        return result
+
+    def get_range_from_index(self, index_name, start_value=None,
+                             end_value=None):
+        """Return all documents with key values in the specified range."""
+        try:
+            index = self._indexes[index_name]
+        except KeyError:
+            raise errors.IndexDoesNotExist
+        if isinstance(start_value, basestring):
+            start_value = (start_value,)
+        if isinstance(end_value, basestring):
+            end_value = (end_value,)
+        doc_ids = index.lookup_range(start_value, end_value)
+        result = []
+        for doc_id in doc_ids:
+            result.append(self._get_doc(doc_id, check_for_conflicts=True))
+        return result
+
+    def get_index_keys(self, index_name):
+        try:
+            index = self._indexes[index_name]
+        except KeyError:
+            raise errors.IndexDoesNotExist
+        keys = index.keys()
+        # XXX inefficiency warning
+        return list(set([tuple(key.split('\x01')) for key in keys]))
+
+
+couch.CouchDatabase = IndexedCouchDatabase
+
 sync_scenarios = []
 for name, scenario in COUCH_SCENARIOS:
     scenario = dict(scenario)
@@ -343,98 +447,4 @@ class CouchDatabaseSyncTests(test_sync.DatabaseSyncTests, CouchDBTestCase):
         test_sync.DatabaseSyncTests.tearDown(self)
 
 
-#-----------------------------------------------------------------------------
-# The following tests test extra functionality introduced by our backends
-#-----------------------------------------------------------------------------
-
-class CouchDatabaseStorageTests(CouchDBTestCase):
-
-    def _listify(self, l):
-        if type(l) is dict:
-            return {
-                self._listify(a): self._listify(b) for a, b in l.iteritems()}
-        if hasattr(l, '__iter__'):
-            return [self._listify(i) for i in l]
-        return l
-
-    def _fetch_u1db_data(self, db, key):
-        doc = db._get_doc("%s%s" % (db.U1DB_DATA_DOC_ID_PREFIX, key))
-        return doc.content['content']
-
-    def test_transaction_log_storage_after_put(self):
-        db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
-                                 'u1db_tests')
-        db.create_doc({'simple': 'doc'})
-        content = self._fetch_u1db_data(db, db.U1DB_TRANSACTION_LOG_KEY)
-        self.assertEqual(
-            self._listify(db._transaction_log),
-            self._listify(content))
-
-    def test_conflict_log_storage_after_put_if_newer(self):
-        db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
-                                 'u1db_tests')
-        doc = db.create_doc({'simple': 'doc'})
-        doc.set_json(nested_doc)
-        doc.rev = db._replica_uid + ':2'
-        db._force_doc_sync_conflict(doc)
-        content = self._fetch_u1db_data(db, db.U1DB_CONFLICTS_KEY)
-        self.assertEqual(
-            self._listify(db._conflicts),
-            self._listify(content))
-
-    def test_other_gens_storage_after_set(self):
-        db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
-                                 'u1db_tests')
-        doc = db.create_doc({'simple': 'doc'})
-        db._set_replica_gen_and_trans_id('a', 'b', 'c')
-        content = self._fetch_u1db_data(db, db.U1DB_OTHER_GENERATIONS_KEY)
-        self.assertEqual(
-            self._listify(db._other_generations),
-            self._listify(content))
-
-    def test_index_storage_after_create(self):
-        db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
-                                 'u1db_tests')
-        doc = db.create_doc({'name': 'john'})
-        db.create_index('myindex', 'name')
-        content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY)
-        myind = db._indexes['myindex']
-        index = {
-            'myindex': {
-                'definition': myind._definition,
-                'name': myind._name,
-                'values': myind._values,
-            }
-        }
-        self.assertEqual(
-            self._listify(index),
-            self._listify(content))
-
-    def test_index_storage_after_delete(self):
-        db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
-                                 'u1db_tests')
-        doc = db.create_doc({'name': 'john'})
-        db.create_index('myindex', 'name')
-        db.create_index('myindex2', 'name')
-        db.delete_index('myindex')
-        content = self._fetch_u1db_data(db, db.U1DB_INDEXES_KEY)
-        myind = db._indexes['myindex2']
-        index = {
-            'myindex2': {
-                'definition': myind._definition,
-                'name': myind._name,
-                'values': myind._values,
-            }
-        }
-        self.assertEqual(
-            self._listify(index),
-            self._listify(content))
-
-    def test_replica_uid_storage_after_db_creation(self):
-        db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
-                                 'u1db_tests')
-        content = self._fetch_u1db_data(db, db.U1DB_REPLICA_UID_KEY)
-        self.assertEqual(db._replica_uid, content)
-
-
 load_tests = tests.load_with_scenarios