summaryrefslogtreecommitdiff
path: root/soledad_server
diff options
context:
space:
mode:
authordrebs <drebs@leap.se>2013-08-19 10:13:51 -0300
committerdrebs <drebs@leap.se>2013-08-21 14:40:33 -0300
commit11ac38cb021e97cca77df2e9cf15f9136b24fc43 (patch)
tree2f4e3e004f57ac3436d8258937bf2145ca3de576 /soledad_server
parentf424e2fee278bd9f3a9fb838025db7d4c1bb44bb (diff)
Split soledad into common, client and server.
Diffstat (limited to 'soledad_server')
-rw-r--r--soledad_server/pkg/soledad68
-rw-r--r--soledad_server/setup.py80
-rw-r--r--soledad_server/src/leap/__init__.py6
-rw-r--r--soledad_server/src/leap/soledad_server/__init__.py125
-rw-r--r--soledad_server/src/leap/soledad_server/auth.py431
-rw-r--r--soledad_server/src/leap/soledad_server/couch.py480
-rw-r--r--soledad_server/src/leap/soledad_server/objectstore.py296
7 files changed, 0 insertions, 1486 deletions
diff --git a/soledad_server/pkg/soledad b/soledad_server/pkg/soledad
deleted file mode 100644
index c233731e..00000000
--- a/soledad_server/pkg/soledad
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/sh
-### BEGIN INIT INFO
-# Provides: soledad
-# Required-Start: $network $named $remote_fs $syslog $time
-# Required-Stop: $network $named $remote_fs $syslog
-# Default-Start: 2 3 4 5
-# Default-Stop: 0 1 6
-# Short-Description: Start soledad daemon at boot time
-# Description: Synchronization of locally encrypted data among devices
-### END INIT INFO
-
-PATH=/sbin:/bin:/usr/sbin:/usr/bin
-PIDFILE=/var/run/soledad.pid
-RUNDIR=/var/lib/soledad/
-OBJ=leap.soledad_server.application
-LOGFILE=/var/log/soledad.log
-HTTPS_PORT=2424
-CERT_PATH=/etc/leap/soledad-server.pem
-PRIVKEY_PATH=/etc/leap/soledad-server.key
-TWISTD_PATH=/usr/bin/twistd
-HOME=/var/lib/soledad/
-
-[ -r /etc/default/soledad ] && . /etc/default/soledad
-
-test -r /etc/leap/ || exit 0
-
-. /lib/lsb/init-functions
-
-
-case "$1" in
- start)
- echo -n "Starting soledad: twistd"
- start-stop-daemon --start --quiet --exec $TWISTD_PATH -- \
- --pidfile=$PIDFILE \
- --logfile=$LOGFILE \
- web \
- --wsgi=$OBJ \
- --port=ssl:$HTTPS_PORT:privateKey=$PRIVKEY_PATH:certKey=$CERT_PATH
- echo "."
- ;;
-
- stop)
- echo -n "Stopping soledad: twistd"
- start-stop-daemon --stop --quiet \
- --pidfile $PIDFILE
- echo "."
- ;;
-
- restart)
- $0 stop
- $0 start
- ;;
-
- force-reload)
- $0 restart
- ;;
-
- status)
- status_of_proc -p $PIDFILE $TWISTD_PATH soledad && exit 0 || exit $?
- ;;
-
- *)
- echo "Usage: /etc/init.d/soledad {start|stop|restart|force-reload|status}" >&2
- exit 1
- ;;
-esac
-
-exit 0
diff --git a/soledad_server/setup.py b/soledad_server/setup.py
deleted file mode 100644
index 6229cf62..00000000
--- a/soledad_server/setup.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# -*- coding: utf-8 -*-
-# setup.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import os
-from setuptools import (
- setup,
- find_packages
-)
-
-
-install_requirements = [
- 'configparser',
- 'couchdb',
- 'simplejson',
- 'twisted>=12.0.0', # TODO: maybe we just want twisted-web?
- 'oauth', # this is not strictly needed by us, but we need it
- # until u1db adds it to its release as a dep.
- 'u1db',
- 'six==1.1.0',
- 'routes',
- 'PyOpenSSL',
- 'leap.soledad>=0.3.0',
-]
-
-
-if os.environ.get('VIRTUAL_ENV', None):
- data_files = None
-else:
- # XXX this should go only for linux/mac
- data_files = [("/etc/init.d/", ["pkg/soledad"])]
-
-trove_classifiers = (
- "Development Status :: 3 - Alpha",
- "Intended Audience :: Developers",
- "License :: OSI Approved :: "
- "GNU General Public License v3 or later (GPLv3+)",
- "Environment :: Console",
- "Operating System :: OS Independent",
- "Operating System :: POSIX",
- "Programming Language :: Python :: 2.6",
- "Programming Language :: Python :: 2.7",
- "Topic :: Database :: Front-Ends",
- "Topic :: Software Development :: Libraries :: Python Modules"
-)
-
-setup(
- name='leap.soledad_server',
- version='0.3.0',
- url='https://leap.se/',
- license='GPLv3+',
- description='Synchronization of locally encrypted data among devices.',
- author='The LEAP Encryption Access Project',
- author_email='info@leap.se',
- long_description=(
- "Soledad is the part of LEAP that allows application data to be "
- "securely shared among devices. It provides, to other parts of the "
- "LEAP client, an API for data storage and sync."
- ),
- namespace_packages=["leap"],
- packages=find_packages('src'),
- package_dir={'': 'src'},
- install_requires=install_requirements,
- data_files=data_files,
- classifiers=trove_classifiers,
-)
diff --git a/soledad_server/src/leap/__init__.py b/soledad_server/src/leap/__init__.py
deleted file mode 100644
index f48ad105..00000000
--- a/soledad_server/src/leap/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
-try:
- __import__('pkg_resources').declare_namespace(__name__)
-except ImportError:
- from pkgutil import extend_path
- __path__ = extend_path(__path__, __name__)
diff --git a/soledad_server/src/leap/soledad_server/__init__.py b/soledad_server/src/leap/soledad_server/__init__.py
deleted file mode 100644
index 18a0546e..00000000
--- a/soledad_server/src/leap/soledad_server/__init__.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# -*- coding: utf-8 -*-
-# server.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-A U1DB server that stores data using CouchDB as its persistence layer.
-
-This should be run with:
- twistd -n web --wsgi=leap.soledad_server.application --port=2424
-"""
-
-import configparser
-
-from u1db.remote import http_app
-
-
-# Keep OpenSSL's tsafe before importing Twisted submodules so we can put
-# it back if Twisted==12.0.0 messes with it.
-from OpenSSL import tsafe
-old_tsafe = tsafe
-
-from twisted.web.wsgi import WSGIResource
-from twisted.internet import reactor
-from twisted import version
-if version.base() == "12.0.0":
- # Put OpenSSL's tsafe back into place. This can probably be removed if we
- # come to use Twisted>=12.3.0.
- import sys
- sys.modules['OpenSSL.tsafe'] = old_tsafe
-
-
-from leap.soledad_server.auth import SoledadTokenAuthMiddleware
-from leap.soledad_server.couch import CouchServerState
-
-
-#-----------------------------------------------------------------------------
-# Soledad WSGI application
-#-----------------------------------------------------------------------------
-
-class SoledadApp(http_app.HTTPApp):
- """
- Soledad WSGI application
- """
-
- SHARED_DB_NAME = 'shared'
- """
- The name of the shared database that holds user's encrypted secrets.
- """
-
- USER_DB_PREFIX = 'user-'
- """
- The string prefix of users' databases.
- """
-
- def __call__(self, environ, start_response):
- """
- Handle a WSGI call to the Soledad application.
-
- @param environ: Dictionary containing CGI variables.
- @type environ: dict
- @param start_response: Callable of the form start_response(status,
- response_headers, exc_info=None).
- @type start_response: callable
-
- @return: HTTP application results.
- @rtype: list
- """
- # ensure the shared database exists
- self.state.ensure_database(self.SHARED_DB_NAME)
- return http_app.HTTPApp.__call__(self, environ, start_response)
-
-
-#-----------------------------------------------------------------------------
-# Auxiliary functions
-#-----------------------------------------------------------------------------
-
-def load_configuration(file_path):
- """
- Load server configuration from file.
-
- @param file_path: The path to the configuration file.
- @type file_path: str
-
- @return: A dictionary with the configuration.
- @rtype: dict
- """
- conf = {
- 'couch_url': 'http://localhost:5984',
- }
- config = configparser.ConfigParser()
- config.read(file_path)
- if 'soledad-server' in config:
- for key in conf:
- if key in config['soledad-server']:
- conf[key] = config['soledad-server'][key]
- # TODO: implement basic parsing/sanitization of options comming from
- # config file.
- return conf
-
-
-#-----------------------------------------------------------------------------
-# Run as Twisted WSGI Resource
-#-----------------------------------------------------------------------------
-
-conf = load_configuration('/etc/leap/soledad-server.conf')
-state = CouchServerState(conf['couch_url'])
-
-# WSGI application that may be used by `twistd -web`
-application = SoledadTokenAuthMiddleware(SoledadApp(state))
-
-resource = WSGIResource(reactor, reactor.getThreadPool(), application) \ No newline at end of file
diff --git a/soledad_server/src/leap/soledad_server/auth.py b/soledad_server/src/leap/soledad_server/auth.py
deleted file mode 100644
index 3bcfcf04..00000000
--- a/soledad_server/src/leap/soledad_server/auth.py
+++ /dev/null
@@ -1,431 +0,0 @@
-# -*- coding: utf-8 -*-
-# auth.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-Authentication facilities for Soledad Server.
-"""
-
-
-import httplib
-import simplejson as json
-
-
-from u1db import DBNAME_CONSTRAINTS
-from abc import ABCMeta, abstractmethod
-from routes.mapper import Mapper
-from couchdb.client import Server
-from twisted.python import log
-
-
-#-----------------------------------------------------------------------------
-# Authentication
-#-----------------------------------------------------------------------------
-
-class Unauthorized(Exception):
- """
- User authentication failed.
- """
-
-
-class URLToAuthorization(object):
- """
- Verify if actions can be performed by a user.
- """
-
- HTTP_METHOD_GET = 'GET'
- HTTP_METHOD_PUT = 'PUT'
- HTTP_METHOD_DELETE = 'DELETE'
- HTTP_METHOD_POST = 'POST'
-
- def __init__(self, uuid, shared_db_name, user_db_prefix):
- """
- Initialize the mapper.
-
- The C{uuid} is used to create the rules that will either allow or
- disallow the user to perform specific actions.
-
- @param uuid: The user uuid.
- @type uuid: str
- @param shared_db_name: The name of the shared database that holds
- user's encrypted secrets.
- @type shared_db_name: str
- @param user_db_prefix: The string prefix of users' databases.
- @type user_db_prefix: str
- """
- self._map = Mapper(controller_scan=None)
- self._user_db_prefix = user_db_prefix
- self._shared_db_name = shared_db_name
- self._register_auth_info(self._uuid_dbname(uuid))
-
- def is_authorized(self, environ):
- """
- Return whether an HTTP request that produced the CGI C{environ}
- corresponds to an authorized action.
-
- @param environ: Dictionary containing CGI variables.
- @type environ: dict
-
- @return: Whether the action is authorized or not.
- @rtype: bool
- """
- return self._map.match(environ=environ) is not None
-
- def _register(self, pattern, http_methods):
- """
- Register a C{pattern} in the mapper as valid for C{http_methods}.
-
- @param pattern: The URL pattern that corresponds to the user action.
- @type pattern: str
- @param http_methods: A list of authorized HTTP methods.
- @type http_methods: list of str
- """
- self._map.connect(
- None, pattern, http_methods=http_methods,
- conditions=dict(method=http_methods),
- requirements={'dbname': DBNAME_CONSTRAINTS})
-
- def _uuid_dbname(self, uuid):
- """
- Return the database name corresponding to C{uuid}.
-
- @param uuid: The user uid.
- @type uuid: str
-
- @return: The database name corresponding to C{uuid}.
- @rtype: str
- """
- return '%s%s' % (self._user_db_prefix, uuid)
-
- def _register_auth_info(self, dbname):
- """
- Register the authorization info in the mapper using C{dbname} as the
- user's database name.
-
- This method sets up the following authorization rules:
-
- URL path | Authorized actions
- --------------------------------------------------
- / | GET
- /shared-db | GET
- /shared-db/docs | -
- /shared-db/doc/{id} | GET, PUT, DELETE
- /shared-db/sync-from/{source} | -
- /user-db | GET, PUT, DELETE
- /user-db/docs | -
- /user-db/doc/{id} | -
- /user-db/sync-from/{source} | GET, PUT, POST
-
- @param dbname: The name of the user's database.
- @type dbname: str
- """
- # auth info for global resource
- self._register('/', [self.HTTP_METHOD_GET])
- # auth info for shared-db database resource
- self._register(
- '/%s' % self._shared_db_name,
- [self.HTTP_METHOD_GET])
- # auth info for shared-db doc resource
- self._register(
- '/%s/doc/{id:.*}' % self._shared_db_name,
- [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT,
- self.HTTP_METHOD_DELETE])
- # auth info for user-db database resource
- self._register(
- '/%s' % dbname,
- [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT,
- self.HTTP_METHOD_DELETE])
- # auth info for user-db sync resource
- self._register(
- '/%s/sync-from/{source_replica_uid}' % dbname,
- [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT,
- self.HTTP_METHOD_POST])
- # generate the regular expressions
- self._map.create_regs()
-
-
-class SoledadAuthMiddleware(object):
- """
- Soledad Authentication WSGI middleware.
-
- This class must be extended to implement specific authentication methods
- (see SoledadTokenAuthMiddleware below).
-
- It expects an HTTP_AUTHORIZATION header containing the the concatenation of
- the following strings:
-
- 1. The authentication scheme. It will be verified by the
- _verify_authentication_scheme() method.
-
- 2. A space character.
-
- 3. The base64 encoded string of the concatenation of the user uuid with
- the authentication data, separated by a collon, like this:
-
- base64("<uuid>:<auth_data>")
-
- After authentication check, the class performs an authorization check to
- verify whether the user is authorized to perform the requested action.
-
- On client-side, 2 methods must be implemented so the soledad client knows
- how to send authentication headers to server:
-
- * set_<method>_credentials: store authentication credentials in the
- class.
-
- * _sign_request: format and include custom authentication data in
- the HTTP_AUTHORIZATION header.
-
- See leap.soledad.auth and u1db.remote.http_client.HTTPClient to understand
- how to do it.
- """
-
- __metaclass__ = ABCMeta
-
- HTTP_AUTH_KEY = "HTTP_AUTHORIZATION"
- PATH_INFO_KEY = "PATH_INFO"
-
- CONTENT_TYPE_JSON = ('content-type', 'application/json')
-
- def __init__(self, app):
- """
- Initialize the Soledad Authentication Middleware.
-
- @param app: The application to run on successfull authentication.
- @type app: u1db.remote.http_app.HTTPApp
- @param prefix: Auth app path prefix.
- @type prefix: str
- """
- self._app = app
-
- def _error(self, start_response, status, description, message=None):
- """
- Send a JSON serialized error to WSGI client.
-
- @param start_response: Callable of the form start_response(status,
- response_headers, exc_info=None).
- @type start_response: callable
- @param status: Status string of the form "999 Message here"
- @type status: str
- @param response_headers: A list of (header_name, header_value) tuples
- describing the HTTP response header.
- @type response_headers: list
- @param description: The error description.
- @type description: str
- @param message: The error message.
- @type message: str
-
- @return: List with JSON serialized error message.
- @rtype list
- """
- start_response("%d %s" % (status, httplib.responses[status]),
- [self.CONTENT_TYPE_JSON])
- err = {"error": description}
- if message:
- err['message'] = message
- return [json.dumps(err)]
-
- def _unauthorized_error(self, start_response, message):
- """
- Send a unauth error.
-
- @param message: The error message.
- @type message: str
- @param start_response: Callable of the form start_response(status,
- response_headers, exc_info=None).
- @type start_response: callable
-
- @return: List with JSON serialized error message.
- @rtype list
- """
- return self._error(
- start_response,
- 401,
- "unauthorized",
- message)
-
- def __call__(self, environ, start_response):
- """
- Handle a WSGI call to the authentication application.
-
- @param environ: Dictionary containing CGI variables.
- @type environ: dict
- @param start_response: Callable of the form start_response(status,
- response_headers, exc_info=None).
- @type start_response: callable
-
- @return: Target application results if authentication succeeds, an
- error message otherwise.
- @rtype: list
- """
- # check for authentication header
- auth = environ.get(self.HTTP_AUTH_KEY)
- if not auth:
- return self._unauthorized_error(
- start_response, "Missing authentication header.")
-
- # get authentication data
- scheme, encoded = auth.split(None, 1)
- uuid, auth_data = encoded.decode('base64').split(':', 1)
- if not self._verify_authentication_scheme(scheme):
- return self._unauthorized_error("Wrong authentication scheme")
-
- # verify if user is athenticated
- if not self._verify_authentication_data(uuid, auth_data):
- return self._unauthorized_error(
- start_response,
- self._get_auth_error_string())
-
- # verify if user is authorized to perform action
- if not self._verify_authorization(environ, uuid):
- return self._unauthorized_error(
- start_response,
- "Unauthorized action.")
-
- # move on to the real Soledad app
- del environ[self.HTTP_AUTH_KEY]
- return self._app(environ, start_response)
-
- @abstractmethod
- def _verify_authentication_scheme(self, scheme):
- """
- Verify if authentication scheme is valid.
-
- @param scheme: Auth scheme extracted from the HTTP_AUTHORIZATION
- header.
- @type scheme: str
-
- @return: Whether the authentitcation scheme is valid.
- """
- return None
-
- @abstractmethod
- def _verify_authentication_data(self, uuid, auth_data):
- """
- Verify valid authenticatiion for this request.
-
- @param uuid: The user's uuid.
- @type uuid: str
- @param auth_data: Authentication data.
- @type auth_data: str
-
- @return: Whether the token is valid for authenticating the request.
- @rtype: bool
- """
- return None
-
- def _verify_authorization(self, environ, uuid):
- """
- Verify if the user is authorized to perform the requested action over
- the requested database.
-
- @param environ: Dictionary containing CGI variables.
- @type environ: dict
- @param uuid: The user's uuid.
- @type uuid: str
-
- @return: Whether the user is authorize to perform the requested action
- over the requested db.
- @rtype: bool
- """
- return URLToAuthorization(
- uuid, self._app.SHARED_DB_NAME,
- self._app.USER_DB_PREFIX
- ).is_authorized(environ)
-
- @abstractmethod
- def _get_auth_error_string(self):
- """
- Return an error string specific for each kind of authentication method.
-
- @return: The error string.
- """
- return None
-
-
-class SoledadTokenAuthMiddleware(SoledadAuthMiddleware):
- """
- Token based authentication.
- """
-
- TOKENS_DB = "tokens"
- TOKENS_TYPE_KEY = "type"
- TOKENS_TYPE_DEF = "Token"
- TOKENS_USER_ID_KEY = "user_id"
-
- TOKEN_AUTH_ERROR_STRING = "Incorrect address or token."
-
- def _verify_authentication_scheme(self, scheme):
- """
- Verify if authentication scheme is valid.
-
- @param scheme: Auth scheme extracted from the HTTP_AUTHORIZATION
- header.
- @type scheme: str
-
- @return: Whether the authentitcation scheme is valid.
- """
- if scheme.lower() != 'token':
- return False
- return True
-
- def _verify_authentication_data(self, uuid, auth_data):
- """
- Extract token from C{auth_data} and proceed with verification of
- C{uuid} authentication.
-
- @param uuid: The user UID.
- @type uuid: str
- @param auth_data: Authentication data (i.e. the token).
- @type auth_data: str
-
- @return: Whether the token is valid for authenticating the request.
- @rtype: bool
- """
- token = auth_data # we expect a cleartext token at this point
- return self._verify_token_in_couchdb(uuid, token)
-
- def _verify_token_in_couchdb(self, uuid, token):
- """
- Query couchdb to decide if C{token} is valid for C{uuid}.
-
- @param uuid: The user uuid.
- @type uuid: str
- @param token: The token.
- @type token: str
- """
- server = Server(url=self._app.state.couch_url)
- try:
- dbname = self.TOKENS_DB
- db = server[dbname]
- token = db.get(token)
- if token is None:
- return False
- return token[self.TOKENS_TYPE_KEY] == self.TOKENS_TYPE_DEF and \
- token[self.TOKENS_USER_ID_KEY] == uuid
- except Exception as e:
- log.err(e)
- return False
- return True
-
- def _get_auth_error_string(self):
- """
- Get the error string for token auth.
-
- @return: The error string.
- """
- return self.TOKEN_AUTH_ERROR_STRING
diff --git a/soledad_server/src/leap/soledad_server/couch.py b/soledad_server/src/leap/soledad_server/couch.py
deleted file mode 100644
index ed5ad6b3..00000000
--- a/soledad_server/src/leap/soledad_server/couch.py
+++ /dev/null
@@ -1,480 +0,0 @@
-# -*- coding: utf-8 -*-
-# couch.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""A U1DB backend that uses CouchDB as its persistence layer."""
-
-# general imports
-import uuid
-import re
-import simplejson as json
-
-
-from base64 import b64encode, b64decode
-from u1db import errors
-from u1db.sync import Synchronizer
-from u1db.backends.inmemory import InMemoryIndex
-from u1db.remote.server_state import ServerState
-from u1db.errors import DatabaseDoesNotExist
-from couchdb.client import Server, Document as CouchDocument
-from couchdb.http import ResourceNotFound
-
-
-from leap.soledad_server.objectstore import (
- ObjectStoreDatabase,
- ObjectStoreSyncTarget,
-)
-
-
-class InvalidURLError(Exception):
- """
- Exception raised when Soledad encounters a malformed URL.
- """
-
-
-class CouchDatabase(ObjectStoreDatabase):
- """
- A U1DB backend that uses Couch as its persistence layer.
- """
-
- U1DB_TRANSACTION_LOG_KEY = 'transaction_log'
- U1DB_CONFLICTS_KEY = 'conflicts'
- U1DB_OTHER_GENERATIONS_KEY = 'other_generations'
- U1DB_INDEXES_KEY = 'indexes'
- U1DB_REPLICA_UID_KEY = 'replica_uid'
-
- COUCH_ID_KEY = '_id'
- COUCH_REV_KEY = '_rev'
- COUCH_U1DB_ATTACHMENT_KEY = 'u1db_json'
- COUCH_U1DB_REV_KEY = 'u1db_rev'
-
- @classmethod
- def open_database(cls, url, create):
- """
- Open a U1DB database using CouchDB as backend.
-
- @param url: the url of the database replica
- @type url: str
- @param create: should the replica be created if it does not exist?
- @type create: bool
-
- @return: the database instance
- @rtype: CouchDatabase
- """
- # get database from url
- m = re.match('(^https?://[^/]+)/(.+)$', url)
- if not m:
- raise InvalidURLError
- url = m.group(1)
- dbname = m.group(2)
- server = Server(url=url)
- try:
- server[dbname]
- except ResourceNotFound:
- if not create:
- raise DatabaseDoesNotExist()
- return cls(url, dbname)
-
- def __init__(self, url, dbname, replica_uid=None, full_commit=True,
- session=None):
- """
- Create a new Couch data container.
-
- @param url: the url of the couch database
- @type url: str
- @param dbname: the database name
- @type dbname: str
- @param replica_uid: an optional unique replica identifier
- @type replica_uid: str
- @param full_commit: turn on the X-Couch-Full-Commit header
- @type full_commit: bool
- @param session: an http.Session instance or None for a default session
- @type session: http.Session
- """
- self._url = url
- self._full_commit = full_commit
- self._session = session
- self._server = Server(url=self._url,
- full_commit=self._full_commit,
- session=self._session)
- self._dbname = dbname
- # this will ensure that transaction and sync logs exist and are
- # up-to-date.
- try:
- self._database = self._server[self._dbname]
- except ResourceNotFound:
- self._server.create(self._dbname)
- self._database = self._server[self._dbname]
- ObjectStoreDatabase.__init__(self, replica_uid=replica_uid)
-
- #-------------------------------------------------------------------------
- # methods from Database
- #-------------------------------------------------------------------------
-
- def _get_doc(self, doc_id, check_for_conflicts=False):
- """
- Get just the document content, without fancy handling.
-
- @param doc_id: The unique document identifier
- @type doc_id: str
- @param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise asking for a deleted
- document will return None.
- @type include_deleted: bool
-
- @return: a Document object.
- @type: u1db.Document
- """
- cdoc = self._database.get(doc_id)
- if cdoc is None:
- return None
- has_conflicts = False
- if check_for_conflicts:
- has_conflicts = self._has_conflicts(doc_id)
- doc = self._factory(
- doc_id=doc_id,
- rev=cdoc[self.COUCH_U1DB_REV_KEY],
- has_conflicts=has_conflicts)
- contents = self._database.get_attachment(
- cdoc,
- self.COUCH_U1DB_ATTACHMENT_KEY)
- if contents:
- doc.content = json.loads(contents.read())
- else:
- doc.make_tombstone()
- return doc
-
- def get_all_docs(self, include_deleted=False):
- """
- Get the JSON content for all documents in the database.
-
- @param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise deleted documents will not
- be included in the results.
- @type include_deleted: bool
-
- @return: (generation, [Document])
- The current generation of the database, followed by a list of all
- the documents in the database.
- @rtype: tuple
- """
- generation = self._get_generation()
- results = []
- for doc_id in self._database:
- if doc_id == self.U1DB_DATA_DOC_ID:
- continue
- doc = self._get_doc(doc_id, check_for_conflicts=True)
- if doc.content is None and not include_deleted:
- continue
- results.append(doc)
- return (generation, results)
-
- def _put_doc(self, doc):
- """
- Update a document.
-
- This is called everytime we just want to do a raw put on the db (i.e.
- without index updates, document constraint checks, and conflict
- checks).
-
- @param doc: The document to update.
- @type doc: u1db.Document
-
- @return: The new revision identifier for the document.
- @rtype: str
- """
- # prepare couch's Document
- cdoc = CouchDocument()
- cdoc[self.COUCH_ID_KEY] = doc.doc_id
- # we have to guarantee that couch's _rev is consistent
- old_cdoc = self._database.get(doc.doc_id)
- if old_cdoc is not None:
- cdoc[self.COUCH_REV_KEY] = old_cdoc[self.COUCH_REV_KEY]
- # store u1db's rev
- cdoc[self.COUCH_U1DB_REV_KEY] = doc.rev
- # save doc in db
- self._database.save(cdoc)
- # store u1db's content as json string
- if not doc.is_tombstone():
- self._database.put_attachment(
- cdoc, doc.get_json(),
- filename=self.COUCH_U1DB_ATTACHMENT_KEY)
- else:
- self._database.delete_attachment(
- cdoc,
- self.COUCH_U1DB_ATTACHMENT_KEY)
-
- def get_sync_target(self):
- """
- Return a SyncTarget object, for another u1db to synchronize with.
-
- @return: The sync target.
- @rtype: CouchSyncTarget
- """
- return CouchSyncTarget(self)
-
- def create_index(self, index_name, *index_expressions):
- """
- Create a named index, which can then be queried for future lookups.
-
- @param index_name: A unique name which can be used as a key prefix.
- @param index_expressions: Index expressions defining the index
- information.
- """
- if index_name in self._indexes:
- if self._indexes[index_name]._definition == list(
- index_expressions):
- return
- raise errors.IndexNameTakenError
- index = InMemoryIndex(index_name, list(index_expressions))
- for doc_id in self._database:
- if doc_id == self.U1DB_DATA_DOC_ID: # skip special file
- continue
- doc = self._get_doc(doc_id)
- if doc.content is not None:
- index.add_json(doc_id, doc.get_json())
- self._indexes[index_name] = index
- # save data in object store
- self._store_u1db_data()
-
- def close(self):
- """
- Release any resources associated with this database.
-
- @return: True if db was succesfully closed.
- @rtype: bool
- """
- # TODO: fix this method so the connection is properly closed and
- # test_close (+tearDown, which deletes the db) works without problems.
- self._url = None
- self._full_commit = None
- self._session = None
- #self._server = None
- self._database = None
- return True
-
- def sync(self, url, creds=None, autocreate=True):
- """
- Synchronize documents with remote replica exposed at url.
-
- @param url: The url of the target replica to sync with.
- @type url: str
- @param creds: optional dictionary giving credentials.
- to authorize the operation with the server.
- @type creds: dict
- @param autocreate: Ask the target to create the db if non-existent.
- @type autocreate: bool
-
- @return: The local generation before the synchronisation was performed.
- @rtype: int
- """
- return Synchronizer(self, CouchSyncTarget(url, creds=creds)).sync(
- autocreate=autocreate)
-
- #-------------------------------------------------------------------------
- # methods from ObjectStoreDatabase
- #-------------------------------------------------------------------------
-
- def _init_u1db_data(self):
- """
- Initialize U1DB info data structure in the couch db.
-
- A U1DB database needs to keep track of all database transactions,
- document conflicts, the generation of other replicas it has seen,
- indexes created by users and so on.
-
- In this implementation, all this information is stored in a special
- document stored in the couch db with id equals to
- CouchDatabse.U1DB_DATA_DOC_ID.
-
- This method initializes the document that will hold such information.
- """
- if self._replica_uid is None:
- self._replica_uid = uuid.uuid4().hex
- # TODO: prevent user from overwriting a document with the same doc_id
- # as this one.
- doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
- doc.content = {
- self.U1DB_TRANSACTION_LOG_KEY: b64encode(json.dumps([])),
- self.U1DB_CONFLICTS_KEY: b64encode(json.dumps({})),
- self.U1DB_OTHER_GENERATIONS_KEY: b64encode(json.dumps({})),
- self.U1DB_INDEXES_KEY: b64encode(json.dumps({})),
- self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
- }
- self._put_doc(doc)
-
- def _fetch_u1db_data(self):
- """
- Fetch U1DB info from the couch db.
-
- See C{_init_u1db_data} documentation.
- """
- # retrieve u1db data from couch db
- cdoc = self._database.get(self.U1DB_DATA_DOC_ID)
- jsonstr = self._database.get_attachment(
- cdoc, self.COUCH_U1DB_ATTACHMENT_KEY).read()
- content = json.loads(jsonstr)
- # set u1db database info
- self._transaction_log = json.loads(
- b64decode(content[self.U1DB_TRANSACTION_LOG_KEY]))
- self._conflicts = json.loads(
- b64decode(content[self.U1DB_CONFLICTS_KEY]))
- self._other_generations = json.loads(
- b64decode(content[self.U1DB_OTHER_GENERATIONS_KEY]))
- self._indexes = self._load_indexes_from_json(
- b64decode(content[self.U1DB_INDEXES_KEY]))
- self._replica_uid = b64decode(content[self.U1DB_REPLICA_UID_KEY])
- # save couch _rev
- self._couch_rev = cdoc[self.COUCH_REV_KEY]
-
- def _store_u1db_data(self):
- """
- Store U1DB info in the couch db.
-
- See C{_init_u1db_data} documentation.
- """
- doc = self._factory(doc_id=self.U1DB_DATA_DOC_ID)
- doc.content = {
- # Here, the b64 encode ensures that document content
- # does not cause strange behaviour in couchdb because
- # of encoding.
- self.U1DB_TRANSACTION_LOG_KEY:
- b64encode(json.dumps(self._transaction_log)),
- self.U1DB_CONFLICTS_KEY: b64encode(json.dumps(self._conflicts)),
- self.U1DB_OTHER_GENERATIONS_KEY:
- b64encode(json.dumps(self._other_generations)),
- self.U1DB_INDEXES_KEY: b64encode(self._dump_indexes_as_json()),
- self.U1DB_REPLICA_UID_KEY: b64encode(self._replica_uid),
- self.COUCH_REV_KEY: self._couch_rev}
- self._put_doc(doc)
-
- #-------------------------------------------------------------------------
- # Couch specific methods
- #-------------------------------------------------------------------------
-
- INDEX_NAME_KEY = 'name'
- INDEX_DEFINITION_KEY = 'definition'
- INDEX_VALUES_KEY = 'values'
-
- def delete_database(self):
- """
- Delete a U1DB CouchDB database.
- """
- del(self._server[self._dbname])
-
- def _dump_indexes_as_json(self):
- """
- Dump index definitions as JSON string.
- """
- indexes = {}
- for name, idx in self._indexes.iteritems():
- indexes[name] = {}
- for attr in [self.INDEX_NAME_KEY, self.INDEX_DEFINITION_KEY,
- self.INDEX_VALUES_KEY]:
- indexes[name][attr] = getattr(idx, '_' + attr)
- return json.dumps(indexes)
-
- def _load_indexes_from_json(self, indexes):
- """
- Load index definitions from JSON string.
-
- @param indexes: A JSON serialization of a list of [('index-name',
- ['field', 'field2'])].
- @type indexes: str
-
- @return: A dictionary with the index definitions.
- @rtype: dict
- """
- dict = {}
- for name, idx_dict in json.loads(indexes).iteritems():
- idx = InMemoryIndex(name, idx_dict[self.INDEX_DEFINITION_KEY])
- idx._values = idx_dict[self.INDEX_VALUES_KEY]
- dict[name] = idx
- return dict
-
-
-class CouchSyncTarget(ObjectStoreSyncTarget):
- """
- Functionality for using a CouchDatabase as a synchronization target.
- """
-
-
-class CouchServerState(ServerState):
- """
- Inteface of the WSGI server with the CouchDB backend.
- """
-
- def __init__(self, couch_url):
- self._couch_url = couch_url
-
- def open_database(self, dbname):
- """
- Open a couch database.
-
- @param dbname: The name of the database to open.
- @type dbname: str
-
- @return: The CouchDatabase object.
- @rtype: CouchDatabase
- """
- # TODO: open couch
- return CouchDatabase.open_database(
- self._couch_url + '/' + dbname,
- create=False)
-
- def ensure_database(self, dbname):
- """
- Ensure couch database exists.
-
- @param dbname: The name of the database to ensure.
- @type dbname: str
-
- @return: The CouchDatabase object and the replica uid.
- @rtype: (CouchDatabase, str)
- """
- db = CouchDatabase.open_database(
- self._couch_url + '/' + dbname,
- create=True)
- return db, db._replica_uid
-
- def delete_database(self, dbname):
- """
- Delete couch database.
-
- @param dbname: The name of the database to delete.
- @type dbname: str
- """
- CouchDatabase.delete_database(self._couch_url + '/' + dbname)
-
- def _set_couch_url(self, url):
- """
- Set the couchdb URL
-
- @param url: CouchDB URL
- @type url: str
- """
- self._couch_url = url
-
- def _get_couch_url(self):
- """
- Return CouchDB URL
-
- @rtype: str
- """
- return self._couch_url
-
- couch_url = property(_get_couch_url, _set_couch_url, doc='CouchDB URL')
diff --git a/soledad_server/src/leap/soledad_server/objectstore.py b/soledad_server/src/leap/soledad_server/objectstore.py
deleted file mode 100644
index 8afac3ec..00000000
--- a/soledad_server/src/leap/soledad_server/objectstore.py
+++ /dev/null
@@ -1,296 +0,0 @@
-# -*- coding: utf-8 -*-
-# objectstore.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""
-Abstract U1DB backend to handle storage using object stores (like CouchDB, for
-example).
-
-Right now, this is only used by CouchDatabase backend, but can also be
-extended to implement OpenStack or Amazon S3 storage, for example.
-
-See U1DB documentation for more information on how to use databases.
-"""
-
-from u1db.backends.inmemory import (
- InMemoryDatabase,
- InMemorySyncTarget,
-)
-from u1db import errors
-
-
-class ObjectStoreDatabase(InMemoryDatabase):
- """
- A backend for storing u1db data in an object store.
- """
-
- @classmethod
- def open_database(cls, url, create, document_factory=None):
- """
- Open a U1DB database using an object store as backend.
-
- @param url: the url of the database replica
- @type url: str
- @param create: should the replica be created if it does not exist?
- @type create: bool
- @param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- @type document_factory: callable
-
- @return: the database instance
- @rtype: CouchDatabase
- """
- raise NotImplementedError(cls.open_database)
-
- def __init__(self, replica_uid=None, document_factory=None):
- """
- Initialize the object store database.
-
- @param replica_uid: an optional unique replica identifier
- @type replica_uid: str
- @param document_factory: A function that will be called with the same
- parameters as Document.__init__.
- @type document_factory: callable
- """
- InMemoryDatabase.__init__(
- self,
- replica_uid,
- document_factory=document_factory)
- # sync data in memory with data in object store
- if not self._get_doc(self.U1DB_DATA_DOC_ID):
- self._init_u1db_data()
- self._fetch_u1db_data()
-
- #-------------------------------------------------------------------------
- # methods from Database
- #-------------------------------------------------------------------------
-
- def _set_replica_uid(self, replica_uid):
- """
- Force the replica_uid to be set.
-
- @param replica_uid: The uid of the replica.
- @type replica_uid: str
- """
- InMemoryDatabase._set_replica_uid(self, replica_uid)
- self._store_u1db_data()
-
- def _put_doc(self, doc):
- """
- Update a document.
-
- This is called everytime we just want to do a raw put on the db (i.e.
- without index updates, document constraint checks, and conflict
- checks).
-
- @param doc: The document to update.
- @type doc: u1db.Document
-
- @return: The new revision identifier for the document.
- @rtype: str
- """
- raise NotImplementedError(self._put_doc)
-
- def _get_doc(self, doc_id):
- """
- Get just the document content, without fancy handling.
-
- @param doc_id: The unique document identifier
- @type doc_id: str
- @param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise asking for a deleted
- document will return None.
- @type include_deleted: bool
-
- @return: a Document object.
- @type: u1db.Document
- """
- raise NotImplementedError(self._get_doc)
-
- def get_all_docs(self, include_deleted=False):
- """
- Get the JSON content for all documents in the database.
-
- @param include_deleted: If set to True, deleted documents will be
- returned with empty content. Otherwise deleted documents will not
- be included in the results.
- @type include_deleted: bool
-
- @return: (generation, [Document])
- The current generation of the database, followed by a list of all
- the documents in the database.
- @rtype: tuple
- """
- raise NotImplementedError(self.get_all_docs)
-
- def delete_doc(self, doc):
- """
- Mark a document as deleted.
-
- @param doc: The document to mark as deleted.
- @type doc: u1db.Document
-
- @return: The new revision id of the document.
- @type: str
- """
- old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
- if old_doc is None:
- raise errors.DocumentDoesNotExist
- if old_doc.rev != doc.rev:
- raise errors.RevisionConflict()
- if old_doc.is_tombstone():
- raise errors.DocumentAlreadyDeleted
- if old_doc.has_conflicts:
- raise errors.ConflictedDoc()
- new_rev = self._allocate_doc_rev(doc.rev)
- doc.rev = new_rev
- doc.make_tombstone()
- self._put_and_update_indexes(old_doc, doc)
- return new_rev
-
- # index-related methods
-
- def create_index(self, index_name, *index_expressions):
- """
- Create a named index, which can then be queried for future lookups.
-
- See U1DB documentation for more information.
-
- @param index_name: A unique name which can be used as a key prefix.
- @param index_expressions: Index expressions defining the index
- information.
- """
- raise NotImplementedError(self.create_index)
-
- def delete_index(self, index_name):
- """
- Remove a named index.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param index_name: The name of the index we are removing.
- @type index_name: str
- """
- InMemoryDatabase.delete_index(self, index_name)
- self._store_u1db_data()
-
- def _replace_conflicts(self, doc, conflicts):
- """
- Set new conflicts for a document.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param doc: The document with a new set of conflicts.
- @param conflicts: The new set of conflicts.
- @type conflicts: list
- """
- InMemoryDatabase._replace_conflicts(self, doc, conflicts)
- self._store_u1db_data()
-
- def _do_set_replica_gen_and_trans_id(self, other_replica_uid,
- other_generation,
- other_transaction_id):
- """
- Set the last-known generation and transaction id for the other
- database replica.
-
- Here we just guarantee that the new info will be stored in the backend
- db after update.
-
- @param other_replica_uid: The U1DB identifier for the other replica.
- @type other_replica_uid: str
- @param other_generation: The generation number for the other replica.
- @type other_generation: int
- @param other_transaction_id: The transaction id associated with the
- generation.
- @type other_transaction_id: str
- """
- InMemoryDatabase._do_set_replica_gen_and_trans_id(
- self,
- other_replica_uid,
- other_generation,
- other_transaction_id)
- self._store_u1db_data()
-
- #-------------------------------------------------------------------------
- # implemented methods from CommonBackend
- #-------------------------------------------------------------------------
-
- def _put_and_update_indexes(self, old_doc, doc):
- """
- Update a document and all indexes related to it.
-
- @param old_doc: The old version of the document.
- @type old_doc: u1db.Document
- @param doc: The new version of the document.
- @type doc: u1db.Document
- """
- for index in self._indexes.itervalues():
- if old_doc is not None and not old_doc.is_tombstone():
- index.remove_json(old_doc.doc_id, old_doc.get_json())
- if not doc.is_tombstone():
- index.add_json(doc.doc_id, doc.get_json())
- trans_id = self._allocate_transaction_id()
- self._put_doc(doc)
- self._transaction_log.append((doc.doc_id, trans_id))
- self._store_u1db_data()
-
- #-------------------------------------------------------------------------
- # methods specific for object stores
- #-------------------------------------------------------------------------
-
- U1DB_DATA_DOC_ID = 'u1db_data'
-
- def _fetch_u1db_data(self):
- """
- Fetch u1db configuration data from backend storage.
-
- See C{_init_u1db_data} documentation.
- """
- NotImplementedError(self._fetch_u1db_data)
-
- def _store_u1db_data(self):
- """
- Store u1db configuration data on backend storage.
-
- See C{_init_u1db_data} documentation.
- """
- NotImplementedError(self._store_u1db_data)
-
- def _init_u1db_data(self):
- """
- Initialize u1db configuration data on backend storage.
-
- A U1DB database needs to keep track of all database transactions,
- document conflicts, the generation of other replicas it has seen,
- indexes created by users and so on.
-
- In this implementation, all this information is stored in a special
- document stored in the couch db with id equals to
- CouchDatabse.U1DB_DATA_DOC_ID.
-
- This method initializes the document that will hold such information.
- """
- NotImplementedError(self._init_u1db_data)
-
-
-class ObjectStoreSyncTarget(InMemorySyncTarget):
- """
- Functionality for using an ObjectStore as a synchronization target.
- """