diff options
17 files changed, 63 insertions, 834 deletions
diff --git a/client/src/leap/soledad/client/http_target/support.py b/client/src/leap/soledad/client/http_target/support.py index d82fe346..6ec98ed4 100644 --- a/client/src/leap/soledad/client/http_target/support.py +++ b/client/src/leap/soledad/client/http_target/support.py @@ -31,6 +31,7 @@ from leap.soledad.common.l2db.remote import http_errors # twisted. Because of that, we redefine the http body reader used by the HTTP # client below. + class ReadBodyProtocol(_ReadBodyProtocol): """ From original Twisted implementation, focused on adding our error diff --git a/common/src/leap/soledad/common/l2db/__init__.py b/common/src/leap/soledad/common/l2db/__init__.py index cc121d06..c0bd15fe 100644 --- a/common/src/leap/soledad/common/l2db/__init__.py +++ b/common/src/leap/soledad/common/l2db/__init__.py @@ -464,8 +464,8 @@ class DocumentBase(object): """ # Since this is just for testing, we don't worry about comparing # against things that aren't a Document. - return ((self.doc_id, self.rev, self.get_json()) - < (other.doc_id, other.rev, other.get_json())) + return ((self.doc_id, self.rev, self.get_json()) < + (other.doc_id, other.rev, other.get_json())) def get_json(self): """Get the json serialization of this document.""" diff --git a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py index 309000ee..ba273039 100644 --- a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py +++ b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py @@ -156,7 +156,7 @@ class SQLiteDatabase(CommonBackend): def _initialize(self, c): """Create the schema in the database.""" - #read the script with sql commands + # read the script with sql commands # TODO: Change how we set up the dependency. Most likely use something # like lp:dirspec to grab the file from a common resource # directory. Doesn't specifically need to be handled until we get @@ -172,7 +172,7 @@ class SQLiteDatabase(CommonBackend): if not line: continue c.execute(line) - #add extra fields + # add extra fields self._extra_schema_init(c) # A unique identifier should be set for this replica. Implementations # don't have to strictly use uuid here, but we do want the uid to be @@ -509,7 +509,8 @@ class SQLiteDatabase(CommonBackend): def _put_doc_if_newer(self, doc, save_conflict, replica_uid=None, replica_gen=None, replica_trans_id=None): with self._db_handle: - return super(SQLiteDatabase, self)._put_doc_if_newer(doc, + return super(SQLiteDatabase, self)._put_doc_if_newer( + doc, save_conflict=save_conflict, replica_uid=replica_uid, replica_gen=replica_gen, replica_trans_id=replica_trans_id) @@ -620,14 +621,14 @@ class SQLiteDatabase(CommonBackend): novalue_where = ["d.doc_id = d%d.doc_id" " AND d%d.field_name = ?" % (i, i) for i in range(len(definition))] - wildcard_where = [novalue_where[i] - + (" AND d%d.value NOT NULL" % (i,)) + wildcard_where = [novalue_where[i] + + (" AND d%d.value NOT NULL" % (i,)) for i in range(len(definition))] - exact_where = [novalue_where[i] - + (" AND d%d.value = ?" % (i,)) + exact_where = [novalue_where[i] + + (" AND d%d.value = ?" % (i,)) for i in range(len(definition))] - like_where = [novalue_where[i] - + (" AND d%d.value GLOB ?" % (i,)) + like_where = [novalue_where[i] + + (" AND d%d.value GLOB ?" % (i,)) for i in range(len(definition))] is_wildcard = False # Merge the lists together, so that: @@ -672,7 +673,8 @@ class SQLiteDatabase(CommonBackend): try: c.execute(statement, tuple(args)) except dbapi2.OperationalError, e: - raise dbapi2.OperationalError(str(e) + + raise dbapi2.OperationalError( + str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args)) res = c.fetchall() results = [] @@ -771,7 +773,8 @@ class SQLiteDatabase(CommonBackend): try: c.execute(statement, tuple(args)) except dbapi2.OperationalError, e: - raise dbapi2.OperationalError(str(e) + + raise dbapi2.OperationalError( + str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args)) res = c.fetchall() results = [] @@ -800,7 +803,8 @@ class SQLiteDatabase(CommonBackend): try: c.execute(statement, tuple(definition)) except dbapi2.OperationalError, e: - raise dbapi2.OperationalError(str(e) + + raise dbapi2.OperationalError( + str(e) + '\nstatement: %s\nargs: %s\n' % (statement, tuple(definition))) return c.fetchall() diff --git a/common/src/leap/soledad/common/l2db/commandline/__init__.py b/common/src/leap/soledad/common/l2db/commandline/__init__.py deleted file mode 100644 index 3f32e381..00000000 --- a/common/src/leap/soledad/common/l2db/commandline/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. diff --git a/common/src/leap/soledad/common/l2db/commandline/client.py b/common/src/leap/soledad/common/l2db/commandline/client.py deleted file mode 100644 index 15bf8561..00000000 --- a/common/src/leap/soledad/common/l2db/commandline/client.py +++ /dev/null @@ -1,497 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""Commandline bindings for the u1db-client program.""" - -import argparse -import os -try: - import simplejson as json -except ImportError: - import json # noqa -import sys - -from u1db import ( - Document, - open as u1db_open, - sync, - errors, - ) -from u1db.commandline import command -from u1db.remote import ( - http_database, - http_target, - ) - - -client_commands = command.CommandGroup() - - -def set_oauth_credentials(client): - keys = os.environ.get('OAUTH_CREDENTIALS', None) - if keys is not None: - consumer_key, consumer_secret, \ - token_key, token_secret = keys.split(":") - client.set_oauth_credentials(consumer_key, consumer_secret, - token_key, token_secret) - - -class OneDbCmd(command.Command): - """Base class for commands operating on one local or remote database.""" - - def _open(self, database, create): - if database.startswith(('http://', 'https://')): - db = http_database.HTTPDatabase(database) - set_oauth_credentials(db) - db.open(create) - return db - else: - return u1db_open(database, create) - - -class CmdCreate(OneDbCmd): - """Create a new document from scratch""" - - name = 'create' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to update', - metavar='database-path-or-url') - parser.add_argument('infile', nargs='?', default=None, - help='The file to read content from.') - parser.add_argument('--id', dest='doc_id', default=None, - help='Set the document identifier') - - def run(self, database, infile, doc_id): - if infile is None: - infile = self.stdin - db = self._open(database, create=False) - doc = db.create_doc_from_json(infile.read(), doc_id=doc_id) - self.stderr.write('id: %s\nrev: %s\n' % (doc.doc_id, doc.rev)) - -client_commands.register(CmdCreate) - - -class CmdDelete(OneDbCmd): - """Delete a document from the database""" - - name = 'delete' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to update', - metavar='database-path-or-url') - parser.add_argument('doc_id', help='The document id to retrieve') - parser.add_argument('doc_rev', - help='The revision of the document (which is being superseded.)') - - def run(self, database, doc_id, doc_rev): - db = self._open(database, create=False) - doc = Document(doc_id, doc_rev, None) - db.delete_doc(doc) - self.stderr.write('rev: %s\n' % (doc.rev,)) - -client_commands.register(CmdDelete) - - -class CmdGet(OneDbCmd): - """Extract a document from the database""" - - name = 'get' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to query', - metavar='database-path-or-url') - parser.add_argument('doc_id', help='The document id to retrieve.') - parser.add_argument('outfile', nargs='?', default=None, - help='The file to write the document to', - type=argparse.FileType('wb')) - - def run(self, database, doc_id, outfile): - if outfile is None: - outfile = self.stdout - try: - db = self._open(database, create=False) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - doc = db.get_doc(doc_id) - if doc is None: - self.stderr.write('Document not found (id: %s)\n' % (doc_id,)) - return 1 # failed - if doc.is_tombstone(): - outfile.write('[document deleted]\n') - else: - outfile.write(doc.get_json() + '\n') - self.stderr.write('rev: %s\n' % (doc.rev,)) - if doc.has_conflicts: - self.stderr.write("Document has conflicts.\n") - -client_commands.register(CmdGet) - - -class CmdGetDocConflicts(OneDbCmd): - """Get the conflicts from a document""" - - name = 'get-doc-conflicts' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local database to query', - metavar='database-path') - parser.add_argument('doc_id', help='The document id to retrieve.') - - def run(self, database, doc_id): - try: - db = self._open(database, False) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - conflicts = db.get_doc_conflicts(doc_id) - if not conflicts: - if db.get_doc(doc_id) is None: - self.stderr.write("Document does not exist.\n") - return 1 - self.stdout.write("[") - for i, doc in enumerate(conflicts): - if i: - self.stdout.write(",") - self.stdout.write( - json.dumps(dict(rev=doc.rev, content=doc.content), indent=4)) - self.stdout.write("]\n") - -client_commands.register(CmdGetDocConflicts) - - -class CmdInitDB(OneDbCmd): - """Create a new database""" - - name = 'init-db' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to create', - metavar='database-path-or-url') - parser.add_argument('--replica-uid', default=None, - help='The unique identifier for this database (not for remote)') - - def run(self, database, replica_uid): - db = self._open(database, create=True) - if replica_uid is not None: - db._set_replica_uid(replica_uid) - -client_commands.register(CmdInitDB) - - -class CmdPut(OneDbCmd): - """Add a document to the database""" - - name = 'put' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to update', - metavar='database-path-or-url'), - parser.add_argument('doc_id', help='The document id to retrieve') - parser.add_argument('doc_rev', - help='The revision of the document (which is being superseded.)') - parser.add_argument('infile', nargs='?', default=None, - help='The filename of the document that will be used for content', - type=argparse.FileType('rb')) - - def run(self, database, doc_id, doc_rev, infile): - if infile is None: - infile = self.stdin - try: - db = self._open(database, create=False) - doc = Document(doc_id, doc_rev, infile.read()) - doc_rev = db.put_doc(doc) - self.stderr.write('rev: %s\n' % (doc_rev,)) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - except errors.RevisionConflict: - if db.get_doc(doc_id) is None: - self.stderr.write("Document does not exist.\n") - else: - self.stderr.write("Given revision is not current.\n") - except errors.ConflictedDoc: - self.stderr.write( - "Document has conflicts.\n" - "Inspect with get-doc-conflicts, then resolve.\n") - else: - return - return 1 - -client_commands.register(CmdPut) - - -class CmdResolve(OneDbCmd): - """Resolve a conflicted document""" - - name = 'resolve-doc' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to update', - metavar='database-path-or-url'), - parser.add_argument('doc_id', help='The conflicted document id') - parser.add_argument('doc_revs', metavar="doc-rev", nargs="+", - help='The revisions that the new content supersedes') - parser.add_argument('--infile', nargs='?', default=None, - help='The filename of the document that will be used for content', - type=argparse.FileType('rb')) - - def run(self, database, doc_id, doc_revs, infile): - if infile is None: - infile = self.stdin - try: - db = self._open(database, create=False) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - doc = db.get_doc(doc_id) - if doc is None: - self.stderr.write("Document does not exist.\n") - return 1 - doc.set_json(infile.read()) - db.resolve_doc(doc, doc_revs) - self.stderr.write("rev: %s\n" % db.get_doc(doc_id).rev) - if doc.has_conflicts: - self.stderr.write("Document still has conflicts.\n") - -client_commands.register(CmdResolve) - - -class CmdSync(command.Command): - """Synchronize two databases""" - - name = 'sync' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('source', help='database to sync from') - parser.add_argument('target', help='database to sync to') - - def _open_target(self, target): - if target.startswith(('http://', 'https://')): - st = http_target.HTTPSyncTarget.connect(target) - set_oauth_credentials(st) - else: - db = u1db_open(target, create=True) - st = db.get_sync_target() - return st - - def run(self, source, target): - """Start a Sync request.""" - source_db = u1db_open(source, create=False) - st = self._open_target(target) - syncer = sync.Synchronizer(source_db, st) - syncer.sync() - source_db.close() - -client_commands.register(CmdSync) - - -class CmdCreateIndex(OneDbCmd): - """Create an index""" - - name = "create-index" - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', help='The local database to update', - metavar='database-path') - parser.add_argument('index', help='the name of the index') - parser.add_argument('expression', help='an index expression', - nargs='+') - - def run(self, database, index, expression): - try: - db = self._open(database, create=False) - db.create_index(index, *expression) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - except errors.IndexNameTakenError: - self.stderr.write("There is already a different index named %r.\n" - % (index,)) - return 1 - except errors.IndexDefinitionParseError: - self.stderr.write("Bad index expression.\n") - return 1 - -client_commands.register(CmdCreateIndex) - - -class CmdListIndexes(OneDbCmd): - """List existing indexes""" - - name = "list-indexes" - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', help='The local database to query', - metavar='database-path') - - def run(self, database): - try: - db = self._open(database, create=False) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - for (index, expression) in db.list_indexes(): - self.stdout.write("%s: %s\n" % (index, ", ".join(expression))) - -client_commands.register(CmdListIndexes) - - -class CmdDeleteIndex(OneDbCmd): - """Delete an index""" - - name = "delete-index" - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', help='The local database to update', - metavar='database-path') - parser.add_argument('index', help='the name of the index') - - def run(self, database, index): - try: - db = self._open(database, create=False) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - db.delete_index(index) - -client_commands.register(CmdDeleteIndex) - - -class CmdGetIndexKeys(OneDbCmd): - """Get the index's keys""" - - name = "get-index-keys" - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', help='The local database to query', - metavar='database-path') - parser.add_argument('index', help='the name of the index') - - def run(self, database, index): - try: - db = self._open(database, create=False) - for key in db.get_index_keys(index): - self.stdout.write("%s\n" % (", ".join( - [i.encode('utf-8') for i in key],))) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - except errors.IndexDoesNotExist: - self.stderr.write("Index does not exist.\n") - else: - return - return 1 - -client_commands.register(CmdGetIndexKeys) - - -class CmdGetFromIndex(OneDbCmd): - """Find documents by searching an index""" - - name = "get-from-index" - argv = None - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', help='The local database to query', - metavar='database-path') - parser.add_argument('index', help='the name of the index') - parser.add_argument('values', metavar="value", - help='the value to look up (one per index column)', - nargs="+") - - def run(self, database, index, values): - try: - db = self._open(database, create=False) - docs = db.get_from_index(index, *values) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - except errors.IndexDoesNotExist: - self.stderr.write("Index does not exist.\n") - except errors.InvalidValueForIndex: - index_def = db._get_index_definition(index) - len_diff = len(index_def) - len(values) - if len_diff == 0: - # can't happen (HAH) - raise - argv = self.argv if self.argv is not None else sys.argv - self.stderr.write( - "Invalid query: " - "index %r requires %d query expression%s%s.\n" - "For example, the following would be valid:\n" - " %s %s %r %r %s\n" - % (index, - len(index_def), - "s" if len(index_def) > 1 else "", - ", not %d" % len(values) if len(values) else "", - argv[0], argv[1], database, index, - " ".join(map(repr, - values[:len(index_def)] - + ["*" for i in range(len_diff)])), - )) - except errors.InvalidGlobbing: - argv = self.argv if self.argv is not None else sys.argv - fixed = [] - for (i, v) in enumerate(values): - fixed.append(v) - if v.endswith('*'): - break - # values has at least one element, so i is defined - fixed.extend('*' * (len(values) - i - 1)) - self.stderr.write( - "Invalid query: a star can only be followed by stars.\n" - "For example, the following would be valid:\n" - " %s %s %r %r %s\n" - % (argv[0], argv[1], database, index, - " ".join(map(repr, fixed)))) - - else: - self.stdout.write("[") - for i, doc in enumerate(docs): - if i: - self.stdout.write(",") - self.stdout.write( - json.dumps( - dict(id=doc.doc_id, rev=doc.rev, content=doc.content), - indent=4)) - self.stdout.write("]\n") - return - return 1 - -client_commands.register(CmdGetFromIndex) - - -def main(args): - return client_commands.run_argv(args, sys.stdin, sys.stdout, sys.stderr) diff --git a/common/src/leap/soledad/common/l2db/commandline/command.py b/common/src/leap/soledad/common/l2db/commandline/command.py deleted file mode 100644 index eace0560..00000000 --- a/common/src/leap/soledad/common/l2db/commandline/command.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""Command infrastructure for u1db""" - -import argparse -import inspect - - -class CommandGroup(object): - """A collection of commands.""" - - def __init__(self, description=None): - self.commands = {} - self.description = description - - def register(self, cmd): - """Register a new command to be incorporated with this group.""" - self.commands[cmd.name] = cmd - - def make_argparser(self): - """Create an argparse.ArgumentParser""" - parser = argparse.ArgumentParser(description=self.description) - subs = parser.add_subparsers(title='commands') - for name, cmd in sorted(self.commands.iteritems()): - sub = subs.add_parser(name, help=cmd.__doc__) - sub.set_defaults(subcommand=cmd) - cmd._populate_subparser(sub) - return parser - - def run_argv(self, argv, stdin, stdout, stderr): - """Run a command, from a sys.argv[1:] style input.""" - parser = self.make_argparser() - args = parser.parse_args(argv) - cmd = args.subcommand(stdin, stdout, stderr) - params, _, _, _ = inspect.getargspec(cmd.run) - vals = [] - for param in params[1:]: - vals.append(getattr(args, param)) - return cmd.run(*vals) - - -class Command(object): - """Definition of a Command that can be run. - - :cvar name: The name of the command, so that you can run - 'u1db-client <name>'. - """ - - name = None - - def __init__(self, stdin, stdout, stderr): - self.stdin = stdin - self.stdout = stdout - self.stderr = stderr - - @classmethod - def _populate_subparser(cls, parser): - """Child classes should override this to provide their arguments.""" - raise NotImplementedError(cls._populate_subparser) - - def run(self, *args): - """This is where the magic happens. - - Subclasses should implement this, requesting their specific arguments. - """ - raise NotImplementedError(self.run) diff --git a/common/src/leap/soledad/common/l2db/commandline/serve.py b/common/src/leap/soledad/common/l2db/commandline/serve.py deleted file mode 100644 index 5e10f9cb..00000000 --- a/common/src/leap/soledad/common/l2db/commandline/serve.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""Build server for u1db-serve.""" -import os - -from paste import httpserver - -from u1db.remote import ( - http_app, - server_state, - cors_middleware - ) - - -class DbListingServerState(server_state.ServerState): - """ServerState capable of listing dbs.""" - - def global_info(self): - """Return list of dbs.""" - dbs = [] - for fname in os.listdir(self._workingdir): - p = os.path.join(self._workingdir, fname) - if os.path.isfile(p) and os.access(p, os.R_OK|os.W_OK): - try: - with open(p, 'rb') as f: - header = f.read(16) - if header == "SQLite format 3\000": - dbs.append(fname) - except IOError: - pass - return {"databases": dict.fromkeys(dbs), "db_count": len(dbs)} - - -def make_server(host, port, working_dir, accept_cors_connections=None): - """Make a server on host and port exposing dbs living in working_dir.""" - state = DbListingServerState() - state.set_workingdir(working_dir) - application = http_app.HTTPApp(state) - if accept_cors_connections: - application = cors_middleware.CORSMiddleware(application, - accept_cors_connections) - server = httpserver.WSGIServer(application, (host, port), - httpserver.WSGIHandler) - return server diff --git a/common/src/leap/soledad/common/l2db/errors.py b/common/src/leap/soledad/common/l2db/errors.py index e5ee8f45..b502fc2d 100644 --- a/common/src/leap/soledad/common/l2db/errors.py +++ b/common/src/leap/soledad/common/l2db/errors.py @@ -185,8 +185,7 @@ class UnknownAuthMethod(U1DBError): # mapping wire (transimission) descriptions/tags for errors to the exceptions wire_description_to_exc = dict( (x.wire_description, x) for x in globals().values() - if getattr(x, 'wire_description', None) not in (None, "error") -) + if getattr(x, 'wire_description', None) not in (None, "error")) wire_description_to_exc["error"] = U1DBError diff --git a/common/src/leap/soledad/common/l2db/query_parser.py b/common/src/leap/soledad/common/l2db/query_parser.py index 7f07b554..dd35b12a 100644 --- a/common/src/leap/soledad/common/l2db/query_parser.py +++ b/common/src/leap/soledad/common/l2db/query_parser.py @@ -358,8 +358,8 @@ class Parser(object): @classmethod def register_transormation(cls, transform): assert transform.name not in cls._transformations, ( - "Transform %s already registered for %s" - % (transform.name, cls._transformations[transform.name])) + "Transform %s already registered for %s" + % (transform.name, cls._transformations[transform.name])) cls._transformations[transform.name] = transform diff --git a/common/src/leap/soledad/common/l2db/remote/cors_middleware.py b/common/src/leap/soledad/common/l2db/remote/cors_middleware.py deleted file mode 100644 index 8041b968..00000000 --- a/common/src/leap/soledad/common/l2db/remote/cors_middleware.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. -"""U1DB Cross-Origin Resource Sharing WSGI middleware.""" - - -class CORSMiddleware(object): - """U1DB Cross-Origin Resource Sharing WSGI middleware.""" - - def __init__(self, app, accept_cors_connections): - self.origins = ' '.join(accept_cors_connections) - self.app = app - - def _cors_headers(self): - return [('access-control-allow-origin', self.origins), - ('access-control-allow-headers', - 'authorization, content-type, x-requested-with'), - ('access-control-allow-methods', - 'GET, POST, PUT, DELETE, OPTIONS')] - - def __call__(self, environ, start_response): - def wrap_start_response(status, headers, exc_info=None): - headers += self._cors_headers() - return start_response(status, headers, exc_info) - - if environ['REQUEST_METHOD'].lower() == 'options': - wrap_start_response("200 OK", [('content-type', 'text/plain')]) - return [''] - - return self.app(environ, wrap_start_response) diff --git a/common/src/leap/soledad/common/l2db/remote/http_app.py b/common/src/leap/soledad/common/l2db/remote/http_app.py index 3b65f5f7..65277bd1 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_app.py +++ b/common/src/leap/soledad/common/l2db/remote/http_app.py @@ -18,7 +18,6 @@ """ HTTP Application exposing U1DB. """ - # TODO -- deprecate, use twisted/txaio. import functools @@ -340,18 +339,18 @@ class DocResource(object): headers={ 'x-u1db-rev': '', 'x-u1db-has-conflicts': 'false' - }) + }) return headers = { 'x-u1db-rev': doc.rev, 'x-u1db-has-conflicts': json.dumps(doc.has_conflicts) - } + } if doc.is_tombstone(): self.responder.send_response_json( - http_errors.wire_description_to_status[ - errors.DOCUMENT_DELETED], - error=errors.DOCUMENT_DELETED, - headers=headers) + http_errors.wire_description_to_status[ + errors.DOCUMENT_DELETED], + error=errors.DOCUMENT_DELETED, + headers=headers) else: self.responder.send_response_content( doc.get_json(), headers=headers) @@ -431,7 +430,7 @@ class SyncResource(object): self.responder.start_response(200) self.responder.start_stream(), header = {"new_generation": new_gen, - "new_transaction_id": self.sync_exch.new_trans_id} + "new_transaction_id": self.sync_exch.new_trans_id} if self.replica_uid is not None: header['replica_uid'] = self.replica_uid self.responder.stream_entry(header) @@ -462,10 +461,11 @@ class HTTPResponder(object): return self._started = True status_text = httplib.responses[status] - self._write = self._start_response('%d %s' % (status, status_text), - [('content-type', self.content_type), - ('cache-control', 'no-cache')] + - headers.items()) + self._write = self._start_response( + '%d %s' % (status, status_text), + [('content-type', self.content_type), + ('cache-control', 'no-cache')] + + headers.items()) # xxx version in headers if obj_dic is not None: self._no_initial_obj = False diff --git a/common/src/leap/soledad/common/l2db/remote/http_database.py b/common/src/leap/soledad/common/l2db/remote/http_database.py index d8dcfd55..b2b48dee 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_database.py +++ b/common/src/leap/soledad/common/l2db/remote/http_database.py @@ -92,9 +92,9 @@ class HTTPDatabase(http_client.HTTPClientBase, Database): return None except errors.HTTPError, e: if (e.status == DOCUMENT_DELETED_STATUS and - 'x-u1db-rev' in e.headers): - res = None - headers = e.headers + 'x-u1db-rev' in e.headers): + res = None + headers = e.headers else: raise doc_rev = headers['x-u1db-rev'] diff --git a/common/src/leap/soledad/common/l2db/remote/http_target.py b/common/src/leap/soledad/common/l2db/remote/http_target.py index 598170e4..7e7f366f 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_target.py +++ b/common/src/leap/soledad/common/l2db/remote/http_target.py @@ -47,7 +47,7 @@ class HTTPSyncTarget(http_client.HTTPClientBase, SyncTarget): if self._trace_hook: # for tests self._trace_hook('record_sync_info') self._request_json('PUT', ['sync-from', source_replica_uid], {}, - {'generation': source_replica_generation, + {'generation': source_replica_generation, 'transaction_id': source_transaction_id}) def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None): diff --git a/common/src/leap/soledad/common/l2db/remote/oauth_middleware.py b/common/src/leap/soledad/common/l2db/remote/oauth_middleware.py deleted file mode 100644 index 5772580a..00000000 --- a/common/src/leap/soledad/common/l2db/remote/oauth_middleware.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. -"""U1DB OAuth authorisation WSGI middleware.""" -import httplib -from oauth import oauth -try: - import simplejson as json -except ImportError: - import json # noqa -from urllib import quote -from wsgiref.util import shift_path_info - - -sign_meth_HMAC_SHA1 = oauth.OAuthSignatureMethod_HMAC_SHA1() -sign_meth_PLAINTEXT = oauth.OAuthSignatureMethod_PLAINTEXT() - - -class OAuthMiddleware(object): - """U1DB OAuth Authorisation WSGI middleware.""" - - # max seconds the request timestamp is allowed to be shifted - # from arrival time - timestamp_threshold = 300 - - def __init__(self, app, base_url, prefix='/~/'): - self.app = app - self.base_url = base_url - self.prefix = prefix - - def get_oauth_data_store(self): - """Provide a oauth.OAuthDataStore.""" - raise NotImplementedError(self.get_oauth_data_store) - - def _error(self, start_response, status, description, message=None): - start_response("%d %s" % (status, httplib.responses[status]), - [('content-type', 'application/json')]) - err = {"error": description} - if message: - err['message'] = message - return [json.dumps(err)] - - def __call__(self, environ, start_response): - if self.prefix and not environ['PATH_INFO'].startswith(self.prefix): - return self._error(start_response, 400, "bad request") - headers = {} - if 'HTTP_AUTHORIZATION' in environ: - headers['Authorization'] = environ['HTTP_AUTHORIZATION'] - oauth_req = oauth.OAuthRequest.from_request( - http_method=environ['REQUEST_METHOD'], - http_url=self.base_url + environ['PATH_INFO'], - headers=headers, - query_string=environ['QUERY_STRING'] - ) - if oauth_req is None: - return self._error(start_response, 401, "unauthorized", - "Missing OAuth.") - try: - self.verify(environ, oauth_req) - except oauth.OAuthError, e: - return self._error(start_response, 401, "unauthorized", - e.message) - shift_path_info(environ) - return self.app(environ, start_response) - - def verify(self, environ, oauth_req): - """Verify OAuth request, put user_id in the environ.""" - oauth_server = oauth.OAuthServer(self.get_oauth_data_store()) - oauth_server.timestamp_threshold = self.timestamp_threshold - oauth_server.add_signature_method(sign_meth_HMAC_SHA1) - oauth_server.add_signature_method(sign_meth_PLAINTEXT) - consumer, token, parameters = oauth_server.verify_request(oauth_req) - # filter out oauth bits - environ['QUERY_STRING'] = '&'.join("%s=%s" % (quote(k, safe=''), - quote(v, safe='')) - for k, v in parameters.iteritems()) - return consumer, token diff --git a/common/src/leap/soledad/common/l2db/remote/server_state.py b/common/src/leap/soledad/common/l2db/remote/server_state.py index 6c1104c6..f131e09e 100644 --- a/common/src/leap/soledad/common/l2db/remote/server_state.py +++ b/common/src/leap/soledad/common/l2db/remote/server_state.py @@ -18,6 +18,7 @@ import os import errno + class ServerState(object): """Passed to a Request when it is instantiated. diff --git a/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py b/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py index fbabc177..ce82f1b2 100644 --- a/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py +++ b/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py @@ -52,13 +52,14 @@ def match_hostname(cert, hostname): return dnsnames.append(value) if len(dnsnames) > 1: - raise CertificateError("hostname %r " - "doesn't match either of %s" + raise CertificateError( + "hostname %r doesn't match either of %s" % (hostname, ', '.join(map(repr, dnsnames)))) elif len(dnsnames) == 1: - raise CertificateError("hostname %r " - "doesn't match %r" + raise CertificateError( + "hostname %r doesn't match %r" % (hostname, dnsnames[0])) else: - raise CertificateError("no appropriate commonName or " + raise CertificateError( + "no appropriate commonName or " "subjectAltName fields were found") diff --git a/common/src/leap/soledad/common/l2db/sync.py b/common/src/leap/soledad/common/l2db/sync.py index 26e67140..c612629f 100644 --- a/common/src/leap/soledad/common/l2db/sync.py +++ b/common/src/leap/soledad/common/l2db/sync.py @@ -53,7 +53,8 @@ class Synchronizer(object): """ # Increases self.num_inserted depending whether the document # was effectively inserted. - state, _ = self.source._put_doc_if_newer(doc, save_conflict=True, + state, _ = self.source._put_doc_if_newer( + doc, save_conflict=True, replica_uid=self.target_replica_uid, replica_gen=replica_gen, replica_trans_id=trans_id) if state == 'inserted': @@ -85,10 +86,10 @@ class Synchronizer(object): new generation. """ cur_gen, trans_id = self.source._get_generation_info() - if (cur_gen == start_generation + self.num_inserted - and self.num_inserted > 0): - self.sync_target.record_sync_info( - self.source._replica_uid, cur_gen, trans_id) + last_gen = start_generation + self.num_inserted + if (cur_gen == last_gen and self.num_inserted > 0): + self.sync_target.record_sync_info( + self.source._replica_uid, cur_gen, trans_id) def sync(self, callback=None, autocreate=False): """Synchronize documents between source and target.""" @@ -124,15 +125,17 @@ class Synchronizer(object): if self.target_replica_uid is None: target_last_known_gen, target_last_known_trans_id = 0, '' else: - target_last_known_gen, target_last_known_trans_id = \ - self.source._get_replica_gen_and_trans_id(self.target_replica_uid) + target_last_known_gen, target_last_known_trans_id = ( + self.source._get_replica_gen_and_trans_id( # nopep8 + self.target_replica_uid)) if not changes and target_last_known_gen == target_gen: if target_trans_id != target_last_known_trans_id: raise errors.InvalidTransactionId return my_gen changed_doc_ids = [doc_id for doc_id, _, _ in changes] # prepare to send all the changed docs - docs_to_send = self.source.get_docs(changed_doc_ids, + docs_to_send = self.source.get_docs( + changed_doc_ids, check_for_conflicts=False, include_deleted=True) # TODO: there must be a way to not iterate twice docs_by_generation = zip( @@ -172,7 +175,7 @@ class SyncExchange(object): self._db._last_exchange_log = { 'receive': {'docs': self._incoming_trace}, 'return': None - } + } def _set_trace_hook(self, cb): self._trace_hook = cb @@ -198,7 +201,8 @@ class SyncExchange(object): :param source_gen: The source generation of doc. :return: None """ - state, at_gen = self._db._put_doc_if_newer(doc, save_conflict=False, + state, at_gen = self._db._put_doc_if_newer( + doc, save_conflict=False, replica_uid=self.source_replica_uid, replica_gen=source_gen, replica_trans_id=trans_id) if state == 'inserted': @@ -217,7 +221,7 @@ class SyncExchange(object): self._db._last_exchange_log['receive'].update({ 'source_uid': self.source_replica_uid, 'source_gen': source_gen - }) + }) def find_changes_to_return(self): """Find changes to return. @@ -232,7 +236,7 @@ class SyncExchange(object): """ self._db._last_exchange_log['receive'].update({ # for tests 'last_known_gen': self.source_last_known_generation - }) + }) self._trace('before whats_changed') gen, trans_id, changes = self._db.whats_changed( self.source_last_known_generation) @@ -242,9 +246,9 @@ class SyncExchange(object): seen_ids = self.seen_ids # changed docs that weren't superseded by or converged with self.changes_to_return = [ - (doc_id, gen, trans_id) for (doc_id, gen, trans_id) in changes + (doc_id, gen, trans_id) for (doc_id, gen, trans_id) in changes if # there was a subsequent update - if doc_id not in seen_ids or seen_ids.get(doc_id) < gen] + doc_id not in seen_ids or seen_ids.get(doc_id) < gen] return self.new_gen def return_docs(self, return_doc_cb): |