From e91ad8c99c7abc99a97b8afb6ac7e3b7e729b219 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 9 Dec 2013 16:57:55 -0400 Subject: pep8 happy --- server/src/leap/soledad/server/__init__.py | 1 - server/src/leap/soledad/server/_version.py | 35 +++++++++++++++++------------- 2 files changed, 20 insertions(+), 16 deletions(-) (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index c80b4c68..9cad6093 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -183,7 +183,6 @@ class LockResource(object): FILESYSTEM_LOCK_TRIES = 5 FILESYSTEM_LOCK_SLEEP_SECONDS = 1 - def __init__(self, uuid, state, responder): """ Initialize the lock resource. Parameters to this constructor are diff --git a/server/src/leap/soledad/server/_version.py b/server/src/leap/soledad/server/_version.py index 85f0e54c..589d42d2 100644 --- a/server/src/leap/soledad/server/_version.py +++ b/server/src/leap/soledad/server/_version.py @@ -17,6 +17,7 @@ git_full = "$Format:%H$" import subprocess import sys + def run_command(args, cwd=None, verbose=False): try: # remember shell=False, so use git.cmd on windows, not just git @@ -36,11 +37,10 @@ def run_command(args, cwd=None, verbose=False): return None return stdout - -import sys import re import os.path + def get_expanded_variables(versionfile_source): # the code embedded in _version.py can just fetch the value of these # variables. When used from setup.py, we don't want to import @@ -48,7 +48,7 @@ def get_expanded_variables(versionfile_source): # used from _version.py. variables = {} try: - f = open(versionfile_source,"r") + f = open(versionfile_source, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) @@ -63,12 +63,13 @@ def get_expanded_variables(versionfile_source): pass return variables + def versions_from_expanded_variables(variables, tag_prefix, verbose=False): refnames = variables["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + return {} # unexpanded, so not in an unpacked git-archive tarball refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -93,13 +94,14 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) - return { "version": r, - "full": variables["full"].strip() } + return {"version": r, + "full": variables["full"].strip()} # no suitable tags, so we use the full revision id if verbose: print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } + return {"version": variables["full"].strip(), + "full": variables["full"].strip()} + def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): # this runs 'git' from the root of the source tree. That either means @@ -116,7 +118,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): here = os.path.abspath(__file__) except NameError: # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct + return {} # not always correct # versionfile_source is the relative path from the top of the source tree # (where the .git directory might live) to this file. Invert this to find @@ -141,7 +143,8 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): return {} if not stdout.startswith(tag_prefix): if verbose: - print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) + print("tag '%s' doesn't start with prefix '%s'" % + (stdout, tag_prefix)) return {} tag = stdout[len(tag_prefix):] stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) @@ -153,7 +156,8 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): return {"version": tag, "full": full} -def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): +def versions_from_parentdir(parentdir_prefix, versionfile_source, + verbose=False): if IN_LONG_VERSION_PY: # We're running from _version.py. If it's from a source tree # (execute-in-place), we can work upwards to find the root of the @@ -163,7 +167,7 @@ def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False) here = os.path.abspath(__file__) except NameError: # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope + return {} # without __file__, we have no hope # versionfile_source is the relative path from the top of the source # tree to _version.py. Invert this to find the root from __file__. root = here @@ -180,7 +184,8 @@ def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False) dirname = os.path.basename(root) if not dirname.startswith(parentdir_prefix): if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % + print("guessing rootdir is '%s', but '%s' doesn't start " + "with prefix '%s'" % (root, dirname, parentdir_prefix)) return None return {"version": dirname[len(parentdir_prefix):], "full": ""} @@ -189,8 +194,9 @@ tag_prefix = "" parentdir_prefix = "leap.soledad.server-" versionfile_source = "src/leap/soledad/server/_version.py" + def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = { "refnames": git_refnames, "full": git_full } + variables = {"refnames": git_refnames, "full": git_full} ver = versions_from_expanded_variables(variables, tag_prefix, verbose) if not ver: ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) @@ -200,4 +206,3 @@ def get_versions(default={"version": "unknown", "full": ""}, verbose=False): if not ver: ver = default return ver - -- cgit v1.2.3 From 678858255bb046f4dd5cbdb51ce9820ce9f651a0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 13 Dec 2013 07:00:45 -0400 Subject: enable gzip compression --- server/src/leap/soledad/server/__init__.py | 7 ++- server/src/leap/soledad/server/gzip.py | 69 ++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 server/src/leap/soledad/server/gzip.py (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 9cad6093..de5daf62 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -110,6 +110,8 @@ if version.base() == "12.0.0": sys.modules['OpenSSL.tsafe'] = old_tsafe from leap.soledad.server.auth import SoledadTokenAuthMiddleware +from leap.soledad.server.gzip import GzipMiddleware + from leap.soledad.common import ( SHARED_DB_NAME, SHARED_DB_LOCK_DOC_ID_PREFIX, @@ -378,8 +380,9 @@ def application(environ, start_response): SoledadApp.SHARED_DB_NAME, SoledadTokenAuthMiddleware.TOKENS_DB) # WSGI application that may be used by `twistd -web` - application = SoledadTokenAuthMiddleware(SoledadApp(state)) - resource = WSGIResource(reactor, reactor.getThreadPool(), application) + application = GzipMiddleware( + SoledadTokenAuthMiddleware(SoledadApp(state))) + return application(environ, start_response) diff --git a/server/src/leap/soledad/server/gzip.py b/server/src/leap/soledad/server/gzip.py new file mode 100644 index 00000000..92906513 --- /dev/null +++ b/server/src/leap/soledad/server/gzip.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# gzip.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Gzip middleware for WSGI apps. +""" +import gzip +import StringIO + + +class GzipMiddleware(object): + """ + GzipMiddleware class for WSGI. + """ + def __init__(self, app, compresslevel=9): + self.app = app + self.compresslevel = compresslevel + + def __call__(self, environ, start_response): + if 'gzip' not in environ.get('HTTP_ACCEPT_ENCODING', ''): + return self.app(environ, start_response) + if (environ['PATH_INFO'][-3:] != '.js' and environ[ + 'PATH_INFO'][-4:] != '.css'): + return self.app(environ, start_response) + buffer = StringIO.StringIO() + output = gzip.GzipFile( + mode='wb', + compresslevel=self.compresslevel, + fileobj=buffer + ) + + start_response_args = [] + + def dummy_start_response(status, headers, exc_info=None): + start_response_args.append(status) + start_response_args.append(headers) + start_response_args.append(exc_info) + return output.write + + app_iter = self.app(environ, dummy_start_response) + for line in app_iter: + output.write(line) + if hasattr(app_iter, 'close'): + app_iter.close() + output.close() + buffer.seek(0) + result = buffer.getvalue() + headers = [] + for name, value in start_response_args[1]: + if name.lower() != 'content-length': + headers.append((name, value)) + headers.append(('Content-Length', str(len(result)))) + headers.append(('Content-Encoding', 'gzip')) + start_response(start_response_args[0], headers, start_response_args[2]) + buffer.close() + return [result] -- cgit v1.2.3 From 45241ec8618d24b61b768bc7027473b52945609f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 16 Dec 2013 13:21:19 -0400 Subject: patch the _version file so it reports the running version until now this was only possible when running python setup.py version from the source tree. now the .__version__ also reports correctly the runnng version --- server/src/leap/soledad/server/_version.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/_version.py b/server/src/leap/soledad/server/_version.py index 589d42d2..ec611c39 100644 --- a/server/src/leap/soledad/server/_version.py +++ b/server/src/leap/soledad/server/_version.py @@ -128,7 +128,16 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): for i in range(len(versionfile_source.split("/"))): root = os.path.dirname(root) else: - root = os.path.dirname(here) + root = os.path.dirname( + os.path.join('..', here)) + + ###################################################### + # XXX patch for our specific configuration with + # the three projects leap.soledad.{common, client, server} + # inside the same repo. + ###################################################### + root = os.path.dirname(os.path.join('..', root)) + if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) -- cgit v1.2.3 From 7b72e60911009e282520c5d8b6c6edd75ab48fd2 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 18 Dec 2013 12:37:44 -0400 Subject: remove conditional gzip, fix import --- server/src/leap/soledad/server/__init__.py | 2 +- server/src/leap/soledad/server/gzip.py | 69 ----------------------- server/src/leap/soledad/server/gzip_middleware.py | 68 ++++++++++++++++++++++ 3 files changed, 69 insertions(+), 70 deletions(-) delete mode 100644 server/src/leap/soledad/server/gzip.py create mode 100644 server/src/leap/soledad/server/gzip_middleware.py (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index de5daf62..a4b25fe2 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -110,7 +110,7 @@ if version.base() == "12.0.0": sys.modules['OpenSSL.tsafe'] = old_tsafe from leap.soledad.server.auth import SoledadTokenAuthMiddleware -from leap.soledad.server.gzip import GzipMiddleware +from leap.soledad.server.gzip_middleware import GzipMiddleware from leap.soledad.common import ( SHARED_DB_NAME, diff --git a/server/src/leap/soledad/server/gzip.py b/server/src/leap/soledad/server/gzip.py deleted file mode 100644 index 92906513..00000000 --- a/server/src/leap/soledad/server/gzip.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -# gzip.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Gzip middleware for WSGI apps. -""" -import gzip -import StringIO - - -class GzipMiddleware(object): - """ - GzipMiddleware class for WSGI. - """ - def __init__(self, app, compresslevel=9): - self.app = app - self.compresslevel = compresslevel - - def __call__(self, environ, start_response): - if 'gzip' not in environ.get('HTTP_ACCEPT_ENCODING', ''): - return self.app(environ, start_response) - if (environ['PATH_INFO'][-3:] != '.js' and environ[ - 'PATH_INFO'][-4:] != '.css'): - return self.app(environ, start_response) - buffer = StringIO.StringIO() - output = gzip.GzipFile( - mode='wb', - compresslevel=self.compresslevel, - fileobj=buffer - ) - - start_response_args = [] - - def dummy_start_response(status, headers, exc_info=None): - start_response_args.append(status) - start_response_args.append(headers) - start_response_args.append(exc_info) - return output.write - - app_iter = self.app(environ, dummy_start_response) - for line in app_iter: - output.write(line) - if hasattr(app_iter, 'close'): - app_iter.close() - output.close() - buffer.seek(0) - result = buffer.getvalue() - headers = [] - for name, value in start_response_args[1]: - if name.lower() != 'content-length': - headers.append((name, value)) - headers.append(('Content-Length', str(len(result)))) - headers.append(('Content-Encoding', 'gzip')) - start_response(start_response_args[0], headers, start_response_args[2]) - buffer.close() - return [result] diff --git a/server/src/leap/soledad/server/gzip_middleware.py b/server/src/leap/soledad/server/gzip_middleware.py new file mode 100644 index 00000000..5a424894 --- /dev/null +++ b/server/src/leap/soledad/server/gzip_middleware.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# gzip_middleware.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Gzip middleware for WSGI apps. +""" +import StringIO +from gzip import GzipFile + + +class GzipMiddleware(object): + """ + GzipMiddleware class for WSGI. + """ + def __init__(self, app, compresslevel=9): + self.app = app + self.compresslevel = compresslevel + + def __call__(self, environ, start_response): + print "environ: ", environ + if 'gzip' not in environ.get('HTTP_ACCEPT_ENCODING', ''): + return self.app(environ, start_response) + + buffer = StringIO.StringIO() + output = GzipFile( + mode='wb', + compresslevel=self.compresslevel, + fileobj=buffer + ) + + start_response_args = [] + + def dummy_start_response(status, headers, exc_info=None): + start_response_args.append(status) + start_response_args.append(headers) + start_response_args.append(exc_info) + return output.write + + app_iter = self.app(environ, dummy_start_response) + for line in app_iter: + output.write(line) + if hasattr(app_iter, 'close'): + app_iter.close() + output.close() + buffer.seek(0) + result = buffer.getvalue() + headers = [] + for name, value in start_response_args[1]: + if name.lower() != 'content-length': + headers.append((name, value)) + headers.append(('Content-Length', str(len(result)))) + headers.append(('Content-Encoding', 'gzip')) + start_response(start_response_args[0], headers, start_response_args[2]) + buffer.close() + return [result] -- cgit v1.2.3 From ecc05503b393844ee8b864db9d32db288b875d11 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 18 Dec 2013 12:48:00 -0400 Subject: add http-encoding header, decode it too --- server/src/leap/soledad/server/gzip_middleware.py | 1 - 1 file changed, 1 deletion(-) (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/gzip_middleware.py b/server/src/leap/soledad/server/gzip_middleware.py index 5a424894..986c5738 100644 --- a/server/src/leap/soledad/server/gzip_middleware.py +++ b/server/src/leap/soledad/server/gzip_middleware.py @@ -30,7 +30,6 @@ class GzipMiddleware(object): self.compresslevel = compresslevel def __call__(self, environ, start_response): - print "environ: ", environ if 'gzip' not in environ.get('HTTP_ACCEPT_ENCODING', ''): return self.app(environ, start_response) -- cgit v1.2.3 From 89d3e4a1321ff9701ac67933f8e649cfecd1d95e Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jan 2014 10:29:43 -0200 Subject: Add proper error reporting to shared db lock. --- server/src/leap/soledad/server/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index a4b25fe2..00d3c8c1 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -121,6 +121,7 @@ from leap.soledad.common.errors import ( InvalidTokenError, NotLockedError, AlreadyLockedError, + LockTimedOutError, ) @@ -225,7 +226,9 @@ class LockResource(object): """ # obtain filesystem lock if not self._try_obtain_filesystem_lock(): - self._responder.send_response_json(408) # error: request timeout + self._responder.send_response_json( + LockTimedOutError.status, # error: request timeout + error=LockTimedOutError.wire_description) return created_lock = False -- cgit v1.2.3 From 82628d8284c5c11452da75a6604f2f68b8dd8520 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 7 Jan 2014 11:05:39 -0200 Subject: Use temp dir for server side locks (#4918). --- server/src/leap/soledad/server/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 00d3c8c1..4380966c 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -90,6 +90,7 @@ import configparser import time import hashlib import os +import tempfile from u1db.remote import http_app @@ -122,6 +123,7 @@ from leap.soledad.common.errors import ( NotLockedError, AlreadyLockedError, LockTimedOutError, + CouldNotObtainLockError, ) @@ -201,7 +203,9 @@ class LockResource(object): self._shared_db = state.open_database(SoledadApp.SHARED_DB_NAME) self._lock_doc_id = '%s%s' % (SHARED_DB_LOCK_DOC_ID_PREFIX, uuid) self._lock = FilesystemLock( - hashlib.sha512(self._lock_doc_id).hexdigest()) + os.path.join( + tempfile.gettmpdir(), + hashlib.sha512(self._lock_doc_id).hexdigest())) self._state = state self._responder = responder @@ -328,8 +332,10 @@ class LockResource(object): while tries > 0: try: return self._lock.lock() - except Exception as e: + except OSError as e: tries -= 1 + if tries == 0: + raise CouldNotObtainLockError(e.message) time.sleep(self.FILESYSTEM_LOCK_SLEEP_SECONDS) return False -- cgit v1.2.3 From 7741a372157972dfe6a46a16ccdf860df1faa943 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 16 Jan 2014 19:19:17 -0200 Subject: Fix typo on tempfile.gettempdir() call. --- server/src/leap/soledad/server/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 4380966c..26c33222 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -204,7 +204,7 @@ class LockResource(object): self._lock_doc_id = '%s%s' % (SHARED_DB_LOCK_DOC_ID_PREFIX, uuid) self._lock = FilesystemLock( os.path.join( - tempfile.gettmpdir(), + tempfile.gettempdir(), hashlib.sha512(self._lock_doc_id).hexdigest())) self._state = state self._responder = responder -- cgit v1.2.3 From 75e9eb251100a2c0e5a93deedbc9fce67f7f1fe6 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 25 Dec 2013 20:59:08 -0200 Subject: Move lock resource to its own file. --- server/src/leap/soledad/server/__init__.py | 211 +-------------------- server/src/leap/soledad/server/lock_resource.py | 232 ++++++++++++++++++++++++ 2 files changed, 235 insertions(+), 208 deletions(-) create mode 100644 server/src/leap/soledad/server/lock_resource.py (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 26c33222..b2d55fcd 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -87,10 +87,6 @@ and lock documents on the shared database is handled by """ import configparser -import time -import hashlib -import os -import tempfile from u1db.remote import http_app @@ -99,10 +95,6 @@ from u1db.remote import http_app from OpenSSL import tsafe old_tsafe = tsafe -from twisted.web.wsgi import WSGIResource -from twisted.internet import reactor -from twisted.internet.error import TimeoutError -from twisted.python.lockfile import FilesystemLock from twisted import version if version.base() == "12.0.0": # Put OpenSSL's tsafe back into place. This can probably be removed if we @@ -112,19 +104,10 @@ if version.base() == "12.0.0": from leap.soledad.server.auth import SoledadTokenAuthMiddleware from leap.soledad.server.gzip_middleware import GzipMiddleware +from leap.soledad.server.lock_resource import LockResource -from leap.soledad.common import ( - SHARED_DB_NAME, - SHARED_DB_LOCK_DOC_ID_PREFIX, -) +from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common.couch import CouchServerState -from leap.soledad.common.errors import ( - InvalidTokenError, - NotLockedError, - AlreadyLockedError, - LockTimedOutError, - CouldNotObtainLockError, -) #----------------------------------------------------------------------------- @@ -159,195 +142,7 @@ class SoledadApp(http_app.HTTPApp): return http_app.HTTPApp.__call__(self, environ, start_response) -# -# LockResource: a lock based on a document in the shared database. -# - -@http_app.url_to_resource.register -class LockResource(object): - """ - Handle requests for locking documents. - - This class uses Twisted's Filesystem lock to manage a lock in the shared - database. - """ - - url_pattern = '/%s/lock/{uuid}' % SoledadApp.SHARED_DB_NAME - """ - """ - - TIMEOUT = 300 # XXX is 5 minutes reasonable? - """ - The timeout after which the lock expires. - """ - - # used for lock doc storage - TIMESTAMP_KEY = '_timestamp' - LOCK_TOKEN_KEY = '_token' - - FILESYSTEM_LOCK_TRIES = 5 - FILESYSTEM_LOCK_SLEEP_SECONDS = 1 - - def __init__(self, uuid, state, responder): - """ - Initialize the lock resource. Parameters to this constructor are - automatically passed by u1db. - - :param uuid: The user unique id. - :type uuid: str - :param state: The backend database state. - :type state: u1db.remote.ServerState - :param responder: The infrastructure to send responses to client. - :type responder: u1db.remote.HTTPResponder - """ - self._shared_db = state.open_database(SoledadApp.SHARED_DB_NAME) - self._lock_doc_id = '%s%s' % (SHARED_DB_LOCK_DOC_ID_PREFIX, uuid) - self._lock = FilesystemLock( - os.path.join( - tempfile.gettempdir(), - hashlib.sha512(self._lock_doc_id).hexdigest())) - self._state = state - self._responder = responder - - @http_app.http_method(content=str) - def put(self, content=None): - """ - Handle a PUT request to the lock document. - - A lock is a document in the shared db with doc_id equal to - 'lock-' and the timestamp of its creation as content. This - method obtains a threaded-lock and creates a lock document if it does - not exist or if it has expired. - - It returns '201 Created' and a pair containing a token to unlock and - the lock timeout, or '403 AlreadyLockedError' and the remaining amount - of seconds the lock will still be valid. - - :param content: The content of the PUT request. It is only here - because PUT requests with empty content are considered - invalid requests by u1db. - :type content: str - """ - # obtain filesystem lock - if not self._try_obtain_filesystem_lock(): - self._responder.send_response_json( - LockTimedOutError.status, # error: request timeout - error=LockTimedOutError.wire_description) - return - - created_lock = False - now = time.time() - token = hashlib.sha256(os.urandom(10)).hexdigest() # for releasing - lock_doc = self._shared_db.get_doc(self._lock_doc_id) - remaining = self._remaining(lock_doc, now) - - # if there's no lock, create one - if lock_doc is None: - lock_doc = self._shared_db.create_doc( - { - self.TIMESTAMP_KEY: now, - self.LOCK_TOKEN_KEY: token, - }, - doc_id=self._lock_doc_id) - created_lock = True - else: - if remaining == 0: - # lock expired, create new one - lock_doc.content = { - self.TIMESTAMP_KEY: now, - self.LOCK_TOKEN_KEY: token, - } - self._shared_db.put_doc(lock_doc) - created_lock = True - - self._try_release_filesystem_lock() - - # send response to client - if created_lock is True: - self._responder.send_response_json( - 201, timeout=self.TIMEOUT, token=token) # success: created - else: - wire_descr = AlreadyLockedError.wire_description - self._responder.send_response_json( - AlreadyLockedError.status, # error: forbidden - error=AlreadyLockedError.wire_description, remaining=remaining) - - @http_app.http_method(token=str) - def delete(self, token=None): - """ - Delete the lock if the C{token} is valid. - - Delete the lock document in case C{token} is equal to the token stored - in the lock document. - - :param token: The token returned when locking. - :type token: str - - :raise NotLockedError: Raised in case the lock is not locked. - :raise InvalidTokenError: Raised in case the token is invalid for - unlocking. - """ - lock_doc = self._shared_db.get_doc(self._lock_doc_id) - if lock_doc is None or self._remaining(lock_doc, time.time()) == 0: - self._responder.send_response_json( - NotLockedError.status, # error: not found - error=NotLockedError.wire_description) - elif token != lock_doc.content[self.LOCK_TOKEN_KEY]: - self._responder.send_response_json( - InvalidTokenError.status, # error: unauthorized - error=InvalidTokenError.wire_description) - else: - self._shared_db.delete_doc(lock_doc) - self._responder.send_response_json(200) # success: should use 204 - # but u1db does not - # support it. - - def _remaining(self, lock_doc, now): - """ - Return the number of seconds the lock contained in C{lock_doc} is - still valid, when compared to C{now}. - - :param lock_doc: The document containing the lock. - :type lock_doc: u1db.Document - :param now: The time to which to compare the lock timestamp. - :type now: float - - :return: The amount of seconds the lock is still valid. - :rtype: float - """ - if lock_doc is not None: - lock_timestamp = lock_doc.content[self.TIMESTAMP_KEY] - remaining = lock_timestamp + self.TIMEOUT - now - return remaining if remaining > 0 else 0.0 - return 0.0 - - def _try_obtain_filesystem_lock(self): - """ - Try to obtain the file system lock. - - @return: Whether the lock was succesfully obtained. - @rtype: bool - """ - tries = self.FILESYSTEM_LOCK_TRIES - while tries > 0: - try: - return self._lock.lock() - except OSError as e: - tries -= 1 - if tries == 0: - raise CouldNotObtainLockError(e.message) - time.sleep(self.FILESYSTEM_LOCK_SLEEP_SECONDS) - return False - - def _try_release_filesystem_lock(self): - """ - Release the filesystem lock. - """ - try: - self._lock.unlock() - return True - except Exception: - return False +http_app.url_to_resource.register(LockResource) #----------------------------------------------------------------------------- diff --git a/server/src/leap/soledad/server/lock_resource.py b/server/src/leap/soledad/server/lock_resource.py new file mode 100644 index 00000000..a7870f77 --- /dev/null +++ b/server/src/leap/soledad/server/lock_resource.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- +# lock_resource.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +LockResource: a lock based on a document in the shared database. +""" + + +import hashlib +import time +import os +import tempfile +import errno + + +from u1db.remote import http_app +from twisted.python.lockfile import FilesystemLock + + +from leap.soledad.common import ( + SHARED_DB_NAME, + SHARED_DB_LOCK_DOC_ID_PREFIX, +) +from leap.soledad.common.errors import ( + InvalidTokenError, + NotLockedError, + AlreadyLockedError, + LockTimedOutError, + CouldNotObtainLockError, +) + + +class LockResource(object): + """ + Handle requests for locking documents. + + This class uses Twisted's Filesystem lock to manage a lock in the shared + database. + """ + + url_pattern = '/%s/lock/{uuid}' % SHARED_DB_NAME + """ + """ + + TIMEOUT = 300 # XXX is 5 minutes reasonable? + """ + The timeout after which the lock expires. + """ + + # used for lock doc storage + TIMESTAMP_KEY = '_timestamp' + LOCK_TOKEN_KEY = '_token' + + FILESYSTEM_LOCK_TRIES = 5 + FILESYSTEM_LOCK_SLEEP_SECONDS = 1 + + def __init__(self, uuid, state, responder): + """ + Initialize the lock resource. Parameters to this constructor are + automatically passed by u1db. + + :param uuid: The user unique id. + :type uuid: str + :param state: The backend database state. + :type state: u1db.remote.ServerState + :param responder: The infrastructure to send responses to client. + :type responder: u1db.remote.HTTPResponder + """ + self._shared_db = state.open_database(SHARED_DB_NAME) + self._lock_doc_id = '%s%s' % (SHARED_DB_LOCK_DOC_ID_PREFIX, uuid) + self._lock = FilesystemLock( + os.path.join( + tempfile.gettempdir(), + hashlib.sha512(self._lock_doc_id).hexdigest())) + self._state = state + self._responder = responder + + @http_app.http_method(content=str) + def put(self, content=None): + """ + Handle a PUT request to the lock document. + + A lock is a document in the shared db with doc_id equal to + 'lock-' and the timestamp of its creation as content. This + method obtains a threaded-lock and creates a lock document if it does + not exist or if it has expired. + + It returns '201 Created' and a pair containing a token to unlock and + the lock timeout, or '403 AlreadyLockedError' and the remaining amount + of seconds the lock will still be valid. + + :param content: The content of the PUT request. It is only here + because PUT requests with empty content are considered + invalid requests by u1db. + :type content: str + """ + # obtain filesystem lock + if not self._try_obtain_filesystem_lock(): + self._responder.send_response_json( + LockTimedOutError.status, # error: request timeout + error=LockTimedOutError.wire_description) + return + + created_lock = False + now = time.time() + token = hashlib.sha256(os.urandom(10)).hexdigest() # for releasing + lock_doc = self._shared_db.get_doc(self._lock_doc_id) + remaining = self._remaining(lock_doc, now) + + # if there's no lock, create one + if lock_doc is None: + lock_doc = self._shared_db.create_doc( + { + self.TIMESTAMP_KEY: now, + self.LOCK_TOKEN_KEY: token, + }, + doc_id=self._lock_doc_id) + created_lock = True + else: + if remaining == 0: + # lock expired, create new one + lock_doc.content = { + self.TIMESTAMP_KEY: now, + self.LOCK_TOKEN_KEY: token, + } + self._shared_db.put_doc(lock_doc) + created_lock = True + + self._try_release_filesystem_lock() + + # send response to client + if created_lock is True: + self._responder.send_response_json( + 201, timeout=self.TIMEOUT, token=token) # success: created + else: + self._responder.send_response_json( + AlreadyLockedError.status, # error: forbidden + error=AlreadyLockedError.wire_description, remaining=remaining) + + @http_app.http_method(token=str) + def delete(self, token=None): + """ + Delete the lock if the C{token} is valid. + + Delete the lock document in case C{token} is equal to the token stored + in the lock document. + + :param token: The token returned when locking. + :type token: str + + :raise NotLockedError: Raised in case the lock is not locked. + :raise InvalidTokenError: Raised in case the token is invalid for + unlocking. + """ + lock_doc = self._shared_db.get_doc(self._lock_doc_id) + if lock_doc is None or self._remaining(lock_doc, time.time()) == 0: + self._responder.send_response_json( + NotLockedError.status, # error: not found + error=NotLockedError.wire_description) + elif token != lock_doc.content[self.LOCK_TOKEN_KEY]: + self._responder.send_response_json( + InvalidTokenError.status, # error: unauthorized + error=InvalidTokenError.wire_description) + else: + self._shared_db.delete_doc(lock_doc) + self._responder.send_response_json(200) # success: should use 204 + # but u1db does not + # support it. + + def _remaining(self, lock_doc, now): + """ + Return the number of seconds the lock contained in C{lock_doc} is + still valid, when compared to C{now}. + + :param lock_doc: The document containing the lock. + :type lock_doc: u1db.Document + :param now: The time to which to compare the lock timestamp. + :type now: float + + :return: The amount of seconds the lock is still valid. + :rtype: float + """ + if lock_doc is not None: + lock_timestamp = lock_doc.content[self.TIMESTAMP_KEY] + remaining = lock_timestamp + self.TIMEOUT - now + return remaining if remaining > 0 else 0.0 + return 0.0 + + def _try_obtain_filesystem_lock(self): + """ + Try to obtain the file system lock. + + @return: Whether the lock was succesfully obtained. + @rtype: bool + """ + tries = self.FILESYSTEM_LOCK_TRIES + while tries > 0: + try: + return self._lock.lock() + except OSError as e: + tries -= 1 + if tries == 0: + raise CouldNotObtainLockError(e.message) + time.sleep(self.FILESYSTEM_LOCK_SLEEP_SECONDS) + return False + + def _try_release_filesystem_lock(self): + """ + Release the filesystem lock. + """ + try: + self._lock.unlock() + return True + except OSError as e: + if e.errno == errno.ENOENT: + return True + return False -- cgit v1.2.3 From f125180609c3e97f56eedfe534f7f1c6f985f8f6 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 25 Dec 2013 21:02:58 -0200 Subject: Allow sync of large files (~100MB) (#4836). --- server/src/leap/soledad/server/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index b2d55fcd..84f6a849 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -114,6 +114,10 @@ from leap.soledad.common.couch import CouchServerState # Soledad WSGI application #----------------------------------------------------------------------------- +MAX_REQUEST_SIZE = 200 # in Mb +MAX_ENTRY_SIZE = 200 # in Mb + + class SoledadApp(http_app.HTTPApp): """ Soledad WSGI application @@ -124,6 +128,9 @@ class SoledadApp(http_app.HTTPApp): The name of the shared database that holds user's encrypted secrets. """ + max_request_size = MAX_REQUEST_SIZE * 1024 * 1024 + max_entry_size = MAX_ENTRY_SIZE * 1024 * 1024 + def __call__(self, environ, start_response): """ Handle a WSGI call to the Soledad application. @@ -143,6 +150,8 @@ class SoledadApp(http_app.HTTPApp): http_app.url_to_resource.register(LockResource) +http_app.SyncResource.max_request_size = MAX_REQUEST_SIZE * 1024 * 1024 +http_app.SyncResource.max_entry_size = MAX_ENTRY_SIZE * 1024 * 1024 #----------------------------------------------------------------------------- -- cgit v1.2.3 From c97f48660a1aaad96f7356ac1a5fce6265241e0f Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 24 Jan 2014 19:09:46 -0200 Subject: Improve unauthorized error messages. --- server/src/leap/soledad/server/auth.py | 62 ++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 25 deletions(-) (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 0ae49576..11805005 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -36,19 +36,10 @@ from leap.soledad.common import ( SHARED_DB_NAME, SHARED_DB_LOCK_DOC_ID_PREFIX, USER_DB_PREFIX, + errors, ) -#----------------------------------------------------------------------------- -# Authentication -#----------------------------------------------------------------------------- - -class Unauthorized(Exception): - """ - User authentication failed. - """ - - class URLToAuthorization(object): """ Verify if actions can be performed by a user. @@ -279,10 +270,16 @@ class SoledadAuthMiddleware(object): return self._unauthorized_error("Wrong authentication scheme") # verify if user is athenticated - if not self._verify_authentication_data(uuid, auth_data): - return self._unauthorized_error( + try: + if not self._verify_authentication_data(uuid, auth_data): + return self._unauthorized_error( + start_response, + self._get_auth_error_string()) + except Unauthorized as e: + return self._error( start_response, - self._get_auth_error_string()) + 401, + e.wire_description) # verify if user is authorized to perform action if not self._verify_authorization(environ, uuid): @@ -319,6 +316,9 @@ class SoledadAuthMiddleware(object): @return: Whether the token is valid for authenticating the request. @rtype: bool + + @raise Unauthorized: Raised when C{auth_data} is not enough to + authenticate C{uuid}. """ return None @@ -386,9 +386,20 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): @return: Whether the token is valid for authenticating the request. @rtype: bool + + @raise Unauthorized: Raised when C{auth_data} is not enough to + authenticate C{uuid}. """ token = auth_data # we expect a cleartext token at this point - return self._verify_token_in_couchdb(uuid, token) + try: + return self._verify_token_in_couchdb(uuid, token) + except MissingAuthTokenError(): + raise + except TokenMismatchError(): + raise + except Exception as e: + log.err(e) + return False def _verify_token_in_couchdb(self, uuid, token): """ @@ -398,19 +409,20 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): @type uuid: str @param token: The token. @type token: str + + @raise MissingAuthTokenError: Raised when given token is missing in + tokens db. + @raise InvalidAuthTokenError: Raised when token is invalid. """ server = Server(url=self._app.state.couch_url) - try: - dbname = self.TOKENS_DB - db = server[dbname] - token = db.get(token) - if token is None: - return False - return token[self.TOKENS_TYPE_KEY] == self.TOKENS_TYPE_DEF and \ - token[self.TOKENS_USER_ID_KEY] == uuid - except Exception as e: - log.err(e) - return False + dbname = self.TOKENS_DB + db = server[dbname] + token = db.get(token) + if token is None: + raise MissingAuthTokenError() + if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF or \ + token[self.TOKENS_USER_ID_KEY] != uuid: + raise InvalidAuthTokenError() return True def _get_auth_error_string(self): -- cgit v1.2.3 From 1a5f426b70ea690b893438d21eb404474126280b Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 12 Mar 2014 19:41:09 -0300 Subject: Do not ensure shared database upon Soledad Server start (#5302). --- server/src/leap/soledad/server/__init__.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 84f6a849..c170f230 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -144,8 +144,6 @@ class SoledadApp(http_app.HTTPApp): @return: HTTP application results. @rtype: list """ - # ensure the shared database exists - self.state.ensure_database(self.SHARED_DB_NAME) return http_app.HTTPApp.__call__(self, environ, start_response) -- cgit v1.2.3 From 288f506daed66e4acb08617dc1db127da4d36241 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 13 Mar 2014 15:08:55 -0300 Subject: Fix raising of auth token errors (#5191). --- server/src/leap/soledad/server/auth.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) (limited to 'server/src/leap/soledad') diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 11805005..e9d2b032 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -25,7 +25,7 @@ import httplib import simplejson as json -from u1db import DBNAME_CONSTRAINTS +from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors from abc import ABCMeta, abstractmethod from routes.mapper import Mapper from couchdb.client import Server @@ -36,8 +36,8 @@ from leap.soledad.common import ( SHARED_DB_NAME, SHARED_DB_LOCK_DOC_ID_PREFIX, USER_DB_PREFIX, - errors, ) +from leap.soledad.common.errors import InvalidAuthTokenError class URLToAuthorization(object): @@ -275,7 +275,7 @@ class SoledadAuthMiddleware(object): return self._unauthorized_error( start_response, self._get_auth_error_string()) - except Unauthorized as e: + except u1db_errors.Unauthorized as e: return self._error( start_response, 401, @@ -392,16 +392,14 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): """ token = auth_data # we expect a cleartext token at this point try: - return self._verify_token_in_couchdb(uuid, token) - except MissingAuthTokenError(): - raise - except TokenMismatchError(): + return self._verify_token_in_couch(uuid, token) + except InvalidAuthTokenError: raise except Exception as e: log.err(e) return False - def _verify_token_in_couchdb(self, uuid, token): + def _verify_token_in_couch(self, uuid, token): """ Query couchdb to decide if C{token} is valid for C{uuid}. @@ -410,17 +408,16 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): @param token: The token. @type token: str - @raise MissingAuthTokenError: Raised when given token is missing in - tokens db. - @raise InvalidAuthTokenError: Raised when token is invalid. + @raise InvalidAuthTokenError: Raised when token received from user is + either missing in the tokens db or is + invalid. """ server = Server(url=self._app.state.couch_url) dbname = self.TOKENS_DB db = server[dbname] token = db.get(token) - if token is None: - raise MissingAuthTokenError() - if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF or \ + if token is None or \ + token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF or \ token[self.TOKENS_USER_ID_KEY] != uuid: raise InvalidAuthTokenError() return True -- cgit v1.2.3