From 7b72e60911009e282520c5d8b6c6edd75ab48fd2 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 18 Dec 2013 12:37:44 -0400 Subject: remove conditional gzip, fix import --- server/src/leap/soledad/server/__init__.py | 2 +- server/src/leap/soledad/server/gzip.py | 69 ----------------------- server/src/leap/soledad/server/gzip_middleware.py | 68 ++++++++++++++++++++++ 3 files changed, 69 insertions(+), 70 deletions(-) delete mode 100644 server/src/leap/soledad/server/gzip.py create mode 100644 server/src/leap/soledad/server/gzip_middleware.py diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index de5daf62..a4b25fe2 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -110,7 +110,7 @@ if version.base() == "12.0.0": sys.modules['OpenSSL.tsafe'] = old_tsafe from leap.soledad.server.auth import SoledadTokenAuthMiddleware -from leap.soledad.server.gzip import GzipMiddleware +from leap.soledad.server.gzip_middleware import GzipMiddleware from leap.soledad.common import ( SHARED_DB_NAME, diff --git a/server/src/leap/soledad/server/gzip.py b/server/src/leap/soledad/server/gzip.py deleted file mode 100644 index 92906513..00000000 --- a/server/src/leap/soledad/server/gzip.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -# gzip.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Gzip middleware for WSGI apps. -""" -import gzip -import StringIO - - -class GzipMiddleware(object): - """ - GzipMiddleware class for WSGI. - """ - def __init__(self, app, compresslevel=9): - self.app = app - self.compresslevel = compresslevel - - def __call__(self, environ, start_response): - if 'gzip' not in environ.get('HTTP_ACCEPT_ENCODING', ''): - return self.app(environ, start_response) - if (environ['PATH_INFO'][-3:] != '.js' and environ[ - 'PATH_INFO'][-4:] != '.css'): - return self.app(environ, start_response) - buffer = StringIO.StringIO() - output = gzip.GzipFile( - mode='wb', - compresslevel=self.compresslevel, - fileobj=buffer - ) - - start_response_args = [] - - def dummy_start_response(status, headers, exc_info=None): - start_response_args.append(status) - start_response_args.append(headers) - start_response_args.append(exc_info) - return output.write - - app_iter = self.app(environ, dummy_start_response) - for line in app_iter: - output.write(line) - if hasattr(app_iter, 'close'): - app_iter.close() - output.close() - buffer.seek(0) - result = buffer.getvalue() - headers = [] - for name, value in start_response_args[1]: - if name.lower() != 'content-length': - headers.append((name, value)) - headers.append(('Content-Length', str(len(result)))) - headers.append(('Content-Encoding', 'gzip')) - start_response(start_response_args[0], headers, start_response_args[2]) - buffer.close() - return [result] diff --git a/server/src/leap/soledad/server/gzip_middleware.py b/server/src/leap/soledad/server/gzip_middleware.py new file mode 100644 index 00000000..5a424894 --- /dev/null +++ b/server/src/leap/soledad/server/gzip_middleware.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# gzip_middleware.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Gzip middleware for WSGI apps. +""" +import StringIO +from gzip import GzipFile + + +class GzipMiddleware(object): + """ + GzipMiddleware class for WSGI. + """ + def __init__(self, app, compresslevel=9): + self.app = app + self.compresslevel = compresslevel + + def __call__(self, environ, start_response): + print "environ: ", environ + if 'gzip' not in environ.get('HTTP_ACCEPT_ENCODING', ''): + return self.app(environ, start_response) + + buffer = StringIO.StringIO() + output = GzipFile( + mode='wb', + compresslevel=self.compresslevel, + fileobj=buffer + ) + + start_response_args = [] + + def dummy_start_response(status, headers, exc_info=None): + start_response_args.append(status) + start_response_args.append(headers) + start_response_args.append(exc_info) + return output.write + + app_iter = self.app(environ, dummy_start_response) + for line in app_iter: + output.write(line) + if hasattr(app_iter, 'close'): + app_iter.close() + output.close() + buffer.seek(0) + result = buffer.getvalue() + headers = [] + for name, value in start_response_args[1]: + if name.lower() != 'content-length': + headers.append((name, value)) + headers.append(('Content-Length', str(len(result)))) + headers.append(('Content-Encoding', 'gzip')) + start_response(start_response_args[0], headers, start_response_args[2]) + buffer.close() + return [result] -- cgit v1.2.3 From ecc05503b393844ee8b864db9d32db288b875d11 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 18 Dec 2013 12:48:00 -0400 Subject: add http-encoding header, decode it too --- client/src/leap/soledad/client/target.py | 120 ++++++++++++++++++++-- server/src/leap/soledad/server/gzip_middleware.py | 1 - 2 files changed, 111 insertions(+), 10 deletions(-) diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index d8899a97..73f719fb 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -14,22 +14,26 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ A U1DB backend for encrypting data before sending to server and decrypting after receiving. """ - -import simplejson as json +import binascii +import cStringIO +import gzip import hashlib import hmac -import binascii +import logging +import urllib +import simplejson as json +from time import sleep -from u1db.remote import utils +from u1db.remote import utils, http_errors from u1db.errors import BrokenSyncStream +from u1db import errors from u1db.remote.http_target import HTTPSyncTarget +from u1db.remote.http_client import _encode_query_parameter from leap.soledad.common import soledad_assert @@ -53,6 +57,7 @@ from leap.soledad.client.crypto import ( UnknownEncryptionMethod, ) +logger = logging.getLogger(__name__) # # Exceptions @@ -222,6 +227,24 @@ def decrypt_doc(crypto, doc): return plainjson +def _gunzip(data): + """ + Uncompress data that is gzipped. + + :param data: gzipped data + :type data: basestring + """ + buffer = cStringIO.StringIO() + buffer.write(data) + buffer.seek(0) + try: + data = gzip.GzipFile(mode='r', fileobj=buffer).read() + except Exception: + logger.warning("Error while decrypting gzipped data") + buffer.close() + return data + + # # SoledadSyncTarget # @@ -353,6 +376,82 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): raise BrokenSyncStream return res + def _request(self, method, url_parts, params=None, body=None, + content_type=None): + """ + Overloaded method. See u1db docs. + Patched for adding gzip encoding. + """ + + self._ensure_connection() + unquoted_url = url_query = self._url.path + if url_parts: + if not url_query.endswith('/'): + url_query += '/' + unquoted_url = url_query + url_query += '/'.join(urllib.quote(part, safe='') + for part in url_parts) + # oauth performs its own quoting + unquoted_url += '/'.join(url_parts) + encoded_params = {} + if params: + for key, value in params.items(): + key = unicode(key).encode('utf-8') + encoded_params[key] = _encode_query_parameter(value) + url_query += ('?' + urllib.urlencode(encoded_params)) + if body is not None and not isinstance(body, basestring): + body = json.dumps(body) + content_type = 'application/json' + headers = {} + if content_type: + headers['content-type'] = content_type + + # Patched: We would like to receive gzip pretty please + # ---------------------------------------------------- + headers['accept-encoding'] = "gzip" + # ---------------------------------------------------- + + headers.update( + self._sign_request(method, unquoted_url, encoded_params)) + + for delay in self._delays: + try: + self._conn.request(method, url_query, body, headers) + return self._response() + except errors.Unavailable, e: + sleep(delay) + raise e + + def _response(self): + """ + Overloaded method, see u1db docs. + We patched it for decrypting gzip content. + """ + resp = self._conn.getresponse() + body = resp.read() + headers = dict(resp.getheaders()) + + # Patched: We would like to decode gzip + # ---------------------------------------------------- + encoding = headers.get('content-encoding', '') + if "gzip" in encoding: + body = _gunzip(body) + # ---------------------------------------------------- + + if resp.status in (200, 201): + return body, headers + elif resp.status in http_errors.ERROR_STATUSES: + try: + respdic = json.loads(body) + except ValueError: + pass + else: + self._error(respdic) + # special case + if resp.status == 503: + raise errors.Unavailable(body, headers) + raise errors.HTTPError(resp.status, body, headers) + def sync_exchange(self, docs_by_generations, source_replica_uid, last_known_generation, last_known_trans_id, return_doc_cb, ensure_callback=None): @@ -364,8 +463,9 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): syncing. :param docs_by_generations: A list of (doc_id, generation, trans_id) - of local documents that were changed since the last local - generation the remote replica knows about. + of local documents that were changed since + the last local generation the remote + replica knows about. :type docs_by_generations: list of tuples :param source_replica_uid: The uid of the source replica. :type source_replica_uid: str @@ -391,6 +491,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): self._conn.putheader('content-type', 'application/x-u1db-sync-stream') for header_name, header_value in self._sign_request('POST', url, {}): self._conn.putheader(header_name, header_value) + self._conn.putheader('accept-encoding', 'gzip') entries = ['['] size = 1 @@ -428,7 +529,8 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): for entry in entries: self._conn.send(entry) entries = None - data, _ = self._response() + data, headers = self._response() + res = self._parse_sync_stream(data, return_doc_cb, ensure_callback) data = None return res['new_generation'], res['new_transaction_id'] diff --git a/server/src/leap/soledad/server/gzip_middleware.py b/server/src/leap/soledad/server/gzip_middleware.py index 5a424894..986c5738 100644 --- a/server/src/leap/soledad/server/gzip_middleware.py +++ b/server/src/leap/soledad/server/gzip_middleware.py @@ -30,7 +30,6 @@ class GzipMiddleware(object): self.compresslevel = compresslevel def __call__(self, environ, start_response): - print "environ: ", environ if 'gzip' not in environ.get('HTTP_ACCEPT_ENCODING', ''): return self.app(environ, start_response) -- cgit v1.2.3