diff options
Diffstat (limited to 'src/leap/soledad/swiftclient/client.py')
| -rw-r--r-- | src/leap/soledad/swiftclient/client.py | 1056 |
1 files changed, 0 insertions, 1056 deletions
diff --git a/src/leap/soledad/swiftclient/client.py b/src/leap/soledad/swiftclient/client.py deleted file mode 100644 index 79e6594f..00000000 --- a/src/leap/soledad/swiftclient/client.py +++ /dev/null @@ -1,1056 +0,0 @@ -# Copyright (c) 2010-2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Cloud Files client library used internally -""" - -import socket -import os -import logging -import httplib - -from urllib import quote as _quote -from urlparse import urlparse, urlunparse, urljoin - -try: - from eventlet.green.httplib import HTTPException, HTTPSConnection -except ImportError: - from httplib import HTTPException, HTTPSConnection - -try: - from eventlet import sleep -except ImportError: - from time import sleep - -try: - from swift.common.bufferedhttp \ - import BufferedHTTPConnection as HTTPConnection -except ImportError: - try: - from eventlet.green.httplib import HTTPConnection - except ImportError: - from httplib import HTTPConnection - -logger = logging.getLogger("swiftclient") - - -def http_log(args, kwargs, resp, body): - if os.environ.get('SWIFTCLIENT_DEBUG', False): - ch = logging.StreamHandler() - logger.setLevel(logging.DEBUG) - logger.addHandler(ch) - elif not logger.isEnabledFor(logging.DEBUG): - return - - string_parts = ['curl -i'] - for element in args: - if element in ('GET', 'POST', 'PUT', 'HEAD'): - string_parts.append(' -X %s' % element) - else: - string_parts.append(' %s' % element) - - if 'headers' in kwargs: - for element in kwargs['headers']: - header = ' -H "%s: %s"' % (element, kwargs['headers'][element]) - string_parts.append(header) - - logger.debug("REQ: %s\n" % "".join(string_parts)) - if 'raw_body' in kwargs: - logger.debug("REQ BODY (RAW): %s\n" % (kwargs['raw_body'])) - if 'body' in kwargs: - logger.debug("REQ BODY: %s\n" % (kwargs['body'])) - - logger.debug("RESP STATUS: %s\n", resp.status) - if body: - logger.debug("RESP BODY: %s\n", body) - - -def quote(value, safe='/'): - """ - Patched version of urllib.quote that encodes utf8 strings before quoting - """ - if isinstance(value, unicode): - value = value.encode('utf8') - return _quote(value, safe) - - -# look for a real json parser first -try: - # simplejson is popular and pretty good - from simplejson import loads as json_loads - from simplejson import dumps as json_dumps -except ImportError: - # 2.6 will have a json module in the stdlib - from json import loads as json_loads - from json import dumps as json_dumps - - -class ClientException(Exception): - - def __init__(self, msg, http_scheme='', http_host='', http_port='', - http_path='', http_query='', http_status=0, http_reason='', - http_device='', http_response_content=''): - Exception.__init__(self, msg) - self.msg = msg - self.http_scheme = http_scheme - self.http_host = http_host - self.http_port = http_port - self.http_path = http_path - self.http_query = http_query - self.http_status = http_status - self.http_reason = http_reason - self.http_device = http_device - self.http_response_content = http_response_content - - def __str__(self): - a = self.msg - b = '' - if self.http_scheme: - b += '%s://' % self.http_scheme - if self.http_host: - b += self.http_host - if self.http_port: - b += ':%s' % self.http_port - if self.http_path: - b += self.http_path - if self.http_query: - b += '?%s' % self.http_query - if self.http_status: - if b: - b = '%s %s' % (b, self.http_status) - else: - b = str(self.http_status) - if self.http_reason: - if b: - b = '%s %s' % (b, self.http_reason) - else: - b = '- %s' % self.http_reason - if self.http_device: - if b: - b = '%s: device %s' % (b, self.http_device) - else: - b = 'device %s' % self.http_device - if self.http_response_content: - if len(self.http_response_content) <= 60: - b += ' %s' % self.http_response_content - else: - b += ' [first 60 chars of response] %s' \ - % self.http_response_content[:60] - return b and '%s: %s' % (a, b) or a - - -def http_connection(url, proxy=None): - """ - Make an HTTPConnection or HTTPSConnection - - :param url: url to connect to - :param proxy: proxy to connect through, if any; None by default; str of the - format 'http://127.0.0.1:8888' to set one - :returns: tuple of (parsed url, connection object) - :raises ClientException: Unable to handle protocol scheme - """ - parsed = urlparse(url) - proxy_parsed = urlparse(proxy) if proxy else None - if parsed.scheme == 'http': - conn = HTTPConnection((proxy_parsed if proxy else parsed).netloc) - elif parsed.scheme == 'https': - conn = HTTPSConnection((proxy_parsed if proxy else parsed).netloc) - else: - raise ClientException('Cannot handle protocol scheme %s for url %s' % - (parsed.scheme, repr(url))) - if proxy: - conn._set_tunnel(parsed.hostname, parsed.port) - return parsed, conn - - -def json_request(method, url, **kwargs): - """Takes a request in json parse it and return in json""" - kwargs.setdefault('headers', {}) - if 'body' in kwargs: - kwargs['headers']['Content-Type'] = 'application/json' - kwargs['body'] = json_dumps(kwargs['body']) - parsed, conn = http_connection(url) - conn.request(method, parsed.path, **kwargs) - resp = conn.getresponse() - body = resp.read() - http_log((url, method,), kwargs, resp, body) - if body: - try: - body = json_loads(body) - except ValueError: - body = None - if not body or resp.status < 200 or resp.status >= 300: - raise ClientException('Auth GET failed', http_scheme=parsed.scheme, - http_host=conn.host, - http_port=conn.port, - http_path=parsed.path, - http_status=resp.status, - http_reason=resp.reason) - return resp, body - - -def _get_auth_v1_0(url, user, key, snet): - parsed, conn = http_connection(url) - method = 'GET' - conn.request(method, parsed.path, '', - {'X-Auth-User': user, 'X-Auth-Key': key}) - resp = conn.getresponse() - body = resp.read() - url = resp.getheader('x-storage-url') - http_log((url, method,), {}, resp, body) - - # There is a side-effect on current Rackspace 1.0 server where a - # bad URL would get you that document page and a 200. We error out - # if we don't have a x-storage-url header and if we get a body. - if resp.status < 200 or resp.status >= 300 or (body and not url): - raise ClientException('Auth GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, - http_path=parsed.path, http_status=resp.status, - http_reason=resp.reason) - if snet: - parsed = list(urlparse(url)) - # Second item in the list is the netloc - netloc = parsed[1] - parsed[1] = 'snet-' + netloc - url = urlunparse(parsed) - return url, resp.getheader('x-storage-token', - resp.getheader('x-auth-token')) - - -def _get_auth_v2_0(url, user, tenant_name, key, snet): - body = {'auth': - {'passwordCredentials': {'password': key, 'username': user}, - 'tenantName': tenant_name}} - token_url = urljoin(url, "tokens") - resp, body = json_request("POST", token_url, body=body) - token_id = None - try: - url = None - catalogs = body['access']['serviceCatalog'] - for service in catalogs: - if service['type'] == 'object-store': - url = service['endpoints'][0]['publicURL'] - token_id = body['access']['token']['id'] - if not url: - raise ClientException("There is no object-store endpoint " - "on this auth server.") - except(KeyError, IndexError): - raise ClientException("Error while getting answers from auth server") - - if snet: - parsed = list(urlparse(url)) - # Second item in the list is the netloc - parsed[1] = 'snet-' + parsed[1] - url = urlunparse(parsed) - - return url, token_id - - -def get_auth(url, user, key, snet=False, tenant_name=None, auth_version="1.0"): - """ - Get authentication/authorization credentials. - - The snet parameter is used for Rackspace's ServiceNet internal network - implementation. In this function, it simply adds *snet-* to the beginning - of the host name for the returned storage URL. With Rackspace Cloud Files, - use of this network path causes no bandwidth charges but requires the - client to be running on Rackspace's ServiceNet network. - - :param url: authentication/authorization URL - :param user: user to authenticate as - :param key: key or password for authorization - :param snet: use SERVICENET internal network (see above), default is False - :param auth_version: OpenStack auth version, default is 1.0 - :param tenant_name: The tenant/account name, required when connecting - to a auth 2.0 system. - :returns: tuple of (storage URL, auth token) - :raises: ClientException: HTTP GET request to auth URL failed - """ - if auth_version in ["1.0", "1"]: - return _get_auth_v1_0(url, user, key, snet) - elif auth_version in ["2.0", "2"]: - if not tenant_name and ':' in user: - (tenant_name, user) = user.split(':') - if not tenant_name: - raise ClientException('No tenant specified') - return _get_auth_v2_0(url, user, tenant_name, key, snet) - else: - raise ClientException('Unknown auth_version %s specified.' - % auth_version) - - -def get_account(url, token, marker=None, limit=None, prefix=None, - http_conn=None, full_listing=False): - """ - Get a listing of containers for the account. - - :param url: storage URL - :param token: auth token - :param marker: marker query - :param limit: limit query - :param prefix: prefix query - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :param full_listing: if True, return a full listing, else returns a max - of 10000 listings - :returns: a tuple of (response headers, a list of containers) The response - headers will be a dict and all header names will be lowercase. - :raises ClientException: HTTP GET request failed - """ - if not http_conn: - http_conn = http_connection(url) - if full_listing: - rv = get_account(url, token, marker, limit, prefix, http_conn) - listing = rv[1] - while listing: - marker = listing[-1]['name'] - listing = \ - get_account(url, token, marker, limit, prefix, http_conn)[1] - if listing: - rv[1].extend(listing) - return rv - parsed, conn = http_conn - qs = 'format=json' - if marker: - qs += '&marker=%s' % quote(marker) - if limit: - qs += '&limit=%d' % limit - if prefix: - qs += '&prefix=%s' % quote(prefix) - full_path = '%s?%s' % (parsed.path, qs) - headers = {'X-Auth-Token': token} - conn.request('GET', full_path, '', - headers) - resp = conn.getresponse() - body = resp.read() - http_log(("%s?%s" % (url, qs), 'GET',), {'headers': headers}, resp, body) - - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - if resp.status < 200 or resp.status >= 300: - raise ClientException('Account GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, - http_path=parsed.path, http_query=qs, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) - if resp.status == 204: - body - return resp_headers, [] - return resp_headers, json_loads(body) - - -def head_account(url, token, http_conn=None): - """ - Get account stats. - - :param url: storage URL - :param token: auth token - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :returns: a dict containing the response's headers (all header names will - be lowercase) - :raises ClientException: HTTP HEAD request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - method = "HEAD" - headers = {'X-Auth-Token': token} - conn.request(method, parsed.path, '', headers) - resp = conn.getresponse() - body = resp.read() - http_log((url, method,), {'headers': headers}, resp, body) - if resp.status < 200 or resp.status >= 300: - raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, - http_path=parsed.path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - return resp_headers - - -def post_account(url, token, headers, http_conn=None): - """ - Update an account's metadata. - - :param url: storage URL - :param token: auth token - :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :raises ClientException: HTTP POST request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - method = 'POST' - headers['X-Auth-Token'] = token - conn.request(method, parsed.path, '', headers) - resp = conn.getresponse() - body = resp.read() - http_log((url, method,), {'headers': headers}, resp, body) - if resp.status < 200 or resp.status >= 300: - raise ClientException('Account POST failed', - http_scheme=parsed.scheme, - http_host=conn.host, - http_port=conn.port, - http_path=parsed.path, - http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) - - -def get_container(url, token, container, marker=None, limit=None, - prefix=None, delimiter=None, http_conn=None, - full_listing=False): - """ - Get a listing of objects for the container. - - :param url: storage URL - :param token: auth token - :param container: container name to get a listing for - :param marker: marker query - :param limit: limit query - :param prefix: prefix query - :param delimeter: string to delimit the queries on - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :param full_listing: if True, return a full listing, else returns a max - of 10000 listings - :returns: a tuple of (response headers, a list of objects) The response - headers will be a dict and all header names will be lowercase. - :raises ClientException: HTTP GET request failed - """ - if not http_conn: - http_conn = http_connection(url) - if full_listing: - rv = get_container(url, token, container, marker, limit, prefix, - delimiter, http_conn) - listing = rv[1] - while listing: - if not delimiter: - marker = listing[-1]['name'] - else: - marker = listing[-1].get('name', listing[-1].get('subdir')) - listing = get_container(url, token, container, marker, limit, - prefix, delimiter, http_conn)[1] - if listing: - rv[1].extend(listing) - return rv - parsed, conn = http_conn - path = '%s/%s' % (parsed.path, quote(container)) - qs = 'format=json' - if marker: - qs += '&marker=%s' % quote(marker) - if limit: - qs += '&limit=%d' % limit - if prefix: - qs += '&prefix=%s' % quote(prefix) - if delimiter: - qs += '&delimiter=%s' % quote(delimiter) - headers = {'X-Auth-Token': token} - method = 'GET' - conn.request(method, '%s?%s' % (path, qs), '', headers) - resp = conn.getresponse() - body = resp.read() - http_log(('%s?%s' % (url, qs), method,), {'headers': headers}, resp, body) - - if resp.status < 200 or resp.status >= 300: - raise ClientException('Container GET failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, - http_query=qs, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - if resp.status == 204: - return resp_headers, [] - return resp_headers, json_loads(body) - - -def head_container(url, token, container, http_conn=None, headers=None): - """ - Get container stats. - - :param url: storage URL - :param token: auth token - :param container: container name to get stats for - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :returns: a dict containing the response's headers (all header names will - be lowercase) - :raises ClientException: HTTP HEAD request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s' % (parsed.path, quote(container)) - method = 'HEAD' - req_headers = {'X-Auth-Token': token} - if headers: - req_headers.update(headers) - conn.request(method, path, '', req_headers) - resp = conn.getresponse() - body = resp.read() - http_log(('%s?%s' % (url, path), method,), - {'headers': req_headers}, resp, body) - - if resp.status < 200 or resp.status >= 300: - raise ClientException('Container HEAD failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - return resp_headers - - -def put_container(url, token, container, headers=None, http_conn=None): - """ - Create a container - - :param url: storage URL - :param token: auth token - :param container: container name to create - :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :raises ClientException: HTTP PUT request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s' % (parsed.path, quote(container)) - method = 'PUT' - if not headers: - headers = {} - headers['X-Auth-Token'] = token - conn.request(method, path, '', headers) - resp = conn.getresponse() - body = resp.read() - http_log(('%s?%s' % (url, path), method,), - {'headers': headers}, resp, body) - if resp.status < 200 or resp.status >= 300: - raise ClientException('Container PUT failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) - - -def post_container(url, token, container, headers, http_conn=None): - """ - Update a container's metadata. - - :param url: storage URL - :param token: auth token - :param container: container name to update - :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :raises ClientException: HTTP POST request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s' % (parsed.path, quote(container)) - method = 'POST' - headers['X-Auth-Token'] = token - conn.request(method, path, '', headers) - resp = conn.getresponse() - body = resp.read() - http_log(('%s?%s' % (url, path), method,), - {'headers': headers}, resp, body) - if resp.status < 200 or resp.status >= 300: - raise ClientException('Container POST failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) - - -def delete_container(url, token, container, http_conn=None): - """ - Delete a container - - :param url: storage URL - :param token: auth token - :param container: container name to delete - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :raises ClientException: HTTP DELETE request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s' % (parsed.path, quote(container)) - headers = {'X-Auth-Token': token} - method = 'DELETE' - conn.request(method, path, '', headers) - resp = conn.getresponse() - body = resp.read() - http_log(('%s?%s' % (url, path), method,), - {'headers': headers}, resp, body) - if resp.status < 200 or resp.status >= 300: - raise ClientException('Container DELETE failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) - - -def get_object(url, token, container, name, http_conn=None, - resp_chunk_size=None): - """ - Get an object - - :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: object name to get - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If - you specify a resp_chunk_size you must fully read - the object's contents before making another - request. - :returns: a tuple of (response headers, the object's contents) The response - headers will be a dict and all header names will be lowercase. - :raises ClientException: HTTP GET request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - method = 'GET' - headers = {'X-Auth-Token': token} - conn.request(method, path, '', headers) - resp = conn.getresponse() - if resp.status < 200 or resp.status >= 300: - body = resp.read() - http_log(('%s?%s' % (url, path), 'POST',), - {'headers': headers}, resp, body) - raise ClientException('Object GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) - if resp_chunk_size: - - def _object_body(): - buf = resp.read(resp_chunk_size) - while buf: - yield buf - buf = resp.read(resp_chunk_size) - object_body = _object_body() - else: - object_body = resp.read() - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - http_log(('%s?%s' % (url, path), 'POST',), - {'headers': headers}, resp, object_body) - return resp_headers, object_body - - -def head_object(url, token, container, name, http_conn=None): - """ - Get object info - - :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: object name to get info for - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :returns: a dict containing the response's headers (all header names will - be lowercase) - :raises ClientException: HTTP HEAD request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - method = 'HEAD' - headers = {'X-Auth-Token': token} - conn.request(method, path, '', headers) - resp = conn.getresponse() - body = resp.read() - http_log(('%s?%s' % (url, path), 'POST',), - {'headers': headers}, resp, body) - if resp.status < 200 or resp.status >= 300: - raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - return resp_headers - - -def put_object(url, token=None, container=None, name=None, contents=None, - content_length=None, etag=None, chunk_size=65536, - content_type=None, headers=None, http_conn=None, proxy=None): - """ - Put an object - - :param url: storage URL - :param token: auth token; if None, no token will be sent - :param container: container name that the object is in; if None, the - container name is expected to be part of the url - :param name: object name to put; if None, the object name is expected to be - part of the url - :param contents: a string or a file like object to read object data from; - if None, a zero-byte put will be done - :param content_length: value to send as content-length header; also limits - the amount read from contents; if None, it will be - computed via the contents or chunked transfer - encoding will be used - :param etag: etag of contents; if None, no etag will be sent - :param chunk_size: chunk size of data to write; default 65536 - :param content_type: value to send as content-type header; if None, no - content-type will be set (remote end will likely try - to auto-detect it) - :param headers: additional headers to include in the request, if any - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :param proxy: proxy to connect through, if any; None by default; str of the - format 'http://127.0.0.1:8888' to set one - :returns: etag from server response - :raises ClientException: HTTP PUT request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url, proxy=proxy) - path = parsed.path - if container: - path = '%s/%s' % (path.rstrip('/'), quote(container)) - if name: - path = '%s/%s' % (path.rstrip('/'), quote(name)) - if headers: - headers = dict(headers) - else: - headers = {} - if token: - headers['X-Auth-Token'] = token - if etag: - headers['ETag'] = etag.strip('"') - if content_length is not None: - headers['Content-Length'] = str(content_length) - else: - for n, v in headers.iteritems(): - if n.lower() == 'content-length': - content_length = int(v) - if content_type is not None: - headers['Content-Type'] = content_type - if not contents: - headers['Content-Length'] = '0' - if hasattr(contents, 'read'): - conn.putrequest('PUT', path) - for header, value in headers.iteritems(): - conn.putheader(header, value) - if content_length is None: - conn.putheader('Transfer-Encoding', 'chunked') - conn.endheaders() - chunk = contents.read(chunk_size) - while chunk: - conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) - chunk = contents.read(chunk_size) - conn.send('0\r\n\r\n') - else: - conn.endheaders() - left = content_length - while left > 0: - size = chunk_size - if size > left: - size = left - chunk = contents.read(size) - conn.send(chunk) - left -= len(chunk) - else: - conn.request('PUT', path, contents, headers) - resp = conn.getresponse() - body = resp.read() - headers = {'X-Auth-Token': token} - http_log(('%s?%s' % (url, path), 'PUT',), - {'headers': headers}, resp, body) - if resp.status < 200 or resp.status >= 300: - raise ClientException('Object PUT failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) - return resp.getheader('etag', '').strip('"') - - -def post_object(url, token, container, name, headers, http_conn=None): - """ - Update object metadata - - :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: name of the object to update - :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :raises ClientException: HTTP POST request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers['X-Auth-Token'] = token - conn.request('POST', path, '', headers) - resp = conn.getresponse() - body = resp.read() - http_log(('%s?%s' % (url, path), 'POST',), - {'headers': headers}, resp, body) - if resp.status < 200 or resp.status >= 300: - raise ClientException('Object POST failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) - - -def delete_object(url, token=None, container=None, name=None, http_conn=None, - headers=None, proxy=None): - """ - Delete object - - :param url: storage URL - :param token: auth token; if None, no token will be sent - :param container: container name that the object is in; if None, the - container name is expected to be part of the url - :param name: object name to delete; if None, the object name is expected to - be part of the url - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :param headers: additional headers to include in the request - :param proxy: proxy to connect through, if any; None by default; str of the - format 'http://127.0.0.1:8888' to set one - :raises ClientException: HTTP DELETE request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url, proxy=proxy) - path = parsed.path - if container: - path = '%s/%s' % (path.rstrip('/'), quote(container)) - if name: - path = '%s/%s' % (path.rstrip('/'), quote(name)) - if headers: - headers = dict(headers) - else: - headers = {} - if token: - headers['X-Auth-Token'] = token - conn.request('DELETE', path, '', headers) - resp = conn.getresponse() - body = resp.read() - http_log(('%s?%s' % (url, path), 'POST',), - {'headers': headers}, resp, body) - if resp.status < 200 or resp.status >= 300: - raise ClientException('Object DELETE failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) - - -class Connection(object): - """Convenience class to make requests that will also retry the request""" - - def __init__(self, authurl, user, key, retries=5, preauthurl=None, - preauthtoken=None, snet=False, starting_backoff=1, - tenant_name=None, - auth_version="1"): - """ - :param authurl: authentication URL - :param user: user name to authenticate as - :param key: key/password to authenticate with - :param retries: Number of times to retry the request before failing - :param preauthurl: storage URL (if you have already authenticated) - :param preauthtoken: authentication token (if you have already - authenticated) - :param snet: use SERVICENET internal network default is False - :param auth_version: OpenStack auth version, default is 1.0 - :param tenant_name: The tenant/account name, required when connecting - to a auth 2.0 system. - """ - self.authurl = authurl - self.user = user - self.key = key - self.retries = retries - self.http_conn = None - self.url = preauthurl - self.token = preauthtoken - self.attempts = 0 - self.snet = snet - self.starting_backoff = starting_backoff - self.auth_version = auth_version - self.tenant_name = tenant_name - - def get_auth(self): - return get_auth(self.authurl, self.user, - self.key, snet=self.snet, - tenant_name=self.tenant_name, - auth_version=self.auth_version) - - def http_connection(self): - return http_connection(self.url) - - def _retry(self, reset_func, func, *args, **kwargs): - self.attempts = 0 - backoff = self.starting_backoff - while self.attempts <= self.retries: - self.attempts += 1 - try: - if not self.url or not self.token: - self.url, self.token = self.get_auth() - self.http_conn = None - if not self.http_conn: - self.http_conn = self.http_connection() - kwargs['http_conn'] = self.http_conn - rv = func(self.url, self.token, *args, **kwargs) - return rv - except (socket.error, HTTPException): - if self.attempts > self.retries: - raise - self.http_conn = None - except ClientException, err: - if self.attempts > self.retries: - raise - if err.http_status == 401: - self.url = self.token = None - if self.attempts > 1: - raise - elif err.http_status == 408: - self.http_conn = None - elif 500 <= err.http_status <= 599: - pass - else: - raise - sleep(backoff) - backoff *= 2 - if reset_func: - reset_func(func, *args, **kwargs) - - def head_account(self): - """Wrapper for :func:`head_account`""" - return self._retry(None, head_account) - - def get_account(self, marker=None, limit=None, prefix=None, - full_listing=False): - """Wrapper for :func:`get_account`""" - # TODO(unknown): With full_listing=True this will restart the entire - # listing with each retry. Need to make a better version that just - # retries where it left off. - return self._retry(None, get_account, marker=marker, limit=limit, - prefix=prefix, full_listing=full_listing) - - def post_account(self, headers): - """Wrapper for :func:`post_account`""" - return self._retry(None, post_account, headers) - - def head_container(self, container): - """Wrapper for :func:`head_container`""" - return self._retry(None, head_container, container) - - def get_container(self, container, marker=None, limit=None, prefix=None, - delimiter=None, full_listing=False): - """Wrapper for :func:`get_container`""" - # TODO(unknown): With full_listing=True this will restart the entire - # listing with each retry. Need to make a better version that just - # retries where it left off. - return self._retry(None, get_container, container, marker=marker, - limit=limit, prefix=prefix, delimiter=delimiter, - full_listing=full_listing) - - def put_container(self, container, headers=None): - """Wrapper for :func:`put_container`""" - return self._retry(None, put_container, container, headers=headers) - - def post_container(self, container, headers): - """Wrapper for :func:`post_container`""" - return self._retry(None, post_container, container, headers) - - def delete_container(self, container): - """Wrapper for :func:`delete_container`""" - return self._retry(None, delete_container, container) - - def head_object(self, container, obj): - """Wrapper for :func:`head_object`""" - return self._retry(None, head_object, container, obj) - - def get_object(self, container, obj, resp_chunk_size=None): - """Wrapper for :func:`get_object`""" - return self._retry(None, get_object, container, obj, - resp_chunk_size=resp_chunk_size) - - def put_object(self, container, obj, contents, content_length=None, - etag=None, chunk_size=65536, content_type=None, - headers=None): - """Wrapper for :func:`put_object`""" - - def _default_reset(*args, **kwargs): - raise ClientException('put_object(%r, %r, ...) failure and no ' - 'ability to reset contents for reupload.' - % (container, obj)) - - reset_func = _default_reset - tell = getattr(contents, 'tell', None) - seek = getattr(contents, 'seek', None) - if tell and seek: - orig_pos = tell() - reset_func = lambda *a, **k: seek(orig_pos) - elif not contents: - reset_func = lambda *a, **k: None - - return self._retry(reset_func, put_object, container, obj, contents, - content_length=content_length, etag=etag, - chunk_size=chunk_size, content_type=content_type, - headers=headers) - - def post_object(self, container, obj, headers): - """Wrapper for :func:`post_object`""" - return self._retry(None, post_object, container, obj, headers) - - def delete_object(self, container, obj): - """Wrapper for :func:`delete_object`""" - return self._retry(None, delete_object, container, obj) |