summaryrefslogtreecommitdiff
path: root/swiftclient/client.py
diff options
context:
space:
mode:
Diffstat (limited to 'swiftclient/client.py')
-rw-r--r--swiftclient/client.py1056
1 files changed, 1056 insertions, 0 deletions
diff --git a/swiftclient/client.py b/swiftclient/client.py
new file mode 100644
index 00000000..79e6594f
--- /dev/null
+++ b/swiftclient/client.py
@@ -0,0 +1,1056 @@
+# Copyright (c) 2010-2012 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Cloud Files client library used internally
+"""
+
+import socket
+import os
+import logging
+import httplib
+
+from urllib import quote as _quote
+from urlparse import urlparse, urlunparse, urljoin
+
+try:
+ from eventlet.green.httplib import HTTPException, HTTPSConnection
+except ImportError:
+ from httplib import HTTPException, HTTPSConnection
+
+try:
+ from eventlet import sleep
+except ImportError:
+ from time import sleep
+
+try:
+ from swift.common.bufferedhttp \
+ import BufferedHTTPConnection as HTTPConnection
+except ImportError:
+ try:
+ from eventlet.green.httplib import HTTPConnection
+ except ImportError:
+ from httplib import HTTPConnection
+
+logger = logging.getLogger("swiftclient")
+
+
+def http_log(args, kwargs, resp, body):
+ if os.environ.get('SWIFTCLIENT_DEBUG', False):
+ ch = logging.StreamHandler()
+ logger.setLevel(logging.DEBUG)
+ logger.addHandler(ch)
+ elif not logger.isEnabledFor(logging.DEBUG):
+ return
+
+ string_parts = ['curl -i']
+ for element in args:
+ if element in ('GET', 'POST', 'PUT', 'HEAD'):
+ string_parts.append(' -X %s' % element)
+ else:
+ string_parts.append(' %s' % element)
+
+ if 'headers' in kwargs:
+ for element in kwargs['headers']:
+ header = ' -H "%s: %s"' % (element, kwargs['headers'][element])
+ string_parts.append(header)
+
+ logger.debug("REQ: %s\n" % "".join(string_parts))
+ if 'raw_body' in kwargs:
+ logger.debug("REQ BODY (RAW): %s\n" % (kwargs['raw_body']))
+ if 'body' in kwargs:
+ logger.debug("REQ BODY: %s\n" % (kwargs['body']))
+
+ logger.debug("RESP STATUS: %s\n", resp.status)
+ if body:
+ logger.debug("RESP BODY: %s\n", body)
+
+
+def quote(value, safe='/'):
+ """
+ Patched version of urllib.quote that encodes utf8 strings before quoting
+ """
+ if isinstance(value, unicode):
+ value = value.encode('utf8')
+ return _quote(value, safe)
+
+
+# look for a real json parser first
+try:
+ # simplejson is popular and pretty good
+ from simplejson import loads as json_loads
+ from simplejson import dumps as json_dumps
+except ImportError:
+ # 2.6 will have a json module in the stdlib
+ from json import loads as json_loads
+ from json import dumps as json_dumps
+
+
+class ClientException(Exception):
+
+ def __init__(self, msg, http_scheme='', http_host='', http_port='',
+ http_path='', http_query='', http_status=0, http_reason='',
+ http_device='', http_response_content=''):
+ Exception.__init__(self, msg)
+ self.msg = msg
+ self.http_scheme = http_scheme
+ self.http_host = http_host
+ self.http_port = http_port
+ self.http_path = http_path
+ self.http_query = http_query
+ self.http_status = http_status
+ self.http_reason = http_reason
+ self.http_device = http_device
+ self.http_response_content = http_response_content
+
+ def __str__(self):
+ a = self.msg
+ b = ''
+ if self.http_scheme:
+ b += '%s://' % self.http_scheme
+ if self.http_host:
+ b += self.http_host
+ if self.http_port:
+ b += ':%s' % self.http_port
+ if self.http_path:
+ b += self.http_path
+ if self.http_query:
+ b += '?%s' % self.http_query
+ if self.http_status:
+ if b:
+ b = '%s %s' % (b, self.http_status)
+ else:
+ b = str(self.http_status)
+ if self.http_reason:
+ if b:
+ b = '%s %s' % (b, self.http_reason)
+ else:
+ b = '- %s' % self.http_reason
+ if self.http_device:
+ if b:
+ b = '%s: device %s' % (b, self.http_device)
+ else:
+ b = 'device %s' % self.http_device
+ if self.http_response_content:
+ if len(self.http_response_content) <= 60:
+ b += ' %s' % self.http_response_content
+ else:
+ b += ' [first 60 chars of response] %s' \
+ % self.http_response_content[:60]
+ return b and '%s: %s' % (a, b) or a
+
+
+def http_connection(url, proxy=None):
+ """
+ Make an HTTPConnection or HTTPSConnection
+
+ :param url: url to connect to
+ :param proxy: proxy to connect through, if any; None by default; str of the
+ format 'http://127.0.0.1:8888' to set one
+ :returns: tuple of (parsed url, connection object)
+ :raises ClientException: Unable to handle protocol scheme
+ """
+ parsed = urlparse(url)
+ proxy_parsed = urlparse(proxy) if proxy else None
+ if parsed.scheme == 'http':
+ conn = HTTPConnection((proxy_parsed if proxy else parsed).netloc)
+ elif parsed.scheme == 'https':
+ conn = HTTPSConnection((proxy_parsed if proxy else parsed).netloc)
+ else:
+ raise ClientException('Cannot handle protocol scheme %s for url %s' %
+ (parsed.scheme, repr(url)))
+ if proxy:
+ conn._set_tunnel(parsed.hostname, parsed.port)
+ return parsed, conn
+
+
+def json_request(method, url, **kwargs):
+ """Takes a request in json parse it and return in json"""
+ kwargs.setdefault('headers', {})
+ if 'body' in kwargs:
+ kwargs['headers']['Content-Type'] = 'application/json'
+ kwargs['body'] = json_dumps(kwargs['body'])
+ parsed, conn = http_connection(url)
+ conn.request(method, parsed.path, **kwargs)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log((url, method,), kwargs, resp, body)
+ if body:
+ try:
+ body = json_loads(body)
+ except ValueError:
+ body = None
+ if not body or resp.status < 200 or resp.status >= 300:
+ raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
+ http_host=conn.host,
+ http_port=conn.port,
+ http_path=parsed.path,
+ http_status=resp.status,
+ http_reason=resp.reason)
+ return resp, body
+
+
+def _get_auth_v1_0(url, user, key, snet):
+ parsed, conn = http_connection(url)
+ method = 'GET'
+ conn.request(method, parsed.path, '',
+ {'X-Auth-User': user, 'X-Auth-Key': key})
+ resp = conn.getresponse()
+ body = resp.read()
+ url = resp.getheader('x-storage-url')
+ http_log((url, method,), {}, resp, body)
+
+ # There is a side-effect on current Rackspace 1.0 server where a
+ # bad URL would get you that document page and a 200. We error out
+ # if we don't have a x-storage-url header and if we get a body.
+ if resp.status < 200 or resp.status >= 300 or (body and not url):
+ raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
+ http_host=conn.host, http_port=conn.port,
+ http_path=parsed.path, http_status=resp.status,
+ http_reason=resp.reason)
+ if snet:
+ parsed = list(urlparse(url))
+ # Second item in the list is the netloc
+ netloc = parsed[1]
+ parsed[1] = 'snet-' + netloc
+ url = urlunparse(parsed)
+ return url, resp.getheader('x-storage-token',
+ resp.getheader('x-auth-token'))
+
+
+def _get_auth_v2_0(url, user, tenant_name, key, snet):
+ body = {'auth':
+ {'passwordCredentials': {'password': key, 'username': user},
+ 'tenantName': tenant_name}}
+ token_url = urljoin(url, "tokens")
+ resp, body = json_request("POST", token_url, body=body)
+ token_id = None
+ try:
+ url = None
+ catalogs = body['access']['serviceCatalog']
+ for service in catalogs:
+ if service['type'] == 'object-store':
+ url = service['endpoints'][0]['publicURL']
+ token_id = body['access']['token']['id']
+ if not url:
+ raise ClientException("There is no object-store endpoint "
+ "on this auth server.")
+ except(KeyError, IndexError):
+ raise ClientException("Error while getting answers from auth server")
+
+ if snet:
+ parsed = list(urlparse(url))
+ # Second item in the list is the netloc
+ parsed[1] = 'snet-' + parsed[1]
+ url = urlunparse(parsed)
+
+ return url, token_id
+
+
+def get_auth(url, user, key, snet=False, tenant_name=None, auth_version="1.0"):
+ """
+ Get authentication/authorization credentials.
+
+ The snet parameter is used for Rackspace's ServiceNet internal network
+ implementation. In this function, it simply adds *snet-* to the beginning
+ of the host name for the returned storage URL. With Rackspace Cloud Files,
+ use of this network path causes no bandwidth charges but requires the
+ client to be running on Rackspace's ServiceNet network.
+
+ :param url: authentication/authorization URL
+ :param user: user to authenticate as
+ :param key: key or password for authorization
+ :param snet: use SERVICENET internal network (see above), default is False
+ :param auth_version: OpenStack auth version, default is 1.0
+ :param tenant_name: The tenant/account name, required when connecting
+ to a auth 2.0 system.
+ :returns: tuple of (storage URL, auth token)
+ :raises: ClientException: HTTP GET request to auth URL failed
+ """
+ if auth_version in ["1.0", "1"]:
+ return _get_auth_v1_0(url, user, key, snet)
+ elif auth_version in ["2.0", "2"]:
+ if not tenant_name and ':' in user:
+ (tenant_name, user) = user.split(':')
+ if not tenant_name:
+ raise ClientException('No tenant specified')
+ return _get_auth_v2_0(url, user, tenant_name, key, snet)
+ else:
+ raise ClientException('Unknown auth_version %s specified.'
+ % auth_version)
+
+
+def get_account(url, token, marker=None, limit=None, prefix=None,
+ http_conn=None, full_listing=False):
+ """
+ Get a listing of containers for the account.
+
+ :param url: storage URL
+ :param token: auth token
+ :param marker: marker query
+ :param limit: limit query
+ :param prefix: prefix query
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :param full_listing: if True, return a full listing, else returns a max
+ of 10000 listings
+ :returns: a tuple of (response headers, a list of containers) The response
+ headers will be a dict and all header names will be lowercase.
+ :raises ClientException: HTTP GET request failed
+ """
+ if not http_conn:
+ http_conn = http_connection(url)
+ if full_listing:
+ rv = get_account(url, token, marker, limit, prefix, http_conn)
+ listing = rv[1]
+ while listing:
+ marker = listing[-1]['name']
+ listing = \
+ get_account(url, token, marker, limit, prefix, http_conn)[1]
+ if listing:
+ rv[1].extend(listing)
+ return rv
+ parsed, conn = http_conn
+ qs = 'format=json'
+ if marker:
+ qs += '&marker=%s' % quote(marker)
+ if limit:
+ qs += '&limit=%d' % limit
+ if prefix:
+ qs += '&prefix=%s' % quote(prefix)
+ full_path = '%s?%s' % (parsed.path, qs)
+ headers = {'X-Auth-Token': token}
+ conn.request('GET', full_path, '',
+ headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log(("%s?%s" % (url, qs), 'GET',), {'headers': headers}, resp, body)
+
+ resp_headers = {}
+ for header, value in resp.getheaders():
+ resp_headers[header.lower()] = value
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Account GET failed', http_scheme=parsed.scheme,
+ http_host=conn.host, http_port=conn.port,
+ http_path=parsed.path, http_query=qs,
+ http_status=resp.status, http_reason=resp.reason,
+ http_response_content=body)
+ if resp.status == 204:
+ body
+ return resp_headers, []
+ return resp_headers, json_loads(body)
+
+
+def head_account(url, token, http_conn=None):
+ """
+ Get account stats.
+
+ :param url: storage URL
+ :param token: auth token
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :returns: a dict containing the response's headers (all header names will
+ be lowercase)
+ :raises ClientException: HTTP HEAD request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url)
+ method = "HEAD"
+ headers = {'X-Auth-Token': token}
+ conn.request(method, parsed.path, '', headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log((url, method,), {'headers': headers}, resp, body)
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Account HEAD failed', http_scheme=parsed.scheme,
+ http_host=conn.host, http_port=conn.port,
+ http_path=parsed.path, http_status=resp.status,
+ http_reason=resp.reason,
+ http_response_content=body)
+ resp_headers = {}
+ for header, value in resp.getheaders():
+ resp_headers[header.lower()] = value
+ return resp_headers
+
+
+def post_account(url, token, headers, http_conn=None):
+ """
+ Update an account's metadata.
+
+ :param url: storage URL
+ :param token: auth token
+ :param headers: additional headers to include in the request
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :raises ClientException: HTTP POST request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url)
+ method = 'POST'
+ headers['X-Auth-Token'] = token
+ conn.request(method, parsed.path, '', headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log((url, method,), {'headers': headers}, resp, body)
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Account POST failed',
+ http_scheme=parsed.scheme,
+ http_host=conn.host,
+ http_port=conn.port,
+ http_path=parsed.path,
+ http_status=resp.status,
+ http_reason=resp.reason,
+ http_response_content=body)
+
+
+def get_container(url, token, container, marker=None, limit=None,
+ prefix=None, delimiter=None, http_conn=None,
+ full_listing=False):
+ """
+ Get a listing of objects for the container.
+
+ :param url: storage URL
+ :param token: auth token
+ :param container: container name to get a listing for
+ :param marker: marker query
+ :param limit: limit query
+ :param prefix: prefix query
+ :param delimeter: string to delimit the queries on
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :param full_listing: if True, return a full listing, else returns a max
+ of 10000 listings
+ :returns: a tuple of (response headers, a list of objects) The response
+ headers will be a dict and all header names will be lowercase.
+ :raises ClientException: HTTP GET request failed
+ """
+ if not http_conn:
+ http_conn = http_connection(url)
+ if full_listing:
+ rv = get_container(url, token, container, marker, limit, prefix,
+ delimiter, http_conn)
+ listing = rv[1]
+ while listing:
+ if not delimiter:
+ marker = listing[-1]['name']
+ else:
+ marker = listing[-1].get('name', listing[-1].get('subdir'))
+ listing = get_container(url, token, container, marker, limit,
+ prefix, delimiter, http_conn)[1]
+ if listing:
+ rv[1].extend(listing)
+ return rv
+ parsed, conn = http_conn
+ path = '%s/%s' % (parsed.path, quote(container))
+ qs = 'format=json'
+ if marker:
+ qs += '&marker=%s' % quote(marker)
+ if limit:
+ qs += '&limit=%d' % limit
+ if prefix:
+ qs += '&prefix=%s' % quote(prefix)
+ if delimiter:
+ qs += '&delimiter=%s' % quote(delimiter)
+ headers = {'X-Auth-Token': token}
+ method = 'GET'
+ conn.request(method, '%s?%s' % (path, qs), '', headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log(('%s?%s' % (url, qs), method,), {'headers': headers}, resp, body)
+
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Container GET failed',
+ http_scheme=parsed.scheme, http_host=conn.host,
+ http_port=conn.port, http_path=path,
+ http_query=qs, http_status=resp.status,
+ http_reason=resp.reason,
+ http_response_content=body)
+ resp_headers = {}
+ for header, value in resp.getheaders():
+ resp_headers[header.lower()] = value
+ if resp.status == 204:
+ return resp_headers, []
+ return resp_headers, json_loads(body)
+
+
+def head_container(url, token, container, http_conn=None, headers=None):
+ """
+ Get container stats.
+
+ :param url: storage URL
+ :param token: auth token
+ :param container: container name to get stats for
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :returns: a dict containing the response's headers (all header names will
+ be lowercase)
+ :raises ClientException: HTTP HEAD request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url)
+ path = '%s/%s' % (parsed.path, quote(container))
+ method = 'HEAD'
+ req_headers = {'X-Auth-Token': token}
+ if headers:
+ req_headers.update(headers)
+ conn.request(method, path, '', req_headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log(('%s?%s' % (url, path), method,),
+ {'headers': req_headers}, resp, body)
+
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Container HEAD failed',
+ http_scheme=parsed.scheme, http_host=conn.host,
+ http_port=conn.port, http_path=path,
+ http_status=resp.status, http_reason=resp.reason,
+ http_response_content=body)
+ resp_headers = {}
+ for header, value in resp.getheaders():
+ resp_headers[header.lower()] = value
+ return resp_headers
+
+
+def put_container(url, token, container, headers=None, http_conn=None):
+ """
+ Create a container
+
+ :param url: storage URL
+ :param token: auth token
+ :param container: container name to create
+ :param headers: additional headers to include in the request
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :raises ClientException: HTTP PUT request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url)
+ path = '%s/%s' % (parsed.path, quote(container))
+ method = 'PUT'
+ if not headers:
+ headers = {}
+ headers['X-Auth-Token'] = token
+ conn.request(method, path, '', headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log(('%s?%s' % (url, path), method,),
+ {'headers': headers}, resp, body)
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Container PUT failed',
+ http_scheme=parsed.scheme, http_host=conn.host,
+ http_port=conn.port, http_path=path,
+ http_status=resp.status, http_reason=resp.reason,
+ http_response_content=body)
+
+
+def post_container(url, token, container, headers, http_conn=None):
+ """
+ Update a container's metadata.
+
+ :param url: storage URL
+ :param token: auth token
+ :param container: container name to update
+ :param headers: additional headers to include in the request
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :raises ClientException: HTTP POST request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url)
+ path = '%s/%s' % (parsed.path, quote(container))
+ method = 'POST'
+ headers['X-Auth-Token'] = token
+ conn.request(method, path, '', headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log(('%s?%s' % (url, path), method,),
+ {'headers': headers}, resp, body)
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Container POST failed',
+ http_scheme=parsed.scheme, http_host=conn.host,
+ http_port=conn.port, http_path=path,
+ http_status=resp.status, http_reason=resp.reason,
+ http_response_content=body)
+
+
+def delete_container(url, token, container, http_conn=None):
+ """
+ Delete a container
+
+ :param url: storage URL
+ :param token: auth token
+ :param container: container name to delete
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :raises ClientException: HTTP DELETE request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url)
+ path = '%s/%s' % (parsed.path, quote(container))
+ headers = {'X-Auth-Token': token}
+ method = 'DELETE'
+ conn.request(method, path, '', headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log(('%s?%s' % (url, path), method,),
+ {'headers': headers}, resp, body)
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Container DELETE failed',
+ http_scheme=parsed.scheme, http_host=conn.host,
+ http_port=conn.port, http_path=path,
+ http_status=resp.status, http_reason=resp.reason,
+ http_response_content=body)
+
+
+def get_object(url, token, container, name, http_conn=None,
+ resp_chunk_size=None):
+ """
+ Get an object
+
+ :param url: storage URL
+ :param token: auth token
+ :param container: container name that the object is in
+ :param name: object name to get
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If
+ you specify a resp_chunk_size you must fully read
+ the object's contents before making another
+ request.
+ :returns: a tuple of (response headers, the object's contents) The response
+ headers will be a dict and all header names will be lowercase.
+ :raises ClientException: HTTP GET request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url)
+ path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
+ method = 'GET'
+ headers = {'X-Auth-Token': token}
+ conn.request(method, path, '', headers)
+ resp = conn.getresponse()
+ if resp.status < 200 or resp.status >= 300:
+ body = resp.read()
+ http_log(('%s?%s' % (url, path), 'POST',),
+ {'headers': headers}, resp, body)
+ raise ClientException('Object GET failed', http_scheme=parsed.scheme,
+ http_host=conn.host, http_port=conn.port,
+ http_path=path, http_status=resp.status,
+ http_reason=resp.reason,
+ http_response_content=body)
+ if resp_chunk_size:
+
+ def _object_body():
+ buf = resp.read(resp_chunk_size)
+ while buf:
+ yield buf
+ buf = resp.read(resp_chunk_size)
+ object_body = _object_body()
+ else:
+ object_body = resp.read()
+ resp_headers = {}
+ for header, value in resp.getheaders():
+ resp_headers[header.lower()] = value
+ http_log(('%s?%s' % (url, path), 'POST',),
+ {'headers': headers}, resp, object_body)
+ return resp_headers, object_body
+
+
+def head_object(url, token, container, name, http_conn=None):
+ """
+ Get object info
+
+ :param url: storage URL
+ :param token: auth token
+ :param container: container name that the object is in
+ :param name: object name to get info for
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :returns: a dict containing the response's headers (all header names will
+ be lowercase)
+ :raises ClientException: HTTP HEAD request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url)
+ path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
+ method = 'HEAD'
+ headers = {'X-Auth-Token': token}
+ conn.request(method, path, '', headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log(('%s?%s' % (url, path), 'POST',),
+ {'headers': headers}, resp, body)
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Object HEAD failed', http_scheme=parsed.scheme,
+ http_host=conn.host, http_port=conn.port,
+ http_path=path, http_status=resp.status,
+ http_reason=resp.reason,
+ http_response_content=body)
+ resp_headers = {}
+ for header, value in resp.getheaders():
+ resp_headers[header.lower()] = value
+ return resp_headers
+
+
+def put_object(url, token=None, container=None, name=None, contents=None,
+ content_length=None, etag=None, chunk_size=65536,
+ content_type=None, headers=None, http_conn=None, proxy=None):
+ """
+ Put an object
+
+ :param url: storage URL
+ :param token: auth token; if None, no token will be sent
+ :param container: container name that the object is in; if None, the
+ container name is expected to be part of the url
+ :param name: object name to put; if None, the object name is expected to be
+ part of the url
+ :param contents: a string or a file like object to read object data from;
+ if None, a zero-byte put will be done
+ :param content_length: value to send as content-length header; also limits
+ the amount read from contents; if None, it will be
+ computed via the contents or chunked transfer
+ encoding will be used
+ :param etag: etag of contents; if None, no etag will be sent
+ :param chunk_size: chunk size of data to write; default 65536
+ :param content_type: value to send as content-type header; if None, no
+ content-type will be set (remote end will likely try
+ to auto-detect it)
+ :param headers: additional headers to include in the request, if any
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :param proxy: proxy to connect through, if any; None by default; str of the
+ format 'http://127.0.0.1:8888' to set one
+ :returns: etag from server response
+ :raises ClientException: HTTP PUT request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url, proxy=proxy)
+ path = parsed.path
+ if container:
+ path = '%s/%s' % (path.rstrip('/'), quote(container))
+ if name:
+ path = '%s/%s' % (path.rstrip('/'), quote(name))
+ if headers:
+ headers = dict(headers)
+ else:
+ headers = {}
+ if token:
+ headers['X-Auth-Token'] = token
+ if etag:
+ headers['ETag'] = etag.strip('"')
+ if content_length is not None:
+ headers['Content-Length'] = str(content_length)
+ else:
+ for n, v in headers.iteritems():
+ if n.lower() == 'content-length':
+ content_length = int(v)
+ if content_type is not None:
+ headers['Content-Type'] = content_type
+ if not contents:
+ headers['Content-Length'] = '0'
+ if hasattr(contents, 'read'):
+ conn.putrequest('PUT', path)
+ for header, value in headers.iteritems():
+ conn.putheader(header, value)
+ if content_length is None:
+ conn.putheader('Transfer-Encoding', 'chunked')
+ conn.endheaders()
+ chunk = contents.read(chunk_size)
+ while chunk:
+ conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
+ chunk = contents.read(chunk_size)
+ conn.send('0\r\n\r\n')
+ else:
+ conn.endheaders()
+ left = content_length
+ while left > 0:
+ size = chunk_size
+ if size > left:
+ size = left
+ chunk = contents.read(size)
+ conn.send(chunk)
+ left -= len(chunk)
+ else:
+ conn.request('PUT', path, contents, headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ headers = {'X-Auth-Token': token}
+ http_log(('%s?%s' % (url, path), 'PUT',),
+ {'headers': headers}, resp, body)
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Object PUT failed', http_scheme=parsed.scheme,
+ http_host=conn.host, http_port=conn.port,
+ http_path=path, http_status=resp.status,
+ http_reason=resp.reason,
+ http_response_content=body)
+ return resp.getheader('etag', '').strip('"')
+
+
+def post_object(url, token, container, name, headers, http_conn=None):
+ """
+ Update object metadata
+
+ :param url: storage URL
+ :param token: auth token
+ :param container: container name that the object is in
+ :param name: name of the object to update
+ :param headers: additional headers to include in the request
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :raises ClientException: HTTP POST request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url)
+ path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
+ headers['X-Auth-Token'] = token
+ conn.request('POST', path, '', headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log(('%s?%s' % (url, path), 'POST',),
+ {'headers': headers}, resp, body)
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Object POST failed', http_scheme=parsed.scheme,
+ http_host=conn.host, http_port=conn.port,
+ http_path=path, http_status=resp.status,
+ http_reason=resp.reason,
+ http_response_content=body)
+
+
+def delete_object(url, token=None, container=None, name=None, http_conn=None,
+ headers=None, proxy=None):
+ """
+ Delete object
+
+ :param url: storage URL
+ :param token: auth token; if None, no token will be sent
+ :param container: container name that the object is in; if None, the
+ container name is expected to be part of the url
+ :param name: object name to delete; if None, the object name is expected to
+ be part of the url
+ :param http_conn: HTTP connection object (If None, it will create the
+ conn object)
+ :param headers: additional headers to include in the request
+ :param proxy: proxy to connect through, if any; None by default; str of the
+ format 'http://127.0.0.1:8888' to set one
+ :raises ClientException: HTTP DELETE request failed
+ """
+ if http_conn:
+ parsed, conn = http_conn
+ else:
+ parsed, conn = http_connection(url, proxy=proxy)
+ path = parsed.path
+ if container:
+ path = '%s/%s' % (path.rstrip('/'), quote(container))
+ if name:
+ path = '%s/%s' % (path.rstrip('/'), quote(name))
+ if headers:
+ headers = dict(headers)
+ else:
+ headers = {}
+ if token:
+ headers['X-Auth-Token'] = token
+ conn.request('DELETE', path, '', headers)
+ resp = conn.getresponse()
+ body = resp.read()
+ http_log(('%s?%s' % (url, path), 'POST',),
+ {'headers': headers}, resp, body)
+ if resp.status < 200 or resp.status >= 300:
+ raise ClientException('Object DELETE failed',
+ http_scheme=parsed.scheme, http_host=conn.host,
+ http_port=conn.port, http_path=path,
+ http_status=resp.status, http_reason=resp.reason,
+ http_response_content=body)
+
+
+class Connection(object):
+ """Convenience class to make requests that will also retry the request"""
+
+ def __init__(self, authurl, user, key, retries=5, preauthurl=None,
+ preauthtoken=None, snet=False, starting_backoff=1,
+ tenant_name=None,
+ auth_version="1"):
+ """
+ :param authurl: authentication URL
+ :param user: user name to authenticate as
+ :param key: key/password to authenticate with
+ :param retries: Number of times to retry the request before failing
+ :param preauthurl: storage URL (if you have already authenticated)
+ :param preauthtoken: authentication token (if you have already
+ authenticated)
+ :param snet: use SERVICENET internal network default is False
+ :param auth_version: OpenStack auth version, default is 1.0
+ :param tenant_name: The tenant/account name, required when connecting
+ to a auth 2.0 system.
+ """
+ self.authurl = authurl
+ self.user = user
+ self.key = key
+ self.retries = retries
+ self.http_conn = None
+ self.url = preauthurl
+ self.token = preauthtoken
+ self.attempts = 0
+ self.snet = snet
+ self.starting_backoff = starting_backoff
+ self.auth_version = auth_version
+ self.tenant_name = tenant_name
+
+ def get_auth(self):
+ return get_auth(self.authurl, self.user,
+ self.key, snet=self.snet,
+ tenant_name=self.tenant_name,
+ auth_version=self.auth_version)
+
+ def http_connection(self):
+ return http_connection(self.url)
+
+ def _retry(self, reset_func, func, *args, **kwargs):
+ self.attempts = 0
+ backoff = self.starting_backoff
+ while self.attempts <= self.retries:
+ self.attempts += 1
+ try:
+ if not self.url or not self.token:
+ self.url, self.token = self.get_auth()
+ self.http_conn = None
+ if not self.http_conn:
+ self.http_conn = self.http_connection()
+ kwargs['http_conn'] = self.http_conn
+ rv = func(self.url, self.token, *args, **kwargs)
+ return rv
+ except (socket.error, HTTPException):
+ if self.attempts > self.retries:
+ raise
+ self.http_conn = None
+ except ClientException, err:
+ if self.attempts > self.retries:
+ raise
+ if err.http_status == 401:
+ self.url = self.token = None
+ if self.attempts > 1:
+ raise
+ elif err.http_status == 408:
+ self.http_conn = None
+ elif 500 <= err.http_status <= 599:
+ pass
+ else:
+ raise
+ sleep(backoff)
+ backoff *= 2
+ if reset_func:
+ reset_func(func, *args, **kwargs)
+
+ def head_account(self):
+ """Wrapper for :func:`head_account`"""
+ return self._retry(None, head_account)
+
+ def get_account(self, marker=None, limit=None, prefix=None,
+ full_listing=False):
+ """Wrapper for :func:`get_account`"""
+ # TODO(unknown): With full_listing=True this will restart the entire
+ # listing with each retry. Need to make a better version that just
+ # retries where it left off.
+ return self._retry(None, get_account, marker=marker, limit=limit,
+ prefix=prefix, full_listing=full_listing)
+
+ def post_account(self, headers):
+ """Wrapper for :func:`post_account`"""
+ return self._retry(None, post_account, headers)
+
+ def head_container(self, container):
+ """Wrapper for :func:`head_container`"""
+ return self._retry(None, head_container, container)
+
+ def get_container(self, container, marker=None, limit=None, prefix=None,
+ delimiter=None, full_listing=False):
+ """Wrapper for :func:`get_container`"""
+ # TODO(unknown): With full_listing=True this will restart the entire
+ # listing with each retry. Need to make a better version that just
+ # retries where it left off.
+ return self._retry(None, get_container, container, marker=marker,
+ limit=limit, prefix=prefix, delimiter=delimiter,
+ full_listing=full_listing)
+
+ def put_container(self, container, headers=None):
+ """Wrapper for :func:`put_container`"""
+ return self._retry(None, put_container, container, headers=headers)
+
+ def post_container(self, container, headers):
+ """Wrapper for :func:`post_container`"""
+ return self._retry(None, post_container, container, headers)
+
+ def delete_container(self, container):
+ """Wrapper for :func:`delete_container`"""
+ return self._retry(None, delete_container, container)
+
+ def head_object(self, container, obj):
+ """Wrapper for :func:`head_object`"""
+ return self._retry(None, head_object, container, obj)
+
+ def get_object(self, container, obj, resp_chunk_size=None):
+ """Wrapper for :func:`get_object`"""
+ return self._retry(None, get_object, container, obj,
+ resp_chunk_size=resp_chunk_size)
+
+ def put_object(self, container, obj, contents, content_length=None,
+ etag=None, chunk_size=65536, content_type=None,
+ headers=None):
+ """Wrapper for :func:`put_object`"""
+
+ def _default_reset(*args, **kwargs):
+ raise ClientException('put_object(%r, %r, ...) failure and no '
+ 'ability to reset contents for reupload.'
+ % (container, obj))
+
+ reset_func = _default_reset
+ tell = getattr(contents, 'tell', None)
+ seek = getattr(contents, 'seek', None)
+ if tell and seek:
+ orig_pos = tell()
+ reset_func = lambda *a, **k: seek(orig_pos)
+ elif not contents:
+ reset_func = lambda *a, **k: None
+
+ return self._retry(reset_func, put_object, container, obj, contents,
+ content_length=content_length, etag=etag,
+ chunk_size=chunk_size, content_type=content_type,
+ headers=headers)
+
+ def post_object(self, container, obj, headers):
+ """Wrapper for :func:`post_object`"""
+ return self._retry(None, post_object, container, obj, headers)
+
+ def delete_object(self, container, obj):
+ """Wrapper for :func:`delete_object`"""
+ return self._retry(None, delete_object, container, obj)