tempest/tempest/common/rest_client.py

637 lines
25 KiB
Python

# Copyright 2012 OpenStack Foundation
# Copyright 2013 IBM Corp.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import inspect
import json
from lxml import etree
import re
import time
import jsonschema
from tempest.common import http
from tempest.common import xml_utils as common
from tempest import config
from tempest import exceptions
from tempest.openstack.common import log as logging
CONF = config.CONF
# redrive rate limited calls at most twice
MAX_RECURSION_DEPTH = 2
TOKEN_CHARS_RE = re.compile('^[-A-Za-z0-9+/=]*$')
# All the successful HTTP status codes from RFC 2616
HTTP_SUCCESS = (200, 201, 202, 203, 204, 205, 206)
class RestClient(object):
TYPE = "json"
# This is used by _parse_resp method
# Redefine it for purposes of your xml service client
# List should contain top-xml_tag-names of data, which is like list/array
# For example, in keystone it is users, roles, tenants and services
# All of it has children with same tag-names
list_tags = []
# This is used by _parse_resp method too
# Used for selection of dict-like xmls,
# like metadata for Vms in nova, and volumes in cinder
dict_tags = ["metadata", ]
LOG = logging.getLogger(__name__)
def __init__(self, auth_provider):
self.auth_provider = auth_provider
self.endpoint_url = None
self.service = None
# The version of the API this client implements
self.api_version = None
self._skip_path = False
self.build_interval = CONF.compute.build_interval
self.build_timeout = CONF.compute.build_timeout
self.general_header_lc = set(('cache-control', 'connection',
'date', 'pragma', 'trailer',
'transfer-encoding', 'via',
'warning'))
self.response_header_lc = set(('accept-ranges', 'age', 'etag',
'location', 'proxy-authenticate',
'retry-after', 'server',
'vary', 'www-authenticate'))
dscv = CONF.identity.disable_ssl_certificate_validation
self.http_obj = http.ClosingHttp(
disable_ssl_certificate_validation=dscv)
def _get_type(self):
return self.TYPE
def get_headers(self, accept_type=None, send_type=None):
if accept_type is None:
accept_type = self._get_type()
if send_type is None:
send_type = self._get_type()
return {'Content-Type': 'application/%s' % send_type,
'Accept': 'application/%s' % accept_type}
def __str__(self):
STRING_LIMIT = 80
str_format = ("config:%s, service:%s, base_url:%s, "
"filters: %s, build_interval:%s, build_timeout:%s"
"\ntoken:%s..., \nheaders:%s...")
return str_format % (CONF, self.service, self.base_url,
self.filters, self.build_interval,
self.build_timeout,
str(self.token)[0:STRING_LIMIT],
str(self.get_headers())[0:STRING_LIMIT])
def _get_region(self, service):
"""
Returns the region for a specific service
"""
service_region = None
for cfgname in dir(CONF._config):
# Find all config.FOO.catalog_type and assume FOO is a service.
cfg = getattr(CONF, cfgname)
catalog_type = getattr(cfg, 'catalog_type', None)
if catalog_type == service:
service_region = getattr(cfg, 'region', None)
if not service_region:
service_region = CONF.identity.region
return service_region
def _get_endpoint_type(self, service):
"""
Returns the endpoint type for a specific service
"""
# If the client requests a specific endpoint type, then be it
if self.endpoint_url:
return self.endpoint_url
endpoint_type = None
for cfgname in dir(CONF._config):
# Find all config.FOO.catalog_type and assume FOO is a service.
cfg = getattr(CONF, cfgname)
catalog_type = getattr(cfg, 'catalog_type', None)
if catalog_type == service:
endpoint_type = getattr(cfg, 'endpoint_type', 'publicURL')
break
# Special case for compute v3 service which hasn't its own
# configuration group
else:
if service == CONF.compute.catalog_v3_type:
endpoint_type = CONF.compute.endpoint_type
return endpoint_type
@property
def user(self):
return self.auth_provider.credentials.get('username', None)
@property
def tenant_name(self):
return self.auth_provider.credentials.get('tenant_name', None)
@property
def password(self):
return self.auth_provider.credentials.get('password', None)
@property
def base_url(self):
return self.auth_provider.base_url(filters=self.filters)
@property
def token(self):
return self.auth_provider.get_token()
@property
def filters(self):
_filters = dict(
service=self.service,
endpoint_type=self._get_endpoint_type(self.service),
region=self._get_region(self.service)
)
if self.api_version is not None:
_filters['api_version'] = self.api_version
if self._skip_path:
_filters['skip_path'] = self._skip_path
return _filters
def skip_path(self):
"""
When set, ignore the path part of the base URL from the catalog
"""
self._skip_path = True
def reset_path(self):
"""
When reset, use the base URL from the catalog as-is
"""
self._skip_path = False
def expected_success(self, expected_code, read_code):
assert_msg = ("This function only allowed to use for HTTP status"
"codes which explicitly defined in the RFC 2616. {0}"
" is not a defined Success Code!").format(expected_code)
assert expected_code in HTTP_SUCCESS, assert_msg
# NOTE(afazekas): the http status code above 400 is processed by
# the _error_checker method
if read_code < 400 and read_code != expected_code:
pattern = """Unexpected http success status code {0},
The expected status code is {1}"""
details = pattern.format(read_code, expected_code)
raise exceptions.InvalidHttpSuccessCode(details)
def post(self, url, body, headers=None, extra_headers=False):
return self.request('POST', url, extra_headers, headers, body)
def get(self, url, headers=None, extra_headers=False):
return self.request('GET', url, extra_headers, headers)
def delete(self, url, headers=None, body=None, extra_headers=False):
return self.request('DELETE', url, extra_headers, headers, body)
def patch(self, url, body, headers=None, extra_headers=False):
return self.request('PATCH', url, extra_headers, headers, body)
def put(self, url, body, headers=None, extra_headers=False):
return self.request('PUT', url, extra_headers, headers, body)
def head(self, url, headers=None, extra_headers=False):
return self.request('HEAD', url, extra_headers, headers)
def copy(self, url, headers=None, extra_headers=False):
return self.request('COPY', url, extra_headers, headers)
def get_versions(self):
resp, body = self.get('')
body = self._parse_resp(body)
versions = map(lambda x: x['id'], body)
return resp, versions
def _find_caller(self):
"""Find the caller class and test name.
Because we know that the interesting things that call us are
test_* methods, and various kinds of setUp / tearDown, we
can look through the call stack to find appropriate methods,
and the class we were in when those were called.
"""
caller_name = None
names = []
frame = inspect.currentframe()
is_cleanup = False
# Start climbing the ladder until we hit a good method
while True:
try:
frame = frame.f_back
name = frame.f_code.co_name
names.append(name)
if re.search("^(test_|setUp|tearDown)", name):
cname = ""
if 'self' in frame.f_locals:
cname = frame.f_locals['self'].__class__.__name__
if 'cls' in frame.f_locals:
cname = frame.f_locals['cls'].__name__
caller_name = cname + ":" + name
break
elif re.search("^_run_cleanup", name):
is_cleanup = True
else:
cname = ""
if 'self' in frame.f_locals:
cname = frame.f_locals['self'].__class__.__name__
if 'cls' in frame.f_locals:
cname = frame.f_locals['cls'].__name__
# the fact that we are running cleanups is indicated pretty
# deep in the stack, so if we see that we want to just
# start looking for a real class name, and declare victory
# once we do.
if is_cleanup and cname:
if not re.search("^RunTest", cname):
caller_name = cname + ":_run_cleanups"
break
except Exception:
break
# prevents frame leaks
del frame
if caller_name is None:
self.LOG.debug("Sane call name not found in %s" % names)
return caller_name
def _get_request_id(self, resp):
for i in ('x-openstack-request-id', 'x-compute-request-id'):
if i in resp:
return resp[i]
return ""
def _log_request(self, method, req_url, resp,
secs="", req_headers=None,
req_body=None, resp_body=None):
# if we have the request id, put it in the right part of the log
extra = dict(request_id=self._get_request_id(resp))
# NOTE(sdague): while we still have 6 callers to this function
# we're going to just provide work around on who is actually
# providing timings by gracefully adding no content if they don't.
# Once we're down to 1 caller, clean this up.
caller_name = self._find_caller()
if secs:
secs = " %.3fs" % secs
self.LOG.info(
'Request (%s): %s %s %s%s' % (
caller_name,
resp['status'],
method,
req_url,
secs),
extra=extra)
# We intentionally duplicate the info content because in a parallel
# world this is important to match
trace_regex = CONF.debug.trace_requests
if trace_regex and re.search(trace_regex, caller_name):
log_fmt = """Request (%s): %s %s %s%s
Request - Headers: %s
Body: %s
Response - Headers: %s
Body: %s"""
self.LOG.debug(
log_fmt % (
caller_name,
resp['status'],
method,
req_url,
secs,
str(req_headers),
str(req_body)[:2048],
str(resp),
str(resp_body)[:2048]),
extra=extra)
def _parse_resp(self, body):
if self._get_type() is "json":
body = json.loads(body)
# We assume, that if the first value of the deserialized body's
# item set is a dict or a list, that we just return the first value
# of deserialized body.
# Essentially "cutting out" the first placeholder element in a body
# that looks like this:
#
# {
# "users": [
# ...
# ]
# }
try:
# Ensure there are not more than one top-level keys
if len(body.keys()) > 1:
return body
# Just return the "wrapped" element
first_key, first_item = body.items()[0]
if isinstance(first_item, (dict, list)):
return first_item
except (ValueError, IndexError):
pass
return body
elif self._get_type() is "xml":
element = etree.fromstring(body)
if any(s in element.tag for s in self.dict_tags):
# Parse dictionary-like xmls (metadata, etc)
dictionary = {}
for el in element.getchildren():
dictionary[u"%s" % el.get("key")] = u"%s" % el.text
return dictionary
if any(s in element.tag for s in self.list_tags):
# Parse list-like xmls (users, roles, etc)
array = []
for child in element.getchildren():
array.append(common.xml_to_json(child))
return array
# Parse one-item-like xmls (user, role, etc)
return common.xml_to_json(element)
def response_checker(self, method, url, headers, body, resp, resp_body):
if (resp.status in set((204, 205, 304)) or resp.status < 200 or
method.upper() == 'HEAD') and resp_body:
raise exceptions.ResponseWithNonEmptyBody(status=resp.status)
# NOTE(afazekas):
# If the HTTP Status Code is 205
# 'The response MUST NOT include an entity.'
# A HTTP entity has an entity-body and an 'entity-header'.
# In the HTTP response specification (Section 6) the 'entity-header'
# 'generic-header' and 'response-header' are in OR relation.
# All headers not in the above two group are considered as entity
# header in every interpretation.
if (resp.status == 205 and
0 != len(set(resp.keys()) - set(('status',)) -
self.response_header_lc - self.general_header_lc)):
raise exceptions.ResponseWithEntity()
# NOTE(afazekas)
# Now the swift sometimes (delete not empty container)
# returns with non json error response, we can create new rest class
# for swift.
# Usually RFC2616 says error responses SHOULD contain an explanation.
# The warning is normal for SHOULD/SHOULD NOT case
# Likely it will cause an error
if not resp_body and resp.status >= 400:
self.LOG.warning("status >= 400 response with empty body")
def _request(self, method, url, headers=None, body=None):
"""A simple HTTP request interface."""
# Authenticate the request with the auth provider
req_url, req_headers, req_body = self.auth_provider.auth_request(
method, url, headers, body, self.filters)
# Do the actual request, and time it
start = time.time()
resp, resp_body = self.http_obj.request(
req_url, method, headers=req_headers, body=req_body)
end = time.time()
self._log_request(method, req_url, resp, secs=(end - start),
req_headers=req_headers, req_body=req_body,
resp_body=resp_body)
# Verify HTTP response codes
self.response_checker(method, url, req_headers, req_body, resp,
resp_body)
return resp, resp_body
def request(self, method, url, extra_headers=False, headers=None,
body=None):
# if extra_headers is True
# default headers would be added to headers
retry = 0
if headers is None:
# NOTE(vponomaryov): if some client do not need headers,
# it should explicitly pass empty dict
headers = self.get_headers()
elif extra_headers:
try:
headers = headers.copy()
headers.update(self.get_headers())
except (ValueError, TypeError):
headers = self.get_headers()
resp, resp_body = self._request(method, url,
headers=headers, body=body)
while (resp.status == 413 and
'retry-after' in resp and
not self.is_absolute_limit(
resp, self._parse_resp(resp_body)) and
retry < MAX_RECURSION_DEPTH):
retry += 1
delay = int(resp['retry-after'])
time.sleep(delay)
resp, resp_body = self._request(method, url,
headers=headers, body=body)
self._error_checker(method, url, headers, body,
resp, resp_body)
return resp, resp_body
def _error_checker(self, method, url,
headers, body, resp, resp_body):
# NOTE(mtreinish): Check for httplib response from glance_http. The
# object can't be used here because importing httplib breaks httplib2.
# If another object from a class not imported were passed here as
# resp this could possibly fail
if str(type(resp)) == "<type 'instance'>":
ctype = resp.getheader('content-type')
else:
try:
ctype = resp['content-type']
# NOTE(mtreinish): Keystone delete user responses doesn't have a
# content-type header. (They don't have a body) So just pretend it
# is set.
except KeyError:
ctype = 'application/json'
# It is not an error response
if resp.status < 400:
return
JSON_ENC = ['application/json', 'application/json; charset=utf-8']
# NOTE(mtreinish): This is for compatibility with Glance and swift
# APIs. These are the return content types that Glance api v1
# (and occasionally swift) are using.
TXT_ENC = ['text/plain', 'text/html', 'text/html; charset=utf-8',
'text/plain; charset=utf-8']
XML_ENC = ['application/xml', 'application/xml; charset=utf-8']
if ctype.lower() in JSON_ENC or ctype.lower() in XML_ENC:
parse_resp = True
elif ctype.lower() in TXT_ENC:
parse_resp = False
else:
raise exceptions.InvalidContentType(str(resp.status))
if resp.status == 401 or resp.status == 403:
raise exceptions.Unauthorized(resp_body)
if resp.status == 404:
raise exceptions.NotFound(resp_body)
if resp.status == 400:
if parse_resp:
resp_body = self._parse_resp(resp_body)
raise exceptions.BadRequest(resp_body)
if resp.status == 409:
if parse_resp:
resp_body = self._parse_resp(resp_body)
raise exceptions.Conflict(resp_body)
if resp.status == 413:
if parse_resp:
resp_body = self._parse_resp(resp_body)
if self.is_absolute_limit(resp, resp_body):
raise exceptions.OverLimit(resp_body)
else:
raise exceptions.RateLimitExceeded(resp_body)
if resp.status == 422:
if parse_resp:
resp_body = self._parse_resp(resp_body)
raise exceptions.UnprocessableEntity(resp_body)
if resp.status in (500, 501):
message = resp_body
if parse_resp:
try:
resp_body = self._parse_resp(resp_body)
except ValueError:
# If response body is a non-json string message.
# Use resp_body as is and raise InvalidResponseBody
# exception.
raise exceptions.InvalidHTTPResponseBody(message)
else:
if isinstance(resp_body, dict):
# I'm seeing both computeFault
# and cloudServersFault come back.
# Will file a bug to fix, but leave as is for now.
if 'cloudServersFault' in resp_body:
message = resp_body['cloudServersFault']['message']
elif 'computeFault' in resp_body:
message = resp_body['computeFault']['message']
elif 'error' in resp_body: # Keystone errors
message = resp_body['error']['message']
raise exceptions.IdentityError(message)
elif 'message' in resp_body:
message = resp_body['message']
else:
message = resp_body
raise exceptions.ServerFault(message)
if resp.status >= 400:
raise exceptions.UnexpectedResponseCode(str(resp.status))
def is_absolute_limit(self, resp, resp_body):
if (not isinstance(resp_body, collections.Mapping) or
'retry-after' not in resp):
return True
if self._get_type() is "json":
over_limit = resp_body.get('overLimit', None)
if not over_limit:
return True
return 'exceed' in over_limit.get('message', 'blabla')
elif self._get_type() is "xml":
return 'exceed' in resp_body.get('message', 'blabla')
def wait_for_resource_deletion(self, id):
"""Waits for a resource to be deleted."""
start_time = int(time.time())
while True:
if self.is_resource_deleted(id):
return
if int(time.time()) - start_time >= self.build_timeout:
raise exceptions.TimeoutException
time.sleep(self.build_interval)
def is_resource_deleted(self, id):
"""
Subclasses override with specific deletion detection.
"""
message = ('"%s" does not implement is_resource_deleted'
% self.__class__.__name__)
raise NotImplementedError(message)
@classmethod
def validate_response(cls, schema, resp, body):
# Only check the response if the status code is a success code
# TODO(cyeoh): Eventually we should be able to verify that a failure
# code if it exists is something that we expect. This is explicitly
# declared in the V3 API and so we should be able to export this in
# the response schema. For now we'll ignore it.
if resp.status in HTTP_SUCCESS:
response_code = schema['status_code']
if resp.status not in response_code:
msg = ("The status code(%s) is different than the expected "
"one(%s)") % (resp.status, response_code)
raise exceptions.InvalidHttpSuccessCode(msg)
response_schema = schema.get('response_body')
if response_schema:
try:
jsonschema.validate(body, response_schema)
except jsonschema.ValidationError as ex:
msg = ("HTTP response body is invalid (%s)") % ex
raise exceptions.InvalidHTTPResponseBody(msg)
else:
if body:
msg = ("HTTP response body should not exist (%s)") % body
raise exceptions.InvalidHTTPResponseBody(msg)
class NegativeRestClient(RestClient):
"""
Version of RestClient that does not raise exceptions.
"""
def _error_checker(self, method, url,
headers, body, resp, resp_body):
pass
def send_request(self, method, url_template, resources, body=None):
url = url_template % tuple(resources)
if method == "GET":
resp, body = self.get(url)
elif method == "POST":
resp, body = self.post(url, body)
elif method == "PUT":
resp, body = self.put(url, body)
elif method == "PATCH":
resp, body = self.patch(url, body)
elif method == "HEAD":
resp, body = self.head(url)
elif method == "DELETE":
resp, body = self.delete(url)
elif method == "COPY":
resp, body = self.copy(url)
else:
assert False
return resp, body