From 17f13f7bf4cea80e8e1380fbc8295318de5be383 Mon Sep 17 00:00:00 2001
From: Dean Troyer <dtroyer@gmail.com>
Date: Tue, 13 Aug 2013 17:14:42 -0500
Subject: [PATCH] Create a new base REST API interface
* restapi module provides basic REST API support
* uses dicts rather than Resource classes
* JSON serialization/deserialization
* log requests in 'curl' format
* basic API boilerplate for create/delete/list/set/show verbs
* ignore H302 due to urllib import
Change-Id: I3cb91e44e631ee19e9f5dea19b6bac5d599d19ce
---
openstackclient/common/restapi.py | 188 +++++++++++
openstackclient/common/utils.py | 24 ++
openstackclient/shell.py | 5 +
openstackclient/tests/common/__init__.py | 14 +
openstackclient/tests/common/test_restapi.py | 320 +++++++++++++++++++
tox.ini | 2 +-
6 files changed, 552 insertions(+), 1 deletion(-)
create mode 100644 openstackclient/common/restapi.py
create mode 100644 openstackclient/tests/common/__init__.py
create mode 100644 openstackclient/tests/common/test_restapi.py
diff --git a/openstackclient/common/restapi.py b/openstackclient/common/restapi.py
new file mode 100644
index 0000000000..4cea5a06cc
--- /dev/null
+++ b/openstackclient/common/restapi.py
@@ -0,0 +1,188 @@
+# Copyright 2013 Nebula Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+"""REST API bits"""
+
+import json
+import logging
+import requests
+
+try:
+ from urllib.parse import urlencode
+except ImportError:
+ from urllib import urlencode
+
+
+_logger = logging.getLogger(__name__)
+
+
+class RESTApi(object):
+ """A REST api client that handles the interface from us to the server
+
+ RESTApi is an extension of a requests.Session that knows
+ how to do:
+ * JSON serialization/deserialization
+ * log requests in 'curl' format
+ * basic API boilerplate for create/delete/list/set/show verbs
+
+ * authentication is handled elsewhere and a token is passed in
+
+ The expectation that there will be a RESTApi object per authentication
+ token in use, i.e. project/username/auth_endpoint
+
+ On the other hand, a Client knows details about the specific REST Api that
+ it communicates with, such as the available endpoints, API versions, etc.
+ """
+
+ USER_AGENT = 'RAPI'
+
+ def __init__(
+ self,
+ os_auth=None,
+ user_agent=USER_AGENT,
+ debug=None,
+ **kwargs
+ ):
+ self.set_auth(os_auth)
+ self.debug = debug
+ self.session = requests.Session(**kwargs)
+
+ self.set_header('User-Agent', user_agent)
+ self.set_header('Content-Type', 'application/json')
+
+ def set_auth(self, os_auth):
+ """Sets the current auth blob"""
+ self.os_auth = os_auth
+
+ def set_header(self, header, content):
+ """Sets passed in headers into the session headers
+
+ Replaces existing headers!!
+ """
+ if content is None:
+ del self.session.headers[header]
+ else:
+ self.session.headers[header] = content
+
+ def request(self, method, url, **kwargs):
+ if self.os_auth:
+ self.session.headers.setdefault('X-Auth-Token', self.os_auth)
+ if 'data' in kwargs and isinstance(kwargs['data'], type({})):
+ kwargs['data'] = json.dumps(kwargs['data'])
+ log_request(method, url, headers=self.session.headers, **kwargs)
+ response = self.session.request(method, url, **kwargs)
+ log_response(response)
+ return self._error_handler(response)
+
+ def create(self, url, data=None, response_key=None, **kwargs):
+ response = self.request('POST', url, data=data, **kwargs)
+ if response_key:
+ return response.json()[response_key]
+ else:
+ return response.json()
+
+ #with self.completion_cache('human_id', self.resource_class, mode="a"):
+ # with self.completion_cache('uuid', self.resource_class, mode="a"):
+ # return self.resource_class(self, body[response_key])
+
+ def delete(self, url):
+ self.request('DELETE', url)
+
+ def list(self, url, data=None, response_key=None, **kwargs):
+ if data:
+ response = self.request('POST', url, data=data, **kwargs)
+ else:
+ kwargs.setdefault('allow_redirects', True)
+ response = self.request('GET', url, **kwargs)
+
+ return response.json()[response_key]
+
+ ###hack this for keystone!!!
+ #data = body[response_key]
+ # NOTE(ja): keystone returns values as list as {'values': [ ... ]}
+ # unlike other services which just return the list...
+ #if isinstance(data, dict):
+ # try:
+ # data = data['values']
+ # except KeyError:
+ # pass
+
+ #with self.completion_cache('human_id', obj_class, mode="w"):
+ # with self.completion_cache('uuid', obj_class, mode="w"):
+ # return [obj_class(self, res, loaded=True)
+ # for res in data if res]
+
+ def set(self, url, data=None, response_key=None, **kwargs):
+ response = self.request('PUT', url, data=data)
+ if data:
+ if response_key:
+ return response.json()[response_key]
+ else:
+ return response.json()
+ else:
+ return None
+
+ def show(self, url, response_key=None, **kwargs):
+ response = self.request('GET', url, **kwargs)
+ if response_key:
+ return response.json()[response_key]
+ else:
+ return response.json()
+
+ def _error_handler(self, response):
+ if response.status_code < 200 or response.status_code >= 300:
+ _logger.debug(
+ "ERROR: %s",
+ response.text,
+ )
+ response.raise_for_status()
+ return response
+
+
+def log_request(method, url, **kwargs):
+ # put in an early exit if debugging is not enabled?
+ if 'params' in kwargs and kwargs['params'] != {}:
+ url += '?' + urlencode(kwargs['params'])
+
+ string_parts = [
+ "curl -i",
+ "-X '%s'" % method,
+ "'%s'" % url,
+ ]
+
+ for element in kwargs['headers']:
+ header = " -H '%s: %s'" % (element, kwargs['headers'][element])
+ string_parts.append(header)
+
+ _logger.debug("REQ: %s" % " ".join(string_parts))
+ if 'data' in kwargs:
+ _logger.debug("REQ BODY: %s\n" % (kwargs['data']))
+
+
+def log_response(response):
+ _logger.debug(
+ "RESP: [%s] %s\n",
+ response.status_code,
+ response.headers,
+ )
+ if response._content_consumed:
+ _logger.debug(
+ "RESP BODY: %s\n",
+ response.text,
+ )
+ _logger.debug(
+ "encoding: %s",
+ response.encoding,
+ )
diff --git a/openstackclient/common/utils.py b/openstackclient/common/utils.py
index f72bb505a5..91a20895b2 100644
--- a/openstackclient/common/utils.py
+++ b/openstackclient/common/utils.py
@@ -115,6 +115,30 @@ def get_item_properties(item, fields, mixed_case_fields=[], formatters={}):
return tuple(row)
+def get_dict_properties(item, fields, mixed_case_fields=[], formatters={}):
+ """Return a tuple containing the item properties.
+
+ :param item: a single dict resource
+ :param fields: tuple of strings with the desired field names
+ :param mixed_case_fields: tuple of field names to preserve case
+ :param formatters: dictionary mapping field names to callables
+ to format the values
+ """
+ row = []
+
+ for field in fields:
+ if field in mixed_case_fields:
+ field_name = field.replace(' ', '_')
+ else:
+ field_name = field.lower().replace(' ', '_')
+ data = item[field_name] if field_name in item else ''
+ if field in formatters:
+ row.append(formatters[field](data))
+ else:
+ row.append(data)
+ return tuple(row)
+
+
def string_to_bool(arg):
return arg.strip().lower() in ('t', 'true', 'yes', '1')
diff --git a/openstackclient/shell.py b/openstackclient/shell.py
index b66611b163..91b02a2b0a 100644
--- a/openstackclient/shell.py
+++ b/openstackclient/shell.py
@@ -29,6 +29,7 @@ from openstackclient.common import clientmanager
from openstackclient.common import commandmanager
from openstackclient.common import exceptions as exc
from openstackclient.common import openstackkeyring
+from openstackclient.common import restapi
from openstackclient.common import utils
@@ -368,6 +369,9 @@ class OpenStackShell(app.App):
if self.options.deferred_help:
self.DeferredHelpAction(self.parser, self.parser, None, None)
+ # Set up common client session
+ self.restapi = restapi.RESTApi()
+
# If the user is not asking for help, make sure they
# have given us auth.
cmd_name = None
@@ -376,6 +380,7 @@ class OpenStackShell(app.App):
cmd_factory, cmd_name, sub_argv = cmd_info
if self.interactive_mode or cmd_name != 'help':
self.authenticate_user()
+ self.restapi.set_auth(self.client_manager.identity.auth_token)
def prepare_to_run_command(self, cmd):
"""Set up auth and API versions"""
diff --git a/openstackclient/tests/common/__init__.py b/openstackclient/tests/common/__init__.py
new file mode 100644
index 0000000000..c534c012e8
--- /dev/null
+++ b/openstackclient/tests/common/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
diff --git a/openstackclient/tests/common/test_restapi.py b/openstackclient/tests/common/test_restapi.py
new file mode 100644
index 0000000000..4b83ffa460
--- /dev/null
+++ b/openstackclient/tests/common/test_restapi.py
@@ -0,0 +1,320 @@
+# Copyright 2013 Nebula Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+"""Test rest module"""
+
+import json
+import mock
+
+import requests
+
+from openstackclient.common import restapi
+from openstackclient.tests import utils
+
+fake_auth = '11223344556677889900'
+fake_url = 'http://gopher.com'
+fake_key = 'gopher'
+fake_keys = 'gophers'
+fake_gopher_mac = {
+ 'id': 'g1',
+ 'name': 'mac',
+ 'actor': 'Mel Blanc',
+}
+fake_gopher_tosh = {
+ 'id': 'g2',
+ 'name': 'tosh',
+ 'actor': 'Stan Freeberg',
+}
+fake_gopher_single = {
+ fake_key: fake_gopher_mac,
+}
+fake_gopher_list = {
+ fake_keys:
+ [
+ fake_gopher_mac,
+ fake_gopher_tosh,
+ ]
+}
+
+
+class FakeResponse(requests.Response):
+ def __init__(self, headers={}, status_code=None, data=None, encoding=None):
+ super(FakeResponse, self).__init__()
+
+ self.status_code = status_code
+
+ self.headers.update(headers)
+ self._content = json.dumps(data)
+
+
+@mock.patch('openstackclient.common.restapi.requests.Session')
+class TestRESTApi(utils.TestCase):
+
+ def test_request_get(self, session_mock):
+ resp = FakeResponse(status_code=200, data=fake_gopher_single)
+ session_mock.return_value = mock.MagicMock(
+ request=mock.MagicMock(return_value=resp),
+ )
+
+ api = restapi.RESTApi()
+ gopher = api.request('GET', fake_url)
+ session_mock.return_value.request.assert_called_with(
+ 'GET',
+ fake_url,
+ )
+ self.assertEqual(gopher.status_code, 200)
+ self.assertEqual(gopher.json(), fake_gopher_single)
+
+ def test_request_get_return_300(self, session_mock):
+ resp = FakeResponse(status_code=300, data=fake_gopher_single)
+ session_mock.return_value = mock.MagicMock(
+ request=mock.MagicMock(return_value=resp),
+ )
+
+ api = restapi.RESTApi()
+ gopher = api.request('GET', fake_url)
+ session_mock.return_value.request.assert_called_with(
+ 'GET',
+ fake_url,
+ )
+ self.assertEqual(gopher.status_code, 300)
+ self.assertEqual(gopher.json(), fake_gopher_single)
+
+ def test_request_get_fail_404(self, session_mock):
+ resp = FakeResponse(status_code=404, data=fake_gopher_single)
+ session_mock.return_value = mock.MagicMock(
+ request=mock.MagicMock(return_value=resp),
+ )
+
+ api = restapi.RESTApi()
+ self.assertRaises(requests.HTTPError, api.request, 'GET', fake_url)
+ session_mock.return_value.request.assert_called_with(
+ 'GET',
+ fake_url,
+ )
+
+ def test_request_get_auth(self, session_mock):
+ resp = FakeResponse(data=fake_gopher_single)
+ session_mock.return_value = mock.MagicMock(
+ request=mock.MagicMock(return_value=resp),
+ headers=mock.MagicMock(return_value={}),
+ )
+
+ api = restapi.RESTApi(os_auth=fake_auth)
+ gopher = api.request('GET', fake_url)
+ session_mock.return_value.headers.setdefault.assert_called_with(
+ 'X-Auth-Token',
+ fake_auth,
+ )
+ session_mock.return_value.request.assert_called_with(
+ 'GET',
+ fake_url,
+ )
+ self.assertEqual(gopher.json(), fake_gopher_single)
+
+ def test_request_get_header(self, session_mock):
+ resp = FakeResponse(data=fake_gopher_single)
+ session_mock.return_value = mock.MagicMock(
+ request=mock.MagicMock(return_value=resp),
+ headers=mock.MagicMock(return_value={}),
+ )
+
+ api = restapi.RESTApi(user_agent='fake_agent')
+ api.set_header('X-Fake-Header', 'wb')
+ gopher = api.request('GET', fake_url)
+ session_mock.return_value.headers.__setitem__.assert_any_call(
+ 'Content-Type',
+ 'application/json',
+ )
+ session_mock.return_value.headers.__setitem__.assert_any_call(
+ 'User-Agent',
+ 'fake_agent',
+ )
+ session_mock.return_value.headers.__setitem__.assert_any_call(
+ 'X-Fake-Header',
+ 'wb',
+ )
+ session_mock.return_value.request.assert_called_with(
+ 'GET',
+ fake_url,
+ )
+ self.assertEqual(gopher.json(), fake_gopher_single)
+
+ api.set_header('X-Fake-Header', None)
+ session_mock.return_value.headers.__delitem__.assert_any_call(
+ 'X-Fake-Header',
+ )
+
+ def test_request_post(self, session_mock):
+ resp = FakeResponse(data=fake_gopher_single)
+ session_mock.return_value = mock.MagicMock(
+ request=mock.MagicMock(return_value=resp),
+ )
+
+ api = restapi.RESTApi()
+ data = fake_gopher_tosh
+ gopher = api.request('POST', fake_url, data=data)
+ session_mock.return_value.request.assert_called_with(
+ 'POST',
+ fake_url,
+ data=json.dumps(data),
+ )
+ self.assertEqual(gopher.json(), fake_gopher_single)
+
+ def test_create(self, session_mock):
+ resp = FakeResponse(data=fake_gopher_single)
+ session_mock.return_value = mock.MagicMock(
+ request=mock.MagicMock(return_value=resp),
+ )
+
+ api = restapi.RESTApi()
+ data = fake_gopher_mac
+
+ # Test no key
+ gopher = api.create(fake_url, data=data)
+ session_mock.return_value.request.assert_called_with(
+ 'POST',
+ fake_url,
+ data=json.dumps(data),
+ )
+ self.assertEqual(gopher, fake_gopher_single)
+
+ # Test with key
+ gopher = api.create(fake_url, data=data, response_key=fake_key)
+ session_mock.return_value.request.assert_called_with(
+ 'POST',
+ fake_url,
+ data=json.dumps(data),
+ )
+ self.assertEqual(gopher, fake_gopher_mac)
+
+ def test_delete(self, session_mock):
+ resp = FakeResponse(data=None)
+ session_mock.return_value = mock.MagicMock(
+ request=mock.MagicMock(return_value=resp),
+ )
+
+ api = restapi.RESTApi()
+ gopher = api.delete(fake_url)
+ session_mock.return_value.request.assert_called_with(
+ 'DELETE',
+ fake_url,
+ )
+ self.assertEqual(gopher, None)
+
+ def test_list(self, session_mock):
+ resp = FakeResponse(data=fake_gopher_list)
+ session_mock.return_value = mock.MagicMock(
+ request=mock.MagicMock(return_value=resp),
+ )
+
+ # test base
+ api = restapi.RESTApi()
+ gopher = api.list(fake_url, response_key=fake_keys)
+ session_mock.return_value.request.assert_called_with(
+ 'GET',
+ fake_url,
+ allow_redirects=True,
+ )
+ self.assertEqual(gopher, [fake_gopher_mac, fake_gopher_tosh])
+
+ # test body
+ api = restapi.RESTApi()
+ data = {'qwerty': 1}
+ gopher = api.list(fake_url, response_key=fake_keys, data=data)
+ session_mock.return_value.request.assert_called_with(
+ 'POST',
+ fake_url,
+ data=json.dumps(data),
+ )
+ self.assertEqual(gopher, [fake_gopher_mac, fake_gopher_tosh])
+
+ # test query params
+ api = restapi.RESTApi()
+ params = {'qaz': '123'}
+ gophers = api.list(fake_url, response_key=fake_keys, params=params)
+ session_mock.return_value.request.assert_called_with(
+ 'GET',
+ fake_url,
+ allow_redirects=True,
+ params=params,
+ )
+ self.assertEqual(gophers, [fake_gopher_mac, fake_gopher_tosh])
+
+ def test_set(self, session_mock):
+ new_gopher = fake_gopher_single
+ new_gopher[fake_key]['name'] = 'Chip'
+ resp = FakeResponse(data=fake_gopher_single)
+ session_mock.return_value = mock.MagicMock(
+ request=mock.MagicMock(return_value=resp),
+ )
+
+ api = restapi.RESTApi()
+ data = fake_gopher_mac
+ data['name'] = 'Chip'
+
+ # Test no data, no key
+ gopher = api.set(fake_url)
+ session_mock.return_value.request.assert_called_with(
+ 'PUT',
+ fake_url,
+ data=None,
+ )
+ self.assertEqual(gopher, None)
+
+ # Test data, no key
+ gopher = api.set(fake_url, data=data)
+ session_mock.return_value.request.assert_called_with(
+ 'PUT',
+ fake_url,
+ data=json.dumps(data),
+ )
+ self.assertEqual(gopher, fake_gopher_single)
+
+ # NOTE:(dtroyer): Key and no data is not tested as without data
+ # the response_key is moot
+
+ # Test data and key
+ gopher = api.set(fake_url, data=data, response_key=fake_key)
+ session_mock.return_value.request.assert_called_with(
+ 'PUT',
+ fake_url,
+ data=json.dumps(data),
+ )
+ self.assertEqual(gopher, fake_gopher_mac)
+
+ def test_show(self, session_mock):
+ resp = FakeResponse(data=fake_gopher_single)
+ session_mock.return_value = mock.MagicMock(
+ request=mock.MagicMock(return_value=resp),
+ )
+
+ api = restapi.RESTApi()
+
+ # Test no key
+ gopher = api.show(fake_url)
+ session_mock.return_value.request.assert_called_with(
+ 'GET',
+ fake_url,
+ )
+ self.assertEqual(gopher, fake_gopher_single)
+
+ # Test with key
+ gopher = api.show(fake_url, response_key=fake_key)
+ session_mock.return_value.request.assert_called_with(
+ 'GET',
+ fake_url,
+ )
+ self.assertEqual(gopher, fake_gopher_mac)
diff --git a/tox.ini b/tox.ini
index 61bdc9dec4..e2c61af489 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,6 +23,6 @@ commands = python setup.py testr --coverage --testr-args='{posargs}'
downloadcache = ~/cache/pip
[flake8]
-ignore = E126,E202,W602,H402
+ignore = E126,E202,W602,H302,H402
show-source = True
exclude = .venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,tools