Add API endpoint for retrieving capabilities
This endpoint will serve as a proxy for the openstack/defcore repository. Instead of the UI sending requests to GitHub directly where requests might be impeded due to cross-origin blockage or API rate-limiting, the UI will be able to send requests to the Refstack API in order to get capability data. The requests-cache library is used so as not to keep sending requests to GitHub when likely nothing has changed. With this, Refstack no longer has to worry about mirroring capability files and keeping them updated. GET /v1/capabilities - list all capability files GET /v1/capabilities/2015.03 - get contents of file 2015.03.json Change-Id: I6d327273f191e3339219e35eed8768da35c5bffb
This commit is contained in:
Paul Van Eck
parent
ff156545f5
commit
961e46df6c
|
|
@ -9,3 +9,4 @@ ChangeLog
|
|||
build/
|
||||
cover/
|
||||
dist
|
||||
*.sqlite
|
||||
|
|
|
|||
|
|
@ -135,6 +135,15 @@
|
|||
# The format for start_date and end_date parameters (string value)
|
||||
#input_date_format = %Y-%m-%d %H:%M:%S
|
||||
|
||||
# The GitHub API URL of the repository and location of the DefCore
|
||||
# capability files. This URL is used to get a listing of all capability
|
||||
# files.
|
||||
#github_api_capabilities_url = https://api.github.com/repos/openstack/defcore/contents
|
||||
|
||||
# The base URL that is used for retrieving specific capability files.
|
||||
# Capability file names will be appended to this URL to get the contents
|
||||
# of that file.
|
||||
#github_raw_base_url = https://raw.githubusercontent.com/openstack/defcore/master/
|
||||
|
||||
[database]
|
||||
|
||||
|
|
|
|||
|
|
@ -68,6 +68,20 @@ API_OPTS = [
|
|||
default='http://refstack.net/output.html?test_id=%s',
|
||||
help='Template for test result url.'
|
||||
),
|
||||
cfg.StrOpt('github_api_capabilities_url',
|
||||
default='https://api.github.com'
|
||||
'/repos/openstack/defcore/contents',
|
||||
help='The GitHub API URL of the repository and location of '
|
||||
'the DefCore capability files. This URL is used to get '
|
||||
'a listing of all capability files.'
|
||||
),
|
||||
cfg.StrOpt('github_raw_base_url',
|
||||
default='https://raw.githubusercontent.com'
|
||||
'/openstack/defcore/master/',
|
||||
help='This is the base URL that is used for retrieving '
|
||||
'specific capability files. Capability file names will '
|
||||
'be appended to this URL to get the contents of that file.'
|
||||
)
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
|
|
|||
|
|
@ -16,10 +16,14 @@
|
|||
"""Version 1 of the API."""
|
||||
|
||||
import json
|
||||
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log
|
||||
import pecan
|
||||
from pecan import rest
|
||||
import re
|
||||
import requests
|
||||
import requests_cache
|
||||
|
||||
from refstack import db
|
||||
from refstack.api import constants as const
|
||||
|
|
@ -43,6 +47,8 @@ CTRLS_OPTS = [
|
|||
CONF = cfg.CONF
|
||||
|
||||
CONF.register_opts(CTRLS_OPTS, group='api')
|
||||
# Cached requests will expire after 10 minutes.
|
||||
requests_cache.install_cache(cache_name='github_cache', expire_after=600)
|
||||
|
||||
|
||||
class BaseRestControllerWithValidation(rest.RestController):
|
||||
|
|
@ -182,8 +188,62 @@ class ResultsController(BaseRestControllerWithValidation):
|
|||
return page
|
||||
|
||||
|
||||
class CapabilitiesController(rest.RestController):
|
||||
|
||||
"""/v1/capabilities handler. This acts as a proxy for retrieving
|
||||
capability files from the openstack/defcore Github repository."""
|
||||
|
||||
@pecan.expose('json')
|
||||
def get(self):
|
||||
"""Get a list of all available capabilities."""
|
||||
try:
|
||||
response = requests.get(CONF.api.github_api_capabilities_url)
|
||||
LOG.debug("Response Status: %s / Used Requests Cache: %s" %
|
||||
(response.status_code,
|
||||
getattr(response, 'from_cache', False)))
|
||||
if response.status_code == 200:
|
||||
json = response.json()
|
||||
regex = re.compile('^[0-9]{4}\.[0-9]{2}\.json$')
|
||||
capability_files = []
|
||||
for rfile in json:
|
||||
if rfile["type"] == "file" and regex.search(rfile["name"]):
|
||||
capability_files.append(rfile["name"])
|
||||
return capability_files
|
||||
else:
|
||||
LOG.warning('Github returned non-success HTTP '
|
||||
'code: %s' % response.status_code)
|
||||
pecan.abort(response.status_code)
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
LOG.warning('An error occurred trying to get GitHub '
|
||||
'repository contents: %s' % e)
|
||||
pecan.abort(500)
|
||||
|
||||
@pecan.expose('json')
|
||||
def get_one(self, file_name):
|
||||
"""Handler for getting contents of specific capability file."""
|
||||
github_url = ''.join((CONF.api.github_raw_base_url.rstrip('/'),
|
||||
'/', file_name, ".json"))
|
||||
try:
|
||||
response = requests.get(github_url)
|
||||
LOG.debug("Response Status: %s / Used Requests Cache: %s" %
|
||||
(response.status_code,
|
||||
getattr(response, 'from_cache', False)))
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
else:
|
||||
LOG.warning('Github returned non-success HTTP '
|
||||
'code: %s' % response.status_code)
|
||||
pecan.abort(response.status_code)
|
||||
except requests.exceptions.RequestException as e:
|
||||
LOG.warning('An error occurred trying to get GitHub '
|
||||
'capability file contents: %s' % e)
|
||||
pecan.abort(500)
|
||||
|
||||
|
||||
class V1Controller(object):
|
||||
|
||||
"""Version 1 API controller root."""
|
||||
|
||||
results = ResultsController()
|
||||
capabilities = CapabilitiesController()
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
import json
|
||||
import uuid
|
||||
|
||||
import httmock
|
||||
from oslo_config import fixture as config_fixture
|
||||
import six
|
||||
import webtest.app
|
||||
|
|
@ -220,3 +221,37 @@ class TestResultsController(api.FunctionalTest):
|
|||
self.assertEqual(len(filtering_results), 3)
|
||||
for r in slice_results:
|
||||
self.assertEqual(r, filtering_results)
|
||||
|
||||
|
||||
class TestCapabilitiesController(api.FunctionalTest):
|
||||
"""Test case for CapabilitiesController."""
|
||||
|
||||
URL = '/v1/capabilities/'
|
||||
|
||||
def test_get_capability_list(self):
|
||||
@httmock.all_requests
|
||||
def github_api_mock(url, request):
|
||||
headers = {'content-type': 'application/json'}
|
||||
content = [{'name': '2015.03.json', 'type': 'file'},
|
||||
{'name': '2015.next.json', 'type': 'file'},
|
||||
{'name': '2015.03', 'type': 'dir'}]
|
||||
content = json.dumps(content)
|
||||
return httmock.response(200, content, headers, None, 5, request)
|
||||
|
||||
with httmock.HTTMock(github_api_mock):
|
||||
actual_response = self.get_json(self.URL)
|
||||
|
||||
expected_response = ['2015.03.json']
|
||||
self.assertEqual(expected_response, actual_response)
|
||||
|
||||
def test_get_capability_file(self):
|
||||
@httmock.all_requests
|
||||
def github_mock(url, request):
|
||||
content = {'foo': 'bar'}
|
||||
return httmock.response(200, content, None, None, 5, request)
|
||||
url = self.URL + "2015.03"
|
||||
with httmock.HTTMock(github_mock):
|
||||
actual_response = self.get_json(url)
|
||||
|
||||
expected_response = {'foo': 'bar'}
|
||||
self.assertEqual(expected_response, actual_response)
|
||||
|
|
|
|||
|
|
@ -15,9 +15,14 @@
|
|||
|
||||
"""Tests for API's controllers"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
import httmock
|
||||
import mock
|
||||
from oslo_config import fixture as config_fixture
|
||||
from oslotest import base
|
||||
import requests
|
||||
|
||||
from refstack.api import constants as const
|
||||
from refstack.api import utils as api_utils
|
||||
|
|
@ -25,6 +30,15 @@ from refstack.api.controllers import root
|
|||
from refstack.api.controllers import v1
|
||||
|
||||
|
||||
def safe_json_dump(content):
|
||||
if isinstance(content, (dict, list)):
|
||||
if sys.version_info[0] == 3:
|
||||
content = bytes(json.dumps(content), 'utf-8')
|
||||
else:
|
||||
content = json.dumps(content)
|
||||
return content
|
||||
|
||||
|
||||
class RootControllerTestCase(base.BaseTestCase):
|
||||
|
||||
def test_index(self):
|
||||
|
|
@ -230,6 +244,81 @@ class ResultsControllerTestCase(base.BaseTestCase):
|
|||
db_get_test.assert_called_once_with(page_number, per_page, filters)
|
||||
|
||||
|
||||
class CapabilitiesControllerTestCase(base.BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(CapabilitiesControllerTestCase, self).setUp()
|
||||
self.controller = v1.CapabilitiesController()
|
||||
|
||||
def test_get_capabilities(self):
|
||||
"""Test when getting a list of all capability files."""
|
||||
@httmock.all_requests
|
||||
def github_api_mock(url, request):
|
||||
headers = {'content-type': 'application/json'}
|
||||
content = [{'name': '2015.03.json', 'type': 'file'},
|
||||
{'name': '2015.next.json', 'type': 'file'},
|
||||
{'name': '2015.03', 'type': 'dir'}]
|
||||
content = safe_json_dump(content)
|
||||
return httmock.response(200, content, headers, None, 5, request)
|
||||
|
||||
with httmock.HTTMock(github_api_mock):
|
||||
result = self.controller.get()
|
||||
self.assertEqual(['2015.03.json'], result)
|
||||
|
||||
@mock.patch('pecan.abort')
|
||||
def test_get_capabilities_error_code(self, mock_abort):
|
||||
"""Test when the HTTP status code isn't a 200 OK. The status should
|
||||
be propogated."""
|
||||
@httmock.all_requests
|
||||
def github_api_mock(url, request):
|
||||
content = {'title': 'Not Found'}
|
||||
return httmock.response(404, content, None, None, 5, request)
|
||||
|
||||
with httmock.HTTMock(github_api_mock):
|
||||
self.controller.get()
|
||||
mock_abort.assert_called_with(404)
|
||||
|
||||
@mock.patch('requests.get')
|
||||
@mock.patch('pecan.abort')
|
||||
def test_get_capabilities_exception(self, mock_abort, mock_request):
|
||||
"""Test when the GET request raises an exception."""
|
||||
mock_request.side_effect = requests.exceptions.RequestException()
|
||||
self.controller.get()
|
||||
mock_abort.assert_called_with(500)
|
||||
|
||||
def test_get_capability_file(self):
|
||||
"""Test when getting a specific capability file"""
|
||||
@httmock.all_requests
|
||||
def github_mock(url, request):
|
||||
content = {'foo': 'bar'}
|
||||
return httmock.response(200, content, None, None, 5, request)
|
||||
|
||||
with httmock.HTTMock(github_mock):
|
||||
result = self.controller.get_one('2015.03')
|
||||
self.assertEqual({'foo': 'bar'}, result)
|
||||
|
||||
@mock.patch('pecan.abort')
|
||||
def test_get_capability_file_error_code(self, mock_abort):
|
||||
"""Test when the HTTP status code isn't a 200 OK. The status should
|
||||
be propogated."""
|
||||
@httmock.all_requests
|
||||
def github_api_mock(url, request):
|
||||
content = {'title': 'Not Found'}
|
||||
return httmock.response(404, content, None, None, 5, request)
|
||||
|
||||
with httmock.HTTMock(github_api_mock):
|
||||
self.controller.get_one('2010.03')
|
||||
mock_abort.assert_called_with(404)
|
||||
|
||||
@mock.patch('requests.get')
|
||||
@mock.patch('pecan.abort')
|
||||
def test_get_capability_file_exception(self, mock_abort, mock_request):
|
||||
"""Test when the GET request raises an exception."""
|
||||
mock_request.side_effect = requests.exceptions.RequestException()
|
||||
self.controller.get_one('2010.03')
|
||||
mock_abort.assert_called_with(500)
|
||||
|
||||
|
||||
class BaseRestControllerWithValidationTestCase(base.BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ oslo.log
|
|||
pecan>=0.8.2
|
||||
pyOpenSSL==0.13
|
||||
pycrypto>=2.6
|
||||
requests==1.2.3
|
||||
requests>=2.2.0,!=2.4.0
|
||||
requests-cache>=0.4.9
|
||||
jsonschema>=2.0.0,<3.0.0
|
||||
PyMySQL>=0.6.2,!=0.6.4
|
||||
|
|
@ -2,6 +2,7 @@ coverage>=3.6
|
|||
pep8==1.5.7
|
||||
pyflakes==0.8.1
|
||||
flake8==2.2.4
|
||||
httmock
|
||||
mock
|
||||
oslotest>=1.2.0 # Apache-2.0
|
||||
python-subunit>=0.0.18
|
||||
|
|
|
|||
Loading…
Reference in New Issue