openinfra/refstack
Code Issues Proposed changes

Add API endpoint for retrieving capabilities

Browse Source 
This endpoint will serve as a proxy for the openstack/defcore
repository. Instead of the UI sending requests to GitHub directly
where requests might be impeded due to cross-origin blockage or API
rate-limiting, the UI will be able to send requests to the Refstack
API in order to get capability data. The requests-cache library is
used so as not to keep sending requests to GitHub when likely
nothing has changed. With this, Refstack no longer has to worry about
mirroring capability files and keeping them updated.

GET /v1/capabilities - list all capability files
GET /v1/capabilities/2015.03 - get contents of file 2015.03.json

Change-Id: I6d327273f191e3339219e35eed8768da35c5bffb
This commit is contained in:
Paul Van Eck 2015-04-29 13:30:15 -07:00
parent ff156545f5
commit 961e46df6c
8 changed files with 211 additions and 1 deletions

1
.gitignore vendored

@ -9,3 +9,4 @@ ChangeLog
build/
cover/
dist
*.sqlite

9
etc/refstack.conf.sample

@ -135,6 +135,15 @@
# The format for start_date and end_date parameters (string value)
#input_date_format = %Y-%m-%d %H:%M:%S
# The GitHub API URL of the repository and location of the DefCore
# capability files. This URL is used to get a listing of all capability
# files.
#github_api_capabilities_url = https://api.github.com/repos/openstack/defcore/contents
# The base URL that is used for retrieving specific capability files.
# Capability file names will be appended to this URL to get the contents
# of that file.
#github_raw_base_url = https://raw.githubusercontent.com/openstack/defcore/master/
[database]

14
refstack/api/app.py

@ -68,6 +68,20 @@ API_OPTS = [
default='http://refstack.net/output.html?test_id=%s',
help='Template for test result url.'
),
cfg.StrOpt('github_api_capabilities_url',
default='https://api.github.com'
'/repos/openstack/defcore/contents',
help='The GitHub API URL of the repository and location of '
'the DefCore capability files. This URL is used to get '
'a listing of all capability files.'
),
cfg.StrOpt('github_raw_base_url',
default='https://raw.githubusercontent.com'
'/openstack/defcore/master/',
help='This is the base URL that is used for retrieving '
'specific capability files. Capability file names will '
'be appended to this URL to get the contents of that file.'
)
]
CONF = cfg.CONF

60
refstack/api/controllers/v1.py

@ -16,10 +16,14 @@
"""Version 1 of the API."""
import json
from oslo_config import cfg
from oslo_log import log
import pecan
from pecan import rest
import re
import requests
import requests_cache
from refstack import db
from refstack.api import constants as const
@ -43,6 +47,8 @@ CTRLS_OPTS = [
CONF = cfg.CONF
CONF.register_opts(CTRLS_OPTS, group='api')
# Cached requests will expire after 10 minutes.
requests_cache.install_cache(cache_name='github_cache', expire_after=600)
class BaseRestControllerWithValidation(rest.RestController):
@ -182,8 +188,62 @@ class ResultsController(BaseRestControllerWithValidation):
return page
class CapabilitiesController(rest.RestController):
"""/v1/capabilities handler. This acts as a proxy for retrieving
capability files from the openstack/defcore Github repository."""
@pecan.expose('json')
def get(self):
"""Get a list of all available capabilities."""
try:
response = requests.get(CONF.api.github_api_capabilities_url)
LOG.debug("Response Status: %s / Used Requests Cache: %s" %
(response.status_code,
getattr(response, 'from_cache', False)))
if response.status_code == 200:
json = response.json()
regex = re.compile('^[0-9]{4}\.[0-9]{2}\.json$')
capability_files = []
for rfile in json:
if rfile["type"] == "file" and regex.search(rfile["name"]):
capability_files.append(rfile["name"])
return capability_files
else:
LOG.warning('Github returned non-success HTTP '
'code: %s' % response.status_code)
pecan.abort(response.status_code)
except requests.exceptions.RequestException as e:
LOG.warning('An error occurred trying to get GitHub '
'repository contents: %s' % e)
pecan.abort(500)
@pecan.expose('json')
def get_one(self, file_name):
"""Handler for getting contents of specific capability file."""
github_url = ''.join((CONF.api.github_raw_base_url.rstrip('/'),
'/', file_name, ".json"))
try:
response = requests.get(github_url)
LOG.debug("Response Status: %s / Used Requests Cache: %s" %
(response.status_code,
getattr(response, 'from_cache', False)))
if response.status_code == 200:
return response.json()
else:
LOG.warning('Github returned non-success HTTP '
'code: %s' % response.status_code)
pecan.abort(response.status_code)
except requests.exceptions.RequestException as e:
LOG.warning('An error occurred trying to get GitHub '
'capability file contents: %s' % e)
pecan.abort(500)
class V1Controller(object):
"""Version 1 API controller root."""
results = ResultsController()
capabilities = CapabilitiesController()

35
refstack/tests/api/test_api.py

@ -18,6 +18,7 @@
import json
import uuid
import httmock
from oslo_config import fixture as config_fixture
import six
import webtest.app
@ -220,3 +221,37 @@ class TestResultsController(api.FunctionalTest):
self.assertEqual(len(filtering_results), 3)
for r in slice_results:
self.assertEqual(r, filtering_results)
class TestCapabilitiesController(api.FunctionalTest):
"""Test case for CapabilitiesController."""
URL = '/v1/capabilities/'
def test_get_capability_list(self):
@httmock.all_requests
def github_api_mock(url, request):
headers = {'content-type': 'application/json'}
content = [{'name': '2015.03.json', 'type': 'file'},
{'name': '2015.next.json', 'type': 'file'},
{'name': '2015.03', 'type': 'dir'}]
content = json.dumps(content)
return httmock.response(200, content, headers, None, 5, request)
with httmock.HTTMock(github_api_mock):
actual_response = self.get_json(self.URL)
expected_response = ['2015.03.json']
self.assertEqual(expected_response, actual_response)
def test_get_capability_file(self):
@httmock.all_requests
def github_mock(url, request):
content = {'foo': 'bar'}
return httmock.response(200, content, None, None, 5, request)
url = self.URL + "2015.03"
with httmock.HTTMock(github_mock):
actual_response = self.get_json(url)
expected_response = {'foo': 'bar'}
self.assertEqual(expected_response, actual_response)

89
refstack/tests/unit/test_api.py

@ -15,9 +15,14 @@
"""Tests for API's controllers"""
import json
import sys
import httmock
import mock
from oslo_config import fixture as config_fixture
from oslotest import base
import requests
from refstack.api import constants as const
from refstack.api import utils as api_utils
@ -25,6 +30,15 @@ from refstack.api.controllers import root
from refstack.api.controllers import v1
def safe_json_dump(content):
if isinstance(content, (dict, list)):
if sys.version_info[0] == 3:
content = bytes(json.dumps(content), 'utf-8')
else:
content = json.dumps(content)
return content
class RootControllerTestCase(base.BaseTestCase):
def test_index(self):
@ -230,6 +244,81 @@ class ResultsControllerTestCase(base.BaseTestCase):
db_get_test.assert_called_once_with(page_number, per_page, filters)
class CapabilitiesControllerTestCase(base.BaseTestCase):
def setUp(self):
super(CapabilitiesControllerTestCase, self).setUp()
self.controller = v1.CapabilitiesController()
def test_get_capabilities(self):
"""Test when getting a list of all capability files."""
@httmock.all_requests
def github_api_mock(url, request):
headers = {'content-type': 'application/json'}
content = [{'name': '2015.03.json', 'type': 'file'},
{'name': '2015.next.json', 'type': 'file'},
{'name': '2015.03', 'type': 'dir'}]
content = safe_json_dump(content)
return httmock.response(200, content, headers, None, 5, request)
with httmock.HTTMock(github_api_mock):
result = self.controller.get()
self.assertEqual(['2015.03.json'], result)
@mock.patch('pecan.abort')
def test_get_capabilities_error_code(self, mock_abort):
"""Test when the HTTP status code isn't a 200 OK. The status should
be propogated."""
@httmock.all_requests
def github_api_mock(url, request):
content = {'title': 'Not Found'}
return httmock.response(404, content, None, None, 5, request)
with httmock.HTTMock(github_api_mock):
self.controller.get()
mock_abort.assert_called_with(404)
@mock.patch('requests.get')
@mock.patch('pecan.abort')
def test_get_capabilities_exception(self, mock_abort, mock_request):
"""Test when the GET request raises an exception."""
mock_request.side_effect = requests.exceptions.RequestException()
self.controller.get()
mock_abort.assert_called_with(500)
def test_get_capability_file(self):
"""Test when getting a specific capability file"""
@httmock.all_requests
def github_mock(url, request):
content = {'foo': 'bar'}
return httmock.response(200, content, None, None, 5, request)
with httmock.HTTMock(github_mock):
result = self.controller.get_one('2015.03')
self.assertEqual({'foo': 'bar'}, result)
@mock.patch('pecan.abort')
def test_get_capability_file_error_code(self, mock_abort):
"""Test when the HTTP status code isn't a 200 OK. The status should
be propogated."""
@httmock.all_requests
def github_api_mock(url, request):
content = {'title': 'Not Found'}
return httmock.response(404, content, None, None, 5, request)
with httmock.HTTMock(github_api_mock):
self.controller.get_one('2010.03')
mock_abort.assert_called_with(404)
@mock.patch('requests.get')
@mock.patch('pecan.abort')
def test_get_capability_file_exception(self, mock_abort, mock_request):
"""Test when the GET request raises an exception."""
mock_request.side_effect = requests.exceptions.RequestException()
self.controller.get_one('2010.03')
mock_abort.assert_called_with(500)
class BaseRestControllerWithValidationTestCase(base.BaseTestCase):
def setUp(self):

3
requirements.txt

@ -8,6 +8,7 @@ oslo.log
pecan>=0.8.2
pyOpenSSL==0.13
pycrypto>=2.6
requests==1.2.3
requests>=2.2.0,!=2.4.0
requests-cache>=0.4.9
jsonschema>=2.0.0,<3.0.0
PyMySQL>=0.6.2,!=0.6.4

1
test-requirements.txt

@ -2,6 +2,7 @@ coverage>=3.6
pep8==1.5.7
pyflakes==0.8.1
flake8==2.2.4
httmock
mock
oslotest>=1.2.0 # Apache-2.0
python-subunit>=0.0.18