Add API endpoint for retrieving capabilities

This endpoint will serve as a proxy for the openstack/defcore
repository. Instead of the UI sending requests to GitHub directly
where requests might be impeded due to cross-origin blockage or API
rate-limiting, the UI will be able to send requests to the Refstack
API in order to get capability data. The requests-cache library is
used so as not to keep sending requests to GitHub when likely
nothing has changed. With this, Refstack no longer has to worry about
mirroring capability files and keeping them updated.

GET /v1/capabilities - list all capability files
GET /v1/capabilities/2015.03 - get contents of file 2015.03.json

Change-Id: I6d327273f191e3339219e35eed8768da35c5bffb

.gitignore

@@ -9,3 +9,4 @@ ChangeLog
 build/
 cover/
 dist
+*.sqlite

etc/refstack.conf.sample

@@ -135,6 +135,15 @@
 # The format for start_date and end_date parameters (string value)
 #input_date_format = %Y-%m-%d %H:%M:%S
 
+# The GitHub API URL of the repository and location of the DefCore
+# capability files. This URL is used to get a listing of all capability
+# files.
+#github_api_capabilities_url = https://api.github.com/repos/openstack/defcore/contents
+
+# The base URL that is used for retrieving specific capability files.
+# Capability file names will be appended to this URL to get the contents
+# of that file.
+#github_raw_base_url = https://raw.githubusercontent.com/openstack/defcore/master/
 
 [database]
 

refstack/api/app.py

@@ -68,6 +68,20 @@ API_OPTS = [
                default='http://refstack.net/output.html?test_id=%s',
                help='Template for test result url.'
                ),
+    cfg.StrOpt('github_api_capabilities_url',
+               default='https://api.github.com'
+                       '/repos/openstack/defcore/contents',
+               help='The GitHub API URL of the repository and location of '
+                    'the DefCore capability files. This URL is used to get '
+                    'a listing of all capability files.'
+               ),
+    cfg.StrOpt('github_raw_base_url',
+               default='https://raw.githubusercontent.com'
+                       '/openstack/defcore/master/',
+               help='This is the base URL that is used for retrieving '
+                    'specific capability files. Capability file names will '
+                    'be appended to this URL to get the contents of that file.'
+               )
 ]
 
 CONF = cfg.CONF

refstack/api/controllers/v1.py

@@ -16,10 +16,14 @@
 """Version 1 of the API."""
 
 import json
+
 from oslo_config import cfg
 from oslo_log import log
 import pecan
 from pecan import rest
+import re
+import requests
+import requests_cache
 
 from refstack import db
 from refstack.api import constants as const
@@ -43,6 +47,8 @@ CTRLS_OPTS = [
 CONF = cfg.CONF
 
 CONF.register_opts(CTRLS_OPTS, group='api')
+# Cached requests will expire after 10 minutes.
+requests_cache.install_cache(cache_name='github_cache', expire_after=600)
 
 
 class BaseRestControllerWithValidation(rest.RestController):
@@ -182,8 +188,62 @@ class ResultsController(BaseRestControllerWithValidation):
         return page
 
 
+class CapabilitiesController(rest.RestController):
+
+    """/v1/capabilities handler. This acts as a proxy for retrieving
+       capability files from the openstack/defcore Github repository."""
+
+    @pecan.expose('json')
+    def get(self):
+        """Get a list of all available capabilities."""
+        try:
+            response = requests.get(CONF.api.github_api_capabilities_url)
+            LOG.debug("Response Status: %s / Used Requests Cache: %s" %
+                      (response.status_code,
+                       getattr(response, 'from_cache', False)))
+            if response.status_code == 200:
+                json = response.json()
+                regex = re.compile('^[0-9]{4}\.[0-9]{2}\.json$')
+                capability_files = []
+                for rfile in json:
+                    if rfile["type"] == "file" and regex.search(rfile["name"]):
+                        capability_files.append(rfile["name"])
+                return capability_files
+            else:
+                LOG.warning('Github returned non-success HTTP '
+                            'code: %s' % response.status_code)
+                pecan.abort(response.status_code)
+
+        except requests.exceptions.RequestException as e:
+            LOG.warning('An error occurred trying to get GitHub '
+                        'repository contents: %s' % e)
+            pecan.abort(500)
+
+    @pecan.expose('json')
+    def get_one(self, file_name):
+        """Handler for getting contents of specific capability file."""
+        github_url = ''.join((CONF.api.github_raw_base_url.rstrip('/'),
+                              '/', file_name, ".json"))
+        try:
+            response = requests.get(github_url)
+            LOG.debug("Response Status: %s / Used Requests Cache: %s" %
+                      (response.status_code,
+                       getattr(response, 'from_cache', False)))
+            if response.status_code == 200:
+                return response.json()
+            else:
+                LOG.warning('Github returned non-success HTTP '
+                            'code: %s' % response.status_code)
+                pecan.abort(response.status_code)
+        except requests.exceptions.RequestException as e:
+            LOG.warning('An error occurred trying to get GitHub '
+                        'capability file contents: %s' % e)
+            pecan.abort(500)
+
+
 class V1Controller(object):
 
     """Version 1 API controller root."""
 
     results = ResultsController()
+    capabilities = CapabilitiesController()

refstack/tests/api/test_api.py

@@ -18,6 +18,7 @@
 import json
 import uuid
 
+import httmock
 from oslo_config import fixture as config_fixture
 import six
 import webtest.app
@@ -220,3 +221,37 @@ class TestResultsController(api.FunctionalTest):
             self.assertEqual(len(filtering_results), 3)
             for r in slice_results:
                 self.assertEqual(r, filtering_results)
+
+
+class TestCapabilitiesController(api.FunctionalTest):
+    """Test case for CapabilitiesController."""
+
+    URL = '/v1/capabilities/'
+
+    def test_get_capability_list(self):
+        @httmock.all_requests
+        def github_api_mock(url, request):
+            headers = {'content-type': 'application/json'}
+            content = [{'name': '2015.03.json', 'type': 'file'},
+                       {'name': '2015.next.json', 'type': 'file'},
+                       {'name': '2015.03', 'type': 'dir'}]
+            content = json.dumps(content)
+            return httmock.response(200, content, headers, None, 5, request)
+
+        with httmock.HTTMock(github_api_mock):
+            actual_response = self.get_json(self.URL)
+
+        expected_response = ['2015.03.json']
+        self.assertEqual(expected_response, actual_response)
+
+    def test_get_capability_file(self):
+        @httmock.all_requests
+        def github_mock(url, request):
+            content = {'foo': 'bar'}
+            return httmock.response(200, content, None, None, 5, request)
+        url = self.URL + "2015.03"
+        with httmock.HTTMock(github_mock):
+            actual_response = self.get_json(url)
+
+        expected_response = {'foo': 'bar'}
+        self.assertEqual(expected_response, actual_response)

refstack/tests/unit/test_api.py

@@ -15,9 +15,14 @@
 
 """Tests for API's controllers"""
 
+import json
+import sys
+
+import httmock
 import mock
 from oslo_config import fixture as config_fixture
 from oslotest import base
+import requests
 
 from refstack.api import constants as const
 from refstack.api import utils as api_utils
@@ -25,6 +30,15 @@ from refstack.api.controllers import root
 from refstack.api.controllers import v1
 
 
+def safe_json_dump(content):
+    if isinstance(content, (dict, list)):
+        if sys.version_info[0] == 3:
+            content = bytes(json.dumps(content), 'utf-8')
+        else:
+            content = json.dumps(content)
+    return content
+
+
 class RootControllerTestCase(base.BaseTestCase):
 
     def test_index(self):
@@ -230,6 +244,81 @@ class ResultsControllerTestCase(base.BaseTestCase):
         db_get_test.assert_called_once_with(page_number, per_page, filters)
 
 
+class CapabilitiesControllerTestCase(base.BaseTestCase):
+
+    def setUp(self):
+        super(CapabilitiesControllerTestCase, self).setUp()
+        self.controller = v1.CapabilitiesController()
+
+    def test_get_capabilities(self):
+        """Test when getting a list of all capability files."""
+        @httmock.all_requests
+        def github_api_mock(url, request):
+            headers = {'content-type': 'application/json'}
+            content = [{'name': '2015.03.json', 'type': 'file'},
+                       {'name': '2015.next.json', 'type': 'file'},
+                       {'name': '2015.03', 'type': 'dir'}]
+            content = safe_json_dump(content)
+            return httmock.response(200, content, headers, None, 5, request)
+
+        with httmock.HTTMock(github_api_mock):
+            result = self.controller.get()
+        self.assertEqual(['2015.03.json'], result)
+
+    @mock.patch('pecan.abort')
+    def test_get_capabilities_error_code(self, mock_abort):
+        """Test when the HTTP status code isn't a 200 OK. The status should
+           be propogated."""
+        @httmock.all_requests
+        def github_api_mock(url, request):
+            content = {'title': 'Not Found'}
+            return httmock.response(404, content, None, None, 5, request)
+
+        with httmock.HTTMock(github_api_mock):
+            self.controller.get()
+        mock_abort.assert_called_with(404)
+
+    @mock.patch('requests.get')
+    @mock.patch('pecan.abort')
+    def test_get_capabilities_exception(self, mock_abort, mock_request):
+        """Test when the GET request raises an exception."""
+        mock_request.side_effect = requests.exceptions.RequestException()
+        self.controller.get()
+        mock_abort.assert_called_with(500)
+
+    def test_get_capability_file(self):
+        """Test when getting a specific capability file"""
+        @httmock.all_requests
+        def github_mock(url, request):
+            content = {'foo': 'bar'}
+            return httmock.response(200, content, None, None, 5, request)
+
+        with httmock.HTTMock(github_mock):
+            result = self.controller.get_one('2015.03')
+        self.assertEqual({'foo': 'bar'}, result)
+
+    @mock.patch('pecan.abort')
+    def test_get_capability_file_error_code(self, mock_abort):
+        """Test when the HTTP status code isn't a 200 OK. The status should
+           be propogated."""
+        @httmock.all_requests
+        def github_api_mock(url, request):
+            content = {'title': 'Not Found'}
+            return httmock.response(404, content, None, None, 5, request)
+
+        with httmock.HTTMock(github_api_mock):
+            self.controller.get_one('2010.03')
+        mock_abort.assert_called_with(404)
+
+    @mock.patch('requests.get')
+    @mock.patch('pecan.abort')
+    def test_get_capability_file_exception(self, mock_abort, mock_request):
+        """Test when the GET request raises an exception."""
+        mock_request.side_effect = requests.exceptions.RequestException()
+        self.controller.get_one('2010.03')
+        mock_abort.assert_called_with(500)
+
+
 class BaseRestControllerWithValidationTestCase(base.BaseTestCase):
 
     def setUp(self):

requirements.txt

@@ -8,6 +8,7 @@ oslo.log
 pecan>=0.8.2
 pyOpenSSL==0.13
 pycrypto>=2.6
-requests==1.2.3
+requests>=2.2.0,!=2.4.0
+requests-cache>=0.4.9
 jsonschema>=2.0.0,<3.0.0
 PyMySQL>=0.6.2,!=0.6.4

test-requirements.txt

@@ -2,6 +2,7 @@ coverage>=3.6
 pep8==1.5.7
 pyflakes==0.8.1
 flake8==2.2.4
+httmock
 mock
 oslotest>=1.2.0 # Apache-2.0
 python-subunit>=0.0.18