Implement guideline test list API endpoint

This allows users to generate and download test lists
for guidelines.

Implements-Spec: https://review.openstack.org/#/c/296829
Change-Id: I7bd9aedeedd45e8df974359a954221e34401a08f

refstack/api/constants.py

@@ -23,6 +23,12 @@ SIGNED = 'signed'
 OPENID = 'openid'
 USER_PUBKEYS = 'pubkeys'
 
+# Guidelines tests requests parameters
+ALIAS = 'alias'
+FLAG = 'flag'
+TYPE = 'type'
+TARGET = 'target'
+
 # OpenID parameters
 OPENID_MODE = 'openid.mode'
 OPENID_NS = 'openid.ns'

refstack/api/controllers/guidelines.py

@@ -19,11 +19,56 @@ from oslo_log import log
 import pecan
 from pecan import rest
 
+from refstack.api import constants as const
 from refstack.api import guidelines
+from refstack.api import utils as api_utils
 
 LOG = log.getLogger(__name__)
 
 
+class TestsController(rest.RestController):
+    """v1/guidelines/<version>/tests handler.
+
+    This will allow users to retrieve specific test lists from specific
+    guidelines for use with refstack-client.
+    """
+
+    @pecan.expose(content_type='text/plain')
+    def get(self, version):
+        """Get the plain-text test list of the specified guideline version."""
+        # Remove the .json from version if it is there.
+        version.replace('.json', '')
+        g = guidelines.Guidelines()
+        json = g.get_guideline_contents(version)
+
+        if not json:
+            return 'Error getting JSON content for version: ' + version
+
+        if pecan.request.GET.get(const.TYPE):
+            types = pecan.request.GET.get(const.TYPE).split(',')
+        else:
+            types = None
+
+        if pecan.request.GET.get('alias'):
+            alias = api_utils.str_to_bool(pecan.request.GET.get('alias'))
+        else:
+            alias = True
+
+        if pecan.request.GET.get('flag'):
+            flag = api_utils.str_to_bool(pecan.request.GET.get('flag'))
+        else:
+            flag = True
+
+        target = pecan.request.GET.get('target', 'platform')
+        try:
+            target_caps = g.get_target_capabilities(json, types, target)
+            test_list = g.get_test_list(json, target_caps, alias, flag)
+        except KeyError:
+            return 'Invalid target: ' + target
+
+        return '\n'.join(test_list)
+
+
 class GuidelinesController(rest.RestController):
     """/v1/guidelines handler.
 
@@ -31,6 +76,8 @@ class GuidelinesController(rest.RestController):
     from the openstack/defcore Github repository.
     """
 
+    tests = TestsController()
+
     @pecan.expose('json')
     def get(self):
         """Get a list of all available guidelines."""

refstack/api/guidelines.py

@@ -101,3 +101,76 @@ class Guidelines:
             LOG.warning('An error occurred trying to get raw capability file '
                         'contents from %s: %s' % (self.raw_url, e))
             return None
+
+    def get_target_capabilities(self, guideline_json, types=None,
+                                target='platform'):
+        """Get list of capabilities that match the given statuses and target.
+
+        If no list of types in given, then capabilities of all types
+        are given. If not target is specified, then all capabilities are given.
+        """
+        components = guideline_json['components']
+        targets = set()
+        if target != 'platform':
+            targets.add(target)
+        else:
+            targets.update(guideline_json['platform']['required'])
+
+        target_caps = set()
+        for component in targets:
+            for status, capabilities in components[component].items():
+                if types is None or status in types:
+                    target_caps.update(capabilities)
+
+        return list(target_caps)
+
+    def get_test_list(self, guideline_json, capabilities=[],
+                      alias=True, show_flagged=True):
+        """Generate a test list based on input.
+
+        A test list is formed from the given guideline JSON data and
+        list of capabilities. If 'alias' is True, test aliases are
+        included in the list. If 'show_flagged' is True, flagged tests are
+        included in the list.
+        """
+        caps = guideline_json['capabilities']
+        schema = guideline_json['schema']
+        test_list = []
+        for cap, cap_details in caps.items():
+            if cap in capabilities:
+                if schema == '1.2':
+                    for test in cap_details['tests']:
+                        if show_flagged:
+                            test_list.append(test)
+                        elif not show_flagged and \
+                            test not in cap_details['flagged']:
+                            test_list.append(test)
+                else:
+                    for test, test_details in cap_details['tests'].items():
+                        added = False
+                        if test_details.get('flagged'):
+                            if show_flagged:
+                                test_str = '{}[{}]'.format(
+                                    test,
+                                    test_details.get('idempotent_id', '')
+                                )
+                                test_list.append(test_str)
+                                added = True
+                        else:
+                            # Make sure the test UUID is in the test string.
+                            test_str = '{}[{}]'.format(
+                                test,
+                                test_details.get('idempotent_id', '')
+                            )
+                            test_list.append(test_str)
+                            added = True
+
+                        if alias and test_details.get('aliases') and added:
+                            for alias in test_details['aliases']:
+                                test_str = '{}[{}]'.format(
+                                    alias,
+                                    test_details.get('idempotent_id', '')
+                                )
+                                test_list.append(test_str)
+        test_list.sort()
+        return test_list

refstack/api/utils.py

@@ -90,6 +90,11 @@ def parse_input_params(expected_input_params):
     return filters
 
 
+def str_to_bool(param):
+    """Check if a string value should be evaluated as True or False."""
+    return param.lower() in ("true", "yes", "1")
+
+
 def _calculate_pages_number(per_page, records_count):
     """Return pages number.
 

refstack/tests/api/test_guidelines.py

@@ -51,3 +51,51 @@ class TestGuidelinesEndpoint(api.FunctionalTest):
 
         expected_response = {'foo': 'bar'}
         self.assertEqual(expected_response, actual_response)
+
+    def test_get_guideline_test_list(self):
+        @httmock.all_requests
+        def github_mock(url, request):
+            content = {
+                'schema': '1.4',
+                'platform': {'required': ['compute', 'object']},
+                'components': {
+                    'compute': {
+                        'required': ['cap-1'],
+                        'advisory': [],
+                        'deprecated': [],
+                        'removed': []
+                    },
+                    'object': {
+                        'required': ['cap-2'],
+                        'advisory': ['cap-3'],
+                        'deprecated': [],
+                        'removed': []
+                    }
+                },
+                'capabilities': {
+                    'cap-1': {
+                        'tests': {
+                            'test_1': {'idempotent_id': 'id-1234'},
+                            'test_2': {'idempotent_id': 'id-5678',
+                                       'aliases': ['test_2_1']},
+                            'test_3': {'idempotent_id': 'id-1111',
+                                       'flagged': {'reason': 'foo'}}
+                        }
+                    },
+                    'cap-2': {
+                        'tests': {
+                            'test_4': {'idempotent_id': 'id-1233'}
+                        }
+                    }
+                }
+            }
+            return httmock.response(200, content, None, None, 5, request)
+        url = self.URL + "2016.03/tests"
+        with httmock.HTTMock(github_mock):
+            actual_response = self.get_json(url, expect_errors=True)
+
+        expected_list = ['test_1[id-1234]', 'test_2[id-5678]',
+                         'test_2_1[id-5678]', 'test_3[id-1111]',
+                         'test_4[id-1233]']
+        expected_response = '\n'.join(expected_list)
+        self.assertEqual(expected_response, actual_response.text)

refstack/tests/unit/test_api.py

@@ -315,6 +315,78 @@ class GuidelinesControllerTestCase(BaseControllerTestCase):
         self.mock_abort.assert_called_with(500, mock.ANY)
 
 
+class GuidelinesTestsControllerTestCase(BaseControllerTestCase):
+
+    FAKE_GUIDELINES = {
+        'schema': '1.4',
+        'platform': {'required': ['compute', 'object']},
+        'components': {
+            'compute': {
+                'required': ['cap-1'],
+                'advisory': [],
+                'deprecated': [],
+                'removed': []
+            },
+            'object': {
+                'required': ['cap-2'],
+                'advisory': [],
+                'deprecated': [],
+                'removed': []
+            }
+        },
+        'capabilities': {
+            'cap-1': {
+                'tests': {
+                    'test_1': {'idempotent_id': 'id-1234'},
+                    'test_2': {'idempotent_id': 'id-5678',
+                               'aliases': ['test_2_1']},
+                    'test_3': {'idempotent_id': 'id-1111',
+                               'flagged': {'reason': 'foo'}}
+                }
+            },
+            'cap-2': {
+                'tests': {
+                    'test_4': {'idempotent_id': 'id-1233'}
+                }
+            }
+        }
+    }
+
+    def setUp(self):
+        super(GuidelinesTestsControllerTestCase, self).setUp()
+        self.controller = guidelines.TestsController()
+
+    @mock.patch('refstack.api.guidelines.Guidelines.get_guideline_contents')
+    @mock.patch('pecan.request')
+    def test_get_guideline_tests(self, mock_request, mock_get_contents):
+        """Test getting the test list string of a guideline."""
+        mock_get_contents.return_value = self.FAKE_GUIDELINES
+        mock_request.GET = {}
+        test_list_str = self.controller.get('2016,01')
+        expected_list = ['test_1[id-1234]', 'test_2[id-5678]',
+                         'test_2_1[id-5678]', 'test_3[id-1111]',
+                         'test_4[id-1233]']
+        expected_result = '\n'.join(expected_list)
+        self.assertEqual(expected_result, test_list_str)
+
+    @mock.patch('refstack.api.guidelines.Guidelines.get_guideline_contents')
+    def test_get_guideline_tests_fail(self, mock_get_contents):
+        """Test when the JSON content of a guideline can't be retrieved."""
+        mock_get_contents.return_value = None
+        result_str = self.controller.get('2016.02')
+        self.assertIn('Error getting JSON', result_str)
+
+    @mock.patch('refstack.api.guidelines.Guidelines.get_guideline_contents')
+    @mock.patch('pecan.request')
+    def test_get_guideline_tests_invalid_target(self, mock_request,
+                                                mock_get_contents):
+        """Test when the target is invalid."""
+        mock_get_contents.return_value = self.FAKE_GUIDELINES
+        mock_request.GET = {'target': 'foo'}
+        result_str = self.controller.get('2016.02')
+        self.assertIn('Invalid target', result_str)
+
+
 class BaseRestControllerWithValidationTestCase(BaseControllerTestCase):
 
     def setUp(self):

refstack/tests/unit/test_api_utils.py

@@ -183,6 +183,13 @@ class APIUtilsTestCase(base.BaseTestCase):
 
         mock_get_input.assert_called_once_with(expected_params)
 
+    def test_str_to_bool(self):
+        self.assertTrue(api_utils.str_to_bool('True'))
+        self.assertTrue(api_utils.str_to_bool('1'))
+        self.assertTrue(api_utils.str_to_bool('YES'))
+        self.assertFalse(api_utils.str_to_bool('False'))
+        self.assertFalse(api_utils.str_to_bool('no'))
+
     def test_calculate_pages_number_full_pages(self):
         # expected pages number: 20/10 = 2
         page_number = api_utils._calculate_pages_number(10, 20)

refstack/tests/unit/test_guidelines.py

@@ -88,3 +88,83 @@ class GuidelinesTestCase(base.BaseTestCase):
         mock_requests_get.side_effect = requests.exceptions.RequestException()
         result = self.guidelines.get_guideline_contents('2010.03.json')
         self.assertIsNone(result)
+
+    def test_get_target_capabilities(self):
+        """Test getting relevant capabilities."""
+        json = {
+            'platform': {'required': ['compute', 'object']},
+            'schema': '1.4',
+            'components': {
+                'compute': {
+                    'required': ['cap_id_1'],
+                    'advisory': [],
+                    'deprecated': [],
+                    'removed': []
+                },
+                'object': {
+                    'required': ['cap_id_2'],
+                    'advisory': ['cap_id_3'],
+                    'deprecated': [],
+                    'removed': []
+                }
+            }
+        }
+
+        # Test platform capabilities
+        caps = self.guidelines.get_target_capabilities(json)
+        expected = sorted(['cap_id_1', 'cap_id_2', 'cap_id_3'])
+        self.assertEqual(expected, sorted(caps))
+
+        caps = self.guidelines.get_target_capabilities(json,
+                                                       types=['required'],
+                                                       target='object')
+        expected = ['cap_id_2']
+        self.assertEqual(expected, caps)
+
+    def test_get_test_list(self):
+        """Test when getting the guideline test list."""
+
+        # Schema version 1.4
+        json = {
+            'schema': '1.4',
+            'capabilities': {
+                'cap-1': {
+                    'tests': {
+                        'test_1': {'idempotent_id': 'id-1234'},
+                        'test_2': {'idempotent_id': 'id-5678',
+                                   'aliases': ['test_2_1']},
+                        'test_3': {'idempotent_id': 'id-1111',
+                                   'flagged': {'reason': 'foo'}}
+                    }
+                },
+                'cap-2': {
+                    'tests': {
+                        'test_4': {'idempotent_id': 'id-1233'}
+                    }
+                }
+            }
+        }
+        tests = self.guidelines.get_test_list(json, ['cap-1'])
+        expected = ['test_1[id-1234]', 'test_2[id-5678]',
+                    'test_2_1[id-5678]', 'test_3[id-1111]']
+        self.assertEqual(expected, tests)
+
+        tests = self.guidelines.get_test_list(json, ['cap-1'],
+                                              alias=False, show_flagged=False)
+        expected = ['test_1[id-1234]', 'test_2[id-5678]']
+        self.assertEqual(expected, tests)
+
+        # Schema version 1.2
+        json = {
+            'schema': '1.2',
+            'capabilities': {
+                'cap-1': {
+                    'tests': ['test_1', 'test_2']
+                },
+                'cap-2': {
+                    'tests': ['test_3']
+                }
+            }
+        }
+        tests = self.guidelines.get_test_list(json, ['cap-2'])
+        self.assertEqual(['test_3'], tests)