Get rid of Mistral for listing validations

Until now, we were using Mistral to get all the validations available on
the Undercloud. This patch drops the Mistral support for the validations
listing through the CLI and will parse them directly on the filesystem
instead of parsing from the *tripleo-validations* Swift container.

This change makes the CLI faster:
- ~6s with Mistral
- ~1.3s w/o Mistral

Change-Id: I5730fb2e23775f1d666276b6b4d46bcb550fc928
Signed-off-by: Gael Chamoulaud <gchamoul@redhat.com>

releasenotes/notes/get_rid_of_mistral_for_listing_validations-1e9dedf33675f1d6.yaml

@@ -0,0 +1,6 @@
+---
+deprecations:
+  - |
+    The TripleO Validator was using Mistral to get all the Validations available
+    on the Undercloud. From now, The CLI is parsing the Validations directly
+    from the filesystem and the Mistral support has been removed.

tripleoclient/tests/v1/tripleo/test_tripleo_validator.py

@@ -19,6 +19,20 @@ import sys
 from osc_lib.tests import utils
 from tripleoclient.v1 import tripleo_validator
 
+VALIDATIONS_LIST = [{
+    'description': 'My Validation One Description',
+    'groups': ['prep', 'pre-deployment'],
+    'id': 'my_val1',
+    'name': 'My Validition One Name',
+    'parameters': {}
+}, {
+    'description': 'My Validation Two Description',
+    'groups': ['prep', 'pre-introspection'],
+    'id': 'my_val2',
+    'name': 'My Validition Two Name',
+    'parameters': {}
+}]
+
 
 class TestValidatorList(utils.TestCommand):
 
@@ -27,20 +41,16 @@ class TestValidatorList(utils.TestCommand):
 
         # Get the command object to test
         self.cmd = tripleo_validator.TripleOValidatorList(self.app, None)
-        self.app.client_manager.workflow_engine = mock.Mock()
-        self.workflow = self.app.client_manager.workflow_engine
 
-    @mock.patch('tripleoclient.workflows.validations.list_validations',
-                autospec=True)
-    def test_validation_list_noargs(self, plan_mock):
+    @mock.patch('tripleoclient.utils.parse_all_validations_on_disk',
+                return_value=VALIDATIONS_LIST)
+    def test_validation_list_noargs(self, mock_validations):
         arglist = []
         verifylist = []
 
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
 
         self.cmd.take_action(parsed_args)
-        plan_mock.assert_called_once_with(
-            mock.ANY, {'group_names': []})
 
 
 class TestValidatorRun(utils.TestCommand):

tripleoclient/tests/workflows/test_validations.py

@@ -19,33 +19,6 @@ from osc_lib.tests import utils
 from tripleoclient.workflows import validations
 
 
-class TestValidationsList(utils.TestCommand):
-
-    def setUp(self):
-        super(TestValidationsList, self).setUp()
-        self.app.client_manager = mock.Mock()
-        self.app.client_manager.workflow_engine = self.workflow = mock.Mock()
-        self.tripleoclient = mock.Mock()
-        self.websocket = mock.Mock()
-        self.websocket.__enter__ = lambda s: self.websocket
-        self.websocket.__exit__ = lambda s, *exc: None
-        self.tripleoclient.messaging_websocket.return_value = self.websocket
-        self.app.client_manager.tripleoclient = self.tripleoclient
-
-    @mock.patch('tripleoclient.workflows.base.wait_for_messages')
-    @mock.patch('tripleoclient.workflows.base.start_workflow')
-    def test_list_validations(self, start_wf_mock, messages_mock):
-        messages_mock.return_value = []
-        fetch_name = 'tripleo.validations.v1.list'
-        fetch_input = {
-            'group_names': ['pre-deployment']
-        }
-        validations.list_validations(self.app.client_manager, fetch_input)
-        start_wf_mock.assert_called_once_with(self.workflow,
-                                              fetch_name,
-                                              workflow_input=fetch_input)
-
-
 class TestValidationsRun(utils.TestCommand):
 
     def setUp(self):

tripleoclient/utils.py

@@ -1762,6 +1762,60 @@ def _get_from_cfg(cfg, accessor, param, section):
     return val
 
 
+def get_validation_metadata(validation, key):
+    default_metadata = {
+        'name': 'Unnamed',
+        'description': 'No description',
+        'stage': 'No stage',
+        'groups': [],
+    }
+
+    try:
+        return validation[0]['vars']['metadata'].get(key,
+                                                     default_metadata[key])
+    except KeyError:
+        LOG.exception(_("Key '{key}' not even found in "
+                        "default metadata").format(key=key))
+    except TypeError:
+        LOG.exception(_("Failed to get validation metadata."))
+
+
+def get_validation_parameters(validation):
+    try:
+        return {
+            k: v
+            for k, v in validation[0]['vars'].items()
+            if k != 'metadata'
+        }
+    except KeyError:
+        LOG.debug(_("No parameters found for this validation"))
+        return dict()
+
+
+def parse_all_validations_on_disk(path, groups=None):
+    results = []
+    validations_abspath = glob.glob("{path}/*.yaml".format(path=path))
+
+    for pl in validations_abspath:
+        validation_id, ext = os.path.splitext(os.path.basename(pl))
+
+        with open(pl, 'r') as val_playbook:
+            contents = yaml.safe_load(val_playbook)
+
+        validation_groups = get_validation_metadata(contents, 'groups') or []
+        if not groups or set.intersection(set(groups), set(validation_groups)):
+            results.append({
+                'id': validation_id,
+                'name': get_validation_metadata(contents, 'name'),
+                'groups': get_validation_metadata(contents, 'groups'),
+                'description': get_validation_metadata(contents,
+                                                       'description'),
+                'parameters': get_validation_parameters(contents)
+            })
+
+    return results
+
+
 def get_param_field_name(validations_data=None):
     """Get the current parameters field name in a Dict
 

tripleoclient/v1/tripleo_validator.py

@@ -149,18 +149,14 @@ class TripleOValidatorList(command.Command):
                 print('Output: {}'.format(e))
 
     def _run_validator_list(self, parsed_args):
-        clients = self.app.client_manager
-
-        workflow_input = {
-            "group_names": parsed_args.group
-        }
-
         LOG.debug(_('Launch listing the validations'))
         try:
-            output = validations.list_validations(clients, workflow_input)
+            validations = oooutils.parse_all_validations_on_disk(
+                constants.ANSIBLE_VALIDATION_DIR, parsed_args.group)
+
             if parsed_args.parameters:
                 out = oooutils.get_validations_parameters(
-                    {'validations': output},
+                    {'validations': validations},
                     parsed_args.validation_name,
                     parsed_args.group
                 )
@@ -173,13 +169,13 @@ class TripleOValidatorList(command.Command):
             else:
                 if parsed_args.output == 'json':
                     out = oooutils.get_validations_json(
-                        {'validations': output})
+                        {'validations': validations})
                 elif parsed_args.output == 'yaml':
                     out = oooutils.get_validations_yaml(
-                        {'validations': output})
+                        {'validations': validations})
                 else:
                     out = oooutils.get_validations_table(
-                        {'validations': output})
+                        {'validations': validations})
                 print(out)
         except Exception as e:
             raise RuntimeError(_("Validations listing finished with errors\n"

tripleoclient/workflows/validations.py

@@ -12,29 +12,9 @@
 #   License for the specific language governing permissions and limitations
 #   under the License.
 #
-import pprint
-
 from tripleoclient.workflows import base
 
 
-def list_validations(clients, workflow_input):
-
-    workflow_client = clients.workflow_engine
-    tripleoclients = clients.tripleoclient
-
-    with tripleoclients.messaging_websocket() as ws:
-        execution = base.start_workflow(
-            workflow_client,
-            'tripleo.validations.v1.list',
-            workflow_input=workflow_input
-        )
-
-        for payload in base.wait_for_messages(workflow_client, ws, execution):
-            if 'message' in payload:
-                assert payload['status'] == "SUCCESS", pprint.pformat(payload)
-                return payload['validations']
-
-
 def run_validations(clients, workflow_input):
 
     workflow_client = clients.workflow_engine