Add Categories metadata key management

This patch adds the management of the new `categories` metadata key in
the validation playbooks. We can now filter the validations by their
groups and/or by their categories while listing or running them.

The `list` sub command has now a new --category argument. When filtering
by groups and by categories (see the example below), the `list` sub
command will return all the validation playbooks belonging to the prep
group OR all the validation playbooks belonging to the os and/or system
categories:

$ validation list -h
$ validation list --group prep --category os,system

The `run` sub command has also its new --category argument. Note that
this new argument is mutually exclusive with the --validation and
--group arguments:

$ validation run -h
$ validation run --category networking --inventory /etc/ansible/hosts

The `show parameter` sub command has the same new argument which is also
mutually exclusive with the --validation and --group arguments:

$ validation show parameter -h
$ validation show parameter --category os,system,hardware,ram

Change-Id: I4297f83355bdd209d21518fbadb17d1343fd4680
Signed-off-by: Gael Chamoulaud (Strider) <gchamoul@redhat.com>

validations_libs/cli/lister.py

@@ -36,6 +36,13 @@ class ValidationList(Lister):
                                   "separate the group names with commas: "
                                   "--group pre-upgrade,prep | "
                                   "--group openshift-on-openstack"))
+        parser.add_argument('--category',
+                            metavar='<category_id>[,<category_id>,...]',
+                            action=CommaListAction,
+                            default=[],
+                            help=("List specific category of validations, "
+                                  "if more than one category is required "
+                                  "separate the category names with commas."))
         parser.add_argument('--validation-dir', dest='validation_dir',
                             default=constants.ANSIBLE_VALIDATION_DIR,
                             help=("Path where the validation playbooks "
@@ -46,7 +53,9 @@ class ValidationList(Lister):
         """Take validation action"""
 
         group = parsed_args.group
+        category = parsed_args.category
         validation_dir = parsed_args.validation_dir
 
         v_actions = ValidationActions(validation_path=validation_dir)
-        return (v_actions.list_validations(group))
+        return (v_actions.list_validations(groups=group,
+                                           categories=category))

validations_libs/cli/run.py

@@ -139,6 +139,15 @@ class Run(BaseCommand):
                   "--group pre-upgrade,prep | "
                   "--group openshift-on-openstack"))
 
+        ex_group.add_argument(
+            '--category',
+            metavar='<category_id>[,<category_id>,...]',
+            action=CommaListAction,
+            default=[],
+            help=("Run specific validations by category, "
+                  "if more than one category is required "
+                  "separate the category names with commas."))
+
         return parser
 
     def take_action(self, parsed_args):
@@ -168,6 +177,7 @@ class Run(BaseCommand):
                 inventory=parsed_args.inventory,
                 limit_hosts=parsed_args.limit,
                 group=parsed_args.group,
+                category=parsed_args.category,
                 extra_vars=extra_vars,
                 validations_dir=parsed_args.validation_dir,
                 base_dir=parsed_args.ansible_base_dir,

validations_libs/cli/show.py

@@ -109,6 +109,15 @@ class ShowParameter(ShowOne):
                   "openshift-on-openstack")
         )
 
+        ex_group.add_argument(
+            '--category',
+            metavar='<category_id>[,<category_id>,...]',
+            action=CommaListAction,
+            default=[],
+            help=("List specific validations by category, "
+                  "if more than one category is required "
+                  "separate the category names with commas."))
+
         parser.add_argument(
             '--download',
             action='store',
@@ -134,10 +143,11 @@ class ShowParameter(ShowOne):
     def take_action(self, parsed_args):
         v_actions = ValidationActions(parsed_args.validation_dir)
         params = v_actions.show_validations_parameters(
-            parsed_args.validation_name,
+            validations=parsed_args.validation_name,
-            parsed_args.group,
+            groups=parsed_args.group,
-            parsed_args.format_output,
+            categories=parsed_args.category,
-            parsed_args.download)
+            output_format=parsed_args.format_output,
+            download_file=parsed_args.download)
 
         if parsed_args.download:
             self.app.LOG.info(

validations_libs/tests/cli/test_list.py

@@ -36,22 +36,25 @@ class TestList(BaseCommand):
         arglist = ['--validation-dir', 'foo']
         verifylist = [('validation_dir', 'foo')]
 
-        list = [{'description': 'My Validation One Description',
+        val_list = [
-                 'groups': ['prep', 'pre-deployment'],
+            {'description': 'My Validation One Description',
-                 'id': 'my_val1',
+             'groups': ['prep', 'pre-deployment'],
-                 'name': 'My Validation One Name',
+             'categories': ['os', 'system', 'ram'],
-                 'parameters': {}
+             'id': 'my_val1',
-                }, {
+             'name': 'My Validation One Name',
-                 'description': 'My Validation Two Description',
+             'parameters': {}
-                 'groups': ['prep', 'pre-introspection'],
+            }, {
-                 'id': 'my_val2',
+             'description': 'My Validation Two Description',
-                 'name': 'My Validation Two Name',
+             'groups': ['prep', 'pre-introspection'],
-                 'parameters': {'min_value': 8}
+             'categories': ['networking'],
-                }]
+             'id': 'my_val2',
+             'name': 'My Validation Two Name',
+             'parameters': {'min_value': 8}
+            }]
 
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
         result = self.cmd.take_action(parsed_args)
-        self.assertEqual(result, list)
+        self.assertEqual(result, val_list)
 
     @mock.patch('validations_libs.validation_actions.ValidationActions.'
                 'list_validations',
@@ -71,8 +74,21 @@ class TestList(BaseCommand):
         verifylist = [('validation_dir', 'foo'),
                       ('group', ['prep'])]
 
-        list = fakes.VALIDATION_LIST_RESULT
+        val_list = fakes.VALIDATION_LIST_RESULT
 
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
         result = self.cmd.take_action(parsed_args)
-        self.assertEqual(result, list)
+        self.assertEqual(result, val_list)
+
+    @mock.patch('validations_libs.utils.parse_all_validations_on_disk',
+                return_value=fakes.VALIDATIONS_LIST_GROUP)
+    def test_list_validations_by_category(self, mock_list):
+        arglist = ['--validation-dir', 'foo', '--category', 'networking']
+        verifylist = [('validation_dir', 'foo'),
+                      ('category', ['networking'])]
+
+        val_list = fakes.VALIDATION_LIST_RESULT
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+        result = self.cmd.take_action(parsed_args)
+        self.assertEqual(result, val_list)

validations_libs/tests/cli/test_run.py

@@ -71,6 +71,7 @@ class TestRun(BaseCommand):
             'inventory': 'localhost',
             'limit_hosts': None,
             'group': [],
+            'category': [],
             'extra_vars': {'key': 'value'},
             'validations_dir': '/usr/share/ansible/validation-playbooks',
             'base_dir': '/usr/share/ansible',
@@ -103,6 +104,7 @@ class TestRun(BaseCommand):
             'inventory': 'localhost',
             'limit_hosts': None,
             'group': [],
+            'category': [],
             'extra_vars': {'key': 'value2'},
             'validations_dir': '/usr/share/ansible/validation-playbooks',
             'base_dir': '/usr/share/ansible',
@@ -148,6 +150,7 @@ class TestRun(BaseCommand):
             'inventory': 'localhost',
             'limit_hosts': None,
             'group': [],
+            'category': [],
             'extra_vars': {'key': 'value'},
             'validations_dir': '/usr/share/ansible/validation-playbooks',
             'base_dir': '/usr/share/ansible',
@@ -178,6 +181,7 @@ class TestRun(BaseCommand):
             'inventory': 'localhost',
             'limit_hosts': None,
             'group': [],
+            'category': [],
             'extra_vars': None,
             'validations_dir': '/usr/share/ansible/validation-playbooks',
             'base_dir': '/usr/share/ansible',
@@ -213,6 +217,7 @@ class TestRun(BaseCommand):
             'log_path': mock_log_dir,
             'quiet': False,
             'group': [],
+            'category': [],
             'extra_vars': None,
             'validations_dir': '/usr/share/ansible/validation-playbooks',
             'base_dir': '/usr/share/ansible',
@@ -242,6 +247,7 @@ class TestRun(BaseCommand):
             'inventory': 'localhost',
             'limit_hosts': None,
             'group': [],
+            'category': [],
             'extra_vars': None,
             'validations_dir': '/usr/share/ansible/validation-playbooks',
             'base_dir': '/usr/share/ansible',
@@ -276,6 +282,7 @@ class TestRun(BaseCommand):
             'inventory': 'localhost',
             'limit_hosts': None,
             'group': [],
+            'category': [],
             'extra_vars': {'key': 'value'},
             'validations_dir': '/usr/share/ansible/validation-playbooks',
             'base_dir': '/usr/share/ansible',
@@ -317,6 +324,7 @@ class TestRun(BaseCommand):
             'inventory': 'localhost',
             'limit_hosts': None,
             'group': [],
+            'category': [],
             'extra_vars': None,
             'validations_dir': '/usr/share/ansible/validation-playbooks',
             'base_dir': '/usr/share/ansible',
@@ -343,6 +351,7 @@ class TestRun(BaseCommand):
             'inventory': 'localhost',
             'limit_hosts': None,
             'group': [],
+            'category': [],
             'extra_vars': {'key': 'value'},
             'validations_dir': '/usr/share/ansible/validation-playbooks',
             'base_dir': '/usr/share/ansible',

validations_libs/tests/cli/test_show.py

@@ -90,3 +90,11 @@ class TestShowParameter(BaseCommand):
         arglist = ['--group', 'prep']
         verifylist = [('group', ['prep'])]
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+    @mock.patch('validations_libs.validation_actions.ValidationActions.'
+                'show_validations_parameters')
+    def test_show_validations_parameters_by_categories(self, mock_show):
+        arglist = ['--category', 'os']
+        verifylist = [('category', ['os'])]
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+        self.cmd.take_action(parsed_args)

validations_libs/tests/fakes.py

@@ -18,12 +18,14 @@ from validations_libs import constants
 VALIDATIONS_LIST = [{
     'description': 'My Validation One Description',
     'groups': ['prep', 'pre-deployment'],
+    'categories': ['os', 'system', 'ram'],
     'id': 'my_val1',
     'name': 'My Validation One Name',
     'parameters': {}
 }, {
     'description': 'My Validation Two Description',
     'groups': ['prep', 'pre-introspection'],
+    'categories': ['networking'],
     'id': 'my_val2',
     'name': 'My Validation Two Name',
     'parameters': {'min_value': 8}
@@ -32,15 +34,17 @@ VALIDATIONS_LIST = [{
 VALIDATIONS_LIST_GROUP = [{
     'description': 'My Validation Two Description',
     'groups': ['prep', 'pre-introspection'],
+    'categories': ['networking'],
     'id': 'my_val2',
     'name': 'My Validation Two Name',
     'parameters': {'min_value': 8}
 }]
 
 
-VALIDATION_LIST_RESULT = (('ID', 'Name', 'Groups'),
+VALIDATION_LIST_RESULT = (('ID', 'Name', 'Groups', 'Categories'),
                           [('my_val2', 'My Validation Two Name',
-                            ['prep', 'pre-introspection'])])
+                            ['prep', 'pre-introspection'],
+                            ['networking'])])
 
 GROUPS_LIST = [
     ('group1', 'Group1 description'),
@@ -209,6 +213,7 @@ VALIDATIONS_LOGS_CONTENTS_LIST = [{
 
 VALIDATIONS_DATA = {'Description': 'My Validation One Description',
                     'Groups': ['prep', 'pre-deployment'],
+                    'categories': ['os', 'system', 'ram'],
                     'ID': 'my_val1',
                     'Name': 'My Validation One Name',
                     'parameters': {}}
@@ -223,6 +228,7 @@ FAKE_WRONG_PLAYBOOK = [{
         'nometadata': {
             'description': 'foo',
             'groups': ['prep', 'pre-deployment'],
+            'categories': ['os', 'storage'],
             'name': 'Advanced Format 512e Support'
         }
     }
@@ -232,6 +238,7 @@ FAKE_PLAYBOOK = [{'hosts': 'undercloud',
                   'roles': ['advanced_format_512e_support'],
                   'vars': {'metadata': {'description': 'foo',
                                         'groups': ['prep', 'pre-deployment'],
+                                        'categories': ['os', 'storage'],
                                         'name':
                                         'Advanced Format 512e Support'}}}]
 
@@ -239,6 +246,7 @@ FAKE_PLAYBOOK2 = [{'hosts': 'undercloud',
                    'roles': ['advanced_format_512e_support'],
                    'vars': {'metadata': {'description': 'foo',
                                          'groups': ['prep', 'pre-deployment'],
+                                         'categories': ['os', 'storage'],
                                          'name':
                                          'Advanced Format 512e Support'},
                             'foo': 'bar'}}]
@@ -255,11 +263,12 @@ FAKE_VARS = {'foo': 'bar'}
 FAKE_METADATA = {'id': 'foo',
                  'description': 'foo',
                  'groups': ['prep', 'pre-deployment'],
-                 'name':
+                 'categories': ['os', 'storage'],
-                 'Advanced Format 512e Support'}
+                 'name': 'Advanced Format 512e Support'}
 
 FORMATED_DATA = {'Description': 'foo',
                  'Groups': ['prep', 'pre-deployment'],
+                 'Categories': ['os', 'storage'],
                  'ID': 'foo',
                  'Name': 'Advanced Format 512e Support'}
 

validations_libs/tests/test_utils.py

@@ -36,6 +36,7 @@ class TestUtils(TestCase):
     def test_get_validations_data(self, mock_exists, mock_open, mock_data):
         output = {'Name': 'Advanced Format 512e Support',
                   'Description': 'foo', 'Groups': ['prep', 'pre-deployment'],
+                  'Categories': ['os', 'storage'],
                   'ID': '512e',
                   'Parameters': {}}
         res = utils.get_validations_data('512e')
@@ -69,6 +70,12 @@ class TestUtils(TestCase):
                           path='/foo/playbook',
                           groups='foo1,foo2')
 
+    def test_parse_all_validations_on_disk_wrong_categories_type(self):
+        self.assertRaises(TypeError,
+                          utils.parse_all_validations_on_disk,
+                          path='/foo/playbook',
+                          categories='foo1,foo2')
+
     def test_get_validations_playbook_wrong_validation_id_type(self):
         self.assertRaises(TypeError,
                           utils.get_validations_playbook,
@@ -81,6 +88,36 @@ class TestUtils(TestCase):
                           path='/foo/playbook',
                           groups='foo1,foo2')
 
+    def test_get_validations_playbook_wrong_categories_type(self):
+        self.assertRaises(TypeError,
+                          utils.get_validations_playbook,
+                          path='/foo/playbook',
+                          categories='foo1,foo2')
+
+    @mock.patch('yaml.safe_load', return_value=fakes.FAKE_PLAYBOOK)
+    @mock.patch('six.moves.builtins.open')
+    @mock.patch('glob.glob')
+    def test_parse_all_validations_on_disk_by_group(self, mock_glob,
+                                                    mock_open,
+                                                    mock_load):
+        mock_glob.return_value = \
+            ['/foo/playbook/foo.yaml']
+        result = utils.parse_all_validations_on_disk('/foo/playbook',
+                                                     ['prep'])
+        self.assertEqual(result, [fakes.FAKE_METADATA])
+
+    @mock.patch('yaml.safe_load', return_value=fakes.FAKE_PLAYBOOK)
+    @mock.patch('six.moves.builtins.open')
+    @mock.patch('glob.glob')
+    def test_parse_all_validations_on_disk_by_category(self, mock_glob,
+                                                       mock_open,
+                                                       mock_load):
+        mock_glob.return_value = \
+            ['/foo/playbook/foo.yaml']
+        result = utils.parse_all_validations_on_disk('/foo/playbook',
+                                                     categories=['os'])
+        self.assertEqual(result, [fakes.FAKE_METADATA])
+
     def test_get_validations_playbook_wrong_path_type(self):
         self.assertRaises(TypeError,
                           utils.get_validations_playbook,
@@ -124,6 +161,18 @@ class TestUtils(TestCase):
                                                 groups=['no_group'])
         self.assertEqual(result, [])
 
+    @mock.patch('os.path.isfile')
+    @mock.patch('os.listdir')
+    @mock.patch('yaml.safe_load', return_value=fakes.FAKE_PLAYBOOK)
+    @mock.patch('six.moves.builtins.open')
+    def test_get_validations_playbook_by_category(self, mock_open, mock_load,
+                                                  mock_listdir, mock_isfile):
+        mock_listdir.return_value = ['foo.yaml']
+        mock_isfile.return_value = True
+        result = utils.get_validations_playbook('/foo/playbook',
+                                                categories=['os', 'storage'])
+        self.assertEqual(result, ['/foo/playbook/foo.yaml'])
+
     @mock.patch('yaml.safe_load', return_value=fakes.FAKE_PLAYBOOK)
     @mock.patch('six.moves.builtins.open')
     def test_get_validation_parameters(self, mock_open, mock_load):

validations_libs/tests/test_validation.py

@@ -103,6 +103,28 @@ class TestValidation(TestCase):
         groups = val.groups
         self.assertEqual(groups, [])
 
+    @mock.patch('yaml.safe_load', return_value=fakes.FAKE_PLAYBOOK)
+    @mock.patch('six.moves.builtins.open')
+    def test_categories(self, mock_open, mock_yaml):
+        val = Validation('/tmp/foo')
+        categories = val.categories
+        self.assertEqual(categories, ['os', 'storage'])
+
+    @mock.patch('yaml.safe_load', return_value=fakes.FAKE_WRONG_PLAYBOOK)
+    @mock.patch('six.moves.builtins.open')
+    def test_categories_with_no_metadata(self, mock_open, mock_yaml):
+        with self.assertRaises(NameError) as exc_mgr:
+            Validation('/tmp/foo').categories
+        self.assertEqual('No metadata found in validation foo',
+                         str(exc_mgr.exception))
+
+    @mock.patch('yaml.safe_load', return_value=fakes.FAKE_PLAYBOOK3)
+    @mock.patch('six.moves.builtins.open')
+    def test_categories_with_no_existing_categories(self, mock_open, mock_yaml):
+        val = Validation('/tmp/foo')
+        categories = val.categories
+        self.assertEqual(categories, [])
+
     @mock.patch('yaml.safe_load', return_value=fakes.FAKE_PLAYBOOK)
     @mock.patch('six.moves.builtins.open')
     def test_get_ordered_dict(self, mock_open, mock_yaml):

validations_libs/tests/test_validation_actions.py

@@ -30,20 +30,22 @@ class TestValidationActions(TestCase):
 
     def setUp(self):
         super(TestValidationActions, self).setUp()
-        self.column_name = ('ID', 'Name', 'Groups')
+        self.column_name = ('ID', 'Name', 'Groups', 'Categories')
 
     @mock.patch('validations_libs.utils.parse_all_validations_on_disk',
                 return_value=fakes.VALIDATIONS_LIST)
     def test_validation_list(self, mock_validation_dir):
-        validations_list = ValidationActions(fakes.GROUPS_LIST, '/tmp/foo')
+        validations_list = ValidationActions('/tmp/foo')
 
         self.assertEqual(validations_list.list_validations(),
                          (self.column_name, [('my_val1',
                                               'My Validation One Name',
-                                              ['prep', 'pre-deployment']),
+                                              ['prep', 'pre-deployment'],
+                                              ['os', 'system', 'ram']),
                                              ('my_val2',
                                               'My Validation Two Name',
-                                             ['prep', 'pre-introspection'])]))
+                                              ['prep', 'pre-introspection'],
+                                              ['networking'])]))
 
     @mock.patch('validations_libs.utils.os.access', return_value=True)
     @mock.patch('validations_libs.utils.os.path.exists', return_value=True)
@@ -348,6 +350,7 @@ class TestValidationActions(TestCase):
                              mock_parse_validation, mock_data, mock_log):
         data = {'Name': 'Advanced Format 512e Support',
                 'Description': 'foo', 'Groups': ['prep', 'pre-deployment'],
+                'Categories': ['os', 'storage'],
                 'ID': '512e',
                 'Parameters': {}}
         data.update({'Last execution date': '2019-11-25 13:40:14',
@@ -378,6 +381,27 @@ class TestValidationActions(TestCase):
                                   ('post', 'post-foo', 2),
                                   ('pre', 'pre-foo', 2)])
 
+    @mock.patch('six.moves.builtins.open')
+    def test_show_validations_parameters_wrong_validations_type(self, mock_open):
+        v_actions = ValidationActions()
+        self.assertRaises(TypeError,
+                          v_actions.show_validations_parameters,
+                          validations='foo')
+
+    @mock.patch('six.moves.builtins.open')
+    def test_show_validations_parameters_wrong_groups_type(self, mock_open):
+        v_actions = ValidationActions()
+        self.assertRaises(TypeError,
+                          v_actions.show_validations_parameters,
+                          groups=('foo'))
+
+    @mock.patch('six.moves.builtins.open')
+    def test_show_validations_parameters_wrong_categories_type(self, mock_open):
+        v_actions = ValidationActions()
+        self.assertRaises(TypeError,
+                          v_actions.show_validations_parameters,
+                          categories={'foo': 'bar'})
+
     @mock.patch('validations_libs.utils.get_validations_playbook',
                 return_value=['/foo/playbook/foo.yaml'])
     @mock.patch('validations_libs.utils.get_validations_parameters')
@@ -388,7 +412,7 @@ class TestValidationActions(TestCase):
         mock_get_param.return_value = {'foo':
                                        {'parameters': fakes.FAKE_METADATA}}
         v_actions = ValidationActions()
-        result = v_actions.show_validations_parameters('foo')
+        result = v_actions.show_validations_parameters(validations=['foo'])
         self.assertEqual(result, mock_get_param.return_value)
 
     @mock.patch('six.moves.builtins.open')
@@ -396,7 +420,7 @@ class TestValidationActions(TestCase):
         v_actions = ValidationActions()
         self.assertRaises(RuntimeError,
                           v_actions.show_validations_parameters,
-                          validation='foo', output_format='bar')
+                          validations=['foo'], output_format='bar')
 
     @mock.patch('validations_libs.validation_logs.ValidationLogs.'
                 'get_logfile_by_validation',

validations_libs/utils.py

@@ -128,66 +128,93 @@ def create_artifacts_dir(log_path=constants.VALIDATIONS_LOG_BASEDIR,
         raise RuntimeError()
 
 
-def parse_all_validations_on_disk(path, groups=None):
+def parse_all_validations_on_disk(path, groups=None, categories=None):
-    """Return a list of validations metadata which can be sorted by Groups
+    """Return a list of validations metadata which can be sorted by Groups or by
+    Categories.
 
     :param path: The absolute path of the validations directory
     :type path: `string`
+
     :param groups: Groups of validations
     :type groups: `list`
-    :return: A list of validations metadata
+
+    :param categories: Categories of validations
+    :type categories: `list`
+
+    :return: A list of validations metadata.
     :rtype: `list`
 
     :Example:
 
     >>> path = '/foo/bar'
     >>> parse_all_validations_on_disk(path)
-    [{'description': 'Detect whether the node disks use Advanced Format.',
+    [{'categories': ['storage'],
+      'description': 'Detect whether the node disks use Advanced Format.',
       'groups': ['prep', 'pre-deployment'],
       'id': '512e',
       'name': 'Advanced Format 512e Support'},
-     {'description': 'Make sure that the server has enough CPU cores.',
+     {'categories': ['system'],
+      'description': 'Make sure that the server has enough CPU cores.',
       'groups': ['prep', 'pre-introspection'],
       'id': 'check-cpu',
       'name': 'Verify if the server fits the CPU core requirements'}]
     """
     if not isinstance(path, six.string_types):
-        raise TypeError("The 'path' argument should be a String")
+        raise TypeError("The 'path' argument must be a String")
 
     if not groups:
         groups = []
+    elif not isinstance(groups, list):
+        raise TypeError("The 'groups' argument must be a List")
 
-    if not isinstance(groups, list):
+    if not categories:
-        raise TypeError("The 'groups' argument should be a List")
+        categories = []
+    elif not isinstance(categories, list):
+        raise TypeError("The 'categories' argument must be a List")
 
     results = []
     validations_abspath = glob.glob("{path}/*.yaml".format(path=path))
 
     LOG.debug(
-        "Attempting to parse validations of groups `{}` from {}".format(
+        "Attempting to parse validations by:\n"
-            ','.join(groups),
+        "  - groups: {}\n"
-            validations_abspath
+        "  - categories: {}\n"
-        )
+        "from {}".format(groups, categories, validations_abspath)
     )
 
     for playbook in validations_abspath:
         val = Validation(playbook)
 
-        if not groups or set(groups).intersection(val.groups):
+        if not groups and not categories:
             results.append(val.get_metadata)
+            continue
+
+        if set(groups).intersection(val.groups) or \
+           set(categories).intersection(val.categories):
+            results.append(val.get_metadata)
+
     return results
 
 
-def get_validations_playbook(path, validation_id=None, groups=None):
+def get_validations_playbook(path,
-    """Get a list of validations playbooks paths either by their names
+                             validation_id=None,
-    or their groups
+                             groups=None,
+                             categories=None):
+    """Get a list of validations playbooks paths either by their names,
+    their groups or by their categories.
 
     :param path: Path of the validations playbooks
     :type path: `string`
+
     :param validation_id: List of validation name
     :type validation_id: `list`
+
     :param groups: List of validation group
     :type groups: `list`
+
+    :param categories: List of validation category
+    :type categories: `list`
+
     :return: A list of absolute validations playbooks path
     :rtype: `list`
 
@@ -196,24 +223,28 @@ def get_validations_playbook(path, validation_id=None, groups=None):
     >>> path = '/usr/share/validation-playbooks'
     >>> validation_id = ['512e','check-cpu']
     >>> groups = None
-    >>> get_validations_playbook(path, validation_id, groups)
+    >>> categories = None
+    >>> get_validations_playbook(path, validation_id, groups, categories)
     ['/usr/share/ansible/validation-playbooks/512e.yaml',
      '/usr/share/ansible/validation-playbooks/check-cpu.yaml',]
     """
     if not isinstance(path, six.string_types):
-        raise TypeError("The 'path' argument should be a String")
+        raise TypeError("The 'path' argument must be a String")
 
     if not validation_id:
         validation_id = []
-
+    elif not isinstance(validation_id, list):
-    if not isinstance(validation_id, list):
+        raise TypeError("The 'validation_id' argument must be a List")
-        raise TypeError("The 'validation_id' argument should be a List")
 
     if not groups:
         groups = []
+    elif not isinstance(groups, list):
+        raise TypeError("The 'groups' argument must be a List")
 
-    if not isinstance(groups, list):
+    if not categories:
-        raise TypeError("The 'groups' argument should be a List")
+        categories = []
+    elif not isinstance(categories, list):
+        raise TypeError("The 'categories' argument must be a List")
 
     pl = []
     for f in os.listdir(path):
@@ -223,10 +254,14 @@ def get_validations_playbook(path, validation_id=None, groups=None):
                 if os.path.splitext(f)[0] in validation_id or \
                         os.path.basename(f) in validation_id:
                     pl.append(pl_path)
+
+            val = Validation(pl_path)
             if groups:
-                val = Validation(pl_path)
                 if set(groups).intersection(val.groups):
                     pl.append(pl_path)
+            if categories:
+                if set(categories).intersection(val.categories):
+                    pl.append(pl_path)
     return pl
 
 
@@ -290,11 +325,12 @@ def get_validations_details(validation):
     >>> get_validations_details(validation)
     {'description': 'Verify that the server has enough something.',
      'groups': ['group1', 'group2'],
+     'categories': ['category1', 'category2'],
      'id': 'check-something',
      'name': 'Verify the server fits the something requirements'}
     """
     if not isinstance(validation, six.string_types):
-        raise TypeError("The 'validation' argument should be a String")
+        raise TypeError("The 'validation' argument must be a String")
 
     results = parse_all_validations_on_disk(constants.ANSIBLE_VALIDATION_DIR)
     for r in results:
@@ -323,12 +359,13 @@ def get_validations_data(validation, path=constants.ANSIBLE_VALIDATION_DIR):
     >>> get_validations_data(validation)
     {'Description': 'Verify that the server has enough something',
      'Groups': ['group1', 'group2'],
+     'Categories': ['category1', 'category2'],
      'ID': 'check-something',
      'Name': 'Verify the server fits the something requirements',
      'Parameters': {'param1': 24}}
     """
     if not isinstance(validation, six.string_types):
-        raise TypeError("The 'validation' argument should be a String")
+        raise TypeError("The 'validation' argument must be a String")
 
     data = {}
     val_path = "{}/{}.yaml".format(path, validation)
@@ -348,7 +385,8 @@ def get_validations_data(validation, path=constants.ANSIBLE_VALIDATION_DIR):
 
 def get_validations_parameters(validations_data,
                                validation_name=None,
-                               groups=None):
+                               groups=None,
+                               categories=None):
     """Return parameters for a list of validations
 
 
@@ -358,6 +396,8 @@ def get_validations_parameters(validations_data,
     :type validation_name: `list`
     :param groups: A list of validation groups
     :type groups: `list`
+    :param categories: A list of validation categories
+    :type categories: `list`
     :return: a dictionary containing the current parameters for
              each `validation_name` or `groups`
     :rtype: `dict`
@@ -372,24 +412,29 @@ def get_validations_parameters(validations_data,
      'check-ram': {'parameters': {'minimal_ram_gb': 24}}}
     """
     if not isinstance(validations_data, list):
-        raise TypeError("The 'validations_data' argument should be a List")
+        raise TypeError("The 'validations_data' argument must be a List")
 
     if not validation_name:
         validation_name = []
-
+    elif not isinstance(validation_name, list):
-    if not isinstance(validation_name, list):
+        raise TypeError("The 'validation_name' argument must be a List")
-        raise TypeError("The 'validation_name' argument should be a List")
 
     if not groups:
         groups = []
+    elif not isinstance(groups, list):
+        raise TypeError("The 'groups' argument must be a List")
 
-    if not isinstance(groups, list):
+    if not categories:
-        raise TypeError("The 'groups' argument should be a List")
+        categories = []
+    elif not isinstance(categories, list):
+        raise TypeError("The 'categories' argument must be a List")
 
     params = {}
     for val in validations_data:
         v = Validation(val)
-        if v.id in validation_name or set(groups).intersection(v.groups):
+        if v.id in validation_name or \
+           set(groups).intersection(v.groups) or \
+           set(categories).intersection(v.categories):
             params[v.id] = {
                 'parameters': v.get_vars
             }

validations_libs/validation.py

@@ -35,7 +35,7 @@ class Validation(object):
               name: Hello World
               description: This validation prints Hello World!
           roles:
-          - hello-world
+          - hello_world
 
     As shown here, the validation playbook requires three top-level
     directives:
@@ -50,9 +50,14 @@ class Validation(object):
     values are then reported by the API.
 
     The validations can be grouped together by specifying a ``groups``
-    metadata. Groups function similar to tags and a validation can thus be part
+    and a ``categories`` metadata. ``groups`` are the deployment stage the
-    of many groups. Here is, for example, how to have a validation be part of
+    validations should run on and ``categories`` are the technical
-    the `pre-deployment` and `hardware` groups.
+    classification for the validations.
+
+    Groups and Categories function similar to tags and a validation can thus be
+    part of many groups and many categories.
+
+    Here is an example:
 
     .. code-block:: yaml
 
@@ -64,12 +69,17 @@ class Validation(object):
               groups:
                 - pre-deployment
                 - hardware
+              categories:
+                - os
+                - networking
+                - storage
+                - security
           roles:
-          - hello-world
+          - hello_world
 
     """
 
-    _col_keys = ['ID', 'Name', 'Description', 'Groups']
+    _col_keys = ['ID', 'Name', 'Description', 'Groups', 'Categories']
 
     def __init__(self, validation_path):
         self.dict = self._get_content(validation_path)
@@ -96,8 +106,13 @@ class Validation(object):
                   groups:
                     - pre-deployment
                     - hardware
+                  categories:
+                    - os
+                    - networking
+                    - storage
+                    - security
               roles:
-              - hello-world
+              - hello_world
 
         :return: `true` if `vars` is found, `false` if not.
         :rtype: `boolean`
@@ -118,8 +133,13 @@ class Validation(object):
                   groups:
                     - pre-deployment
                     - hardware
+                  categories:
+                    - os
+                    - networking
+                    - storage
+                    - security
               roles:
-              - hello-world
+              - hello_world
 
         :return: `true` if `vars` and metadata are found, `false` if not.
         :rtype: `boolean`
@@ -142,6 +162,7 @@ class Validation(object):
         >>> print(val.get_metadata)
         {'description': 'Val1 desc.',
          'groups': ['group1', 'group2'],
+         'categories': ['category1', 'category2'],
          'id': 'val1',
          'name': 'The validation val1\'s name'}
         """
@@ -197,6 +218,7 @@ class Validation(object):
          'roles': ['val_role'],
          'vars': {'metadata': {'description': 'description of val ',
                                'groups': ['group1', 'group2'],
+                               'categories': ['category1', 'category2'],
                                'name': 'validation one'},
                                'var_name1': 'value1'}}
         """
@@ -225,6 +247,29 @@ class Validation(object):
                 "No metadata found in validation {}".format(self.id)
             )
 
+    @property
+    def categories(self):
+        """Get the validation list of categories
+
+        :return: A list of categories for the validation
+        :rtype: `list` or `None` if no metadata has been found
+        :raise: A `NameError` exception if no metadata has been found in the
+                playbook
+
+        :Example:
+
+        >>> pl = '/foo/bar/val.yaml'
+        >>> val = Validation(pl)
+        >>> print(val.categories)
+        ['category1', 'category2']
+        """
+        if self.has_metadata_dict:
+            return self.dict['vars']['metadata'].get('categories', [])
+        else:
+            raise NameError(
+                "No metadata found in validation {}".format(self.id)
+            )
+
     @property
     def get_id(self):
         """Get the validation id
@@ -256,7 +301,8 @@ class Validation(object):
         """Get basic information from a validation for output display
 
         :return: Basic information of a validation including the `Description`,
-                 the list of `Groups`, the `ID` and the `Name`.
+                 the list of 'Categories', the list of `Groups`, the `ID` and
+                 the `Name`.
         :rtype: `dict`
         :raise: A `NameError` exception if no metadata has been found in the
                 playbook
@@ -266,7 +312,8 @@ class Validation(object):
         >>> pl = '/foo/bar/val.yaml'
         >>> val = Validation(pl)
         >>> print(val.get_formated_data)
-        {'Description': 'description of val',
+        {'Categories': ['category1', 'category2'],
+         'Description': 'description of val',
          'Groups': ['group1', 'group2'],
          'ID': 'val',
          'Name': 'validation one'}

validations_libs/validation_actions.py

@@ -42,57 +42,66 @@ class ValidationActions(object):
 
     """
 
-    def __init__(self, validation_path=None, group=None):
+    def __init__(self, validation_path=None):
         self.log = logging.getLogger(__name__ + ".ValidationActions")
         self.validation_path = (validation_path if validation_path
                                 else constants.ANSIBLE_VALIDATION_DIR)
 
-    def list_validations(self, group=None):
+    def list_validations(self, groups=None, categories=None):
-        """Get a list of the validations selected by group
+        """Get a list of the validations selected by group membership or by
-        membership. With their names and group membership information.
+        category. With their names, group membership information and categories.
+
         This is used to print table from python ``Tuple`` with ``PrettyTable``.
 
-        :param group: Group or multiple groups of validations.
+        :param groups: List of validation groups.
-                      Additional groups have to be separated by comma.
+        :type groups: `list`
-        :type group: `string`
+
+        :param categories: List of validation categories.
+        :type categories: `list`
 
         :return: Column names and a list of the selected validations
         :rtype: `tuple`
 
         .. code:: text
 
-            ----------------+--------------------------+----------------------+
+            -------+-----------+----------------------+---------------+
-            | ID            | Name                     | Groups               |
+            | ID   | Name      | Groups               | Categories    |
-            +---------------+--------------------------+----------------------+
+            +------+-----------+----------------------+---------------+
-            | validation1   | Name of the validation1  | ['group1']           |
+            | val1 | val_name1 | ['group1']           | ['category1'] |
-            | validation2   | Name of the validation2  | ['group1', 'group2'] |
+            | val2 | val_name2 | ['group1', 'group2'] | ['category2'] |
-            | validation3   | Name of the validation3  | ['group4]            |
+            | val3 | val_name3 | ['group4']           | ['category3'] |
-            +---------------+--------------------------+----------------------+
+            +------+-----------+----------------------+---------------+
 
         :Example:
 
         >>> path = "/foo/bar"
+        >>> groups = ['group1']
+        >>> categories = ['category1']
         >>> action = ValidationActions(validation_path=path)
-        >>> results = action.list_validations()
+        >>> results = action.list_validations(groups=groups,
-        >>> print(results)
+                                              categories=categories)
-        (('ID', 'Name', 'Groups'),
+        >>> print(results
-         [('validation1', 'Name of the validation1', ['group1']),
+        (('ID', 'Name', 'Groups', 'Categories'),
-          ('validation2', 'Name of the validation2', ['group1', 'group2'])])
+         [('val1', 'val_name1', ['group1'], ['category1']),
+          ('val2', 'val_name2', ['group1', 'group2'], ['category2'])])
         """
         self.log = logging.getLogger(__name__ + ".list_validations")
 
         validations = v_utils.parse_all_validations_on_disk(
-            self.validation_path, group)
+            path=self.validation_path,
+            groups=groups,
+            categories=categories)
 
         self.log.debug(
             "Parsed {} validations.".format(len(validations))
         )
 
         return_values = [
-            (val.get('id'), val.get('name'), val.get('groups'))
+            (val.get('id'), val.get('name'),
+             val.get('groups'), val.get('categories'))
             for val in validations]
 
-        column_names = ('ID', 'Name', 'Groups')
+        column_names = ('ID', 'Name', 'Groups', 'Categories')
 
         return (column_names, return_values)
 
@@ -117,6 +126,7 @@ class ValidationActions(object):
         >>> print(results)
         {
          'Description': 'Description of the foo validation',
+         'Categories': ['category1', 'category2'],
          'Groups': ['group1', 'group2'],
          'ID': 'foo',
          'Last execution date': None,
@@ -231,13 +241,13 @@ class ValidationActions(object):
         return [path[1] for path in logs[-history_limit:]]
 
     def run_validations(self, validation_name=None, inventory='localhost',
-                        group=None, extra_vars=None, validations_dir=None,
+                        group=None, category=None, extra_vars=None,
-                        extra_env_vars=None, ansible_cfg=None, quiet=True,
+                        validations_dir=None, extra_env_vars=None,
-                        workdir=None, limit_hosts=None, run_async=False,
+                        ansible_cfg=None, quiet=True, workdir=None,
+                        limit_hosts=None, run_async=False,
                         base_dir=constants.DEFAULT_VALIDATIONS_BASEDIR,
                         log_path=constants.VALIDATIONS_LOG_BASEDIR,
-                        python_interpreter=None,
+                        python_interpreter=None, skip_list=None,
-                        skip_list=None,
                         callback_whitelist=None,
                         output_callback='validation_stdout',
                         ssh_user=None):
@@ -250,6 +260,8 @@ class ValidationActions(object):
         :type inventory: ``string``
         :param group: A list of group names
         :type group: ``list``
+        :param category: A list of category names
+        :type category: ``list``
         :param extra_vars: Set additional variables as a Dict or the absolute
                            path of a JSON or YAML file type.
         :type extra_vars: Either a Dict or the absolute path of JSON or YAML
@@ -333,15 +345,16 @@ class ValidationActions(object):
         playbooks = []
         validations_dir = (validations_dir if validations_dir
                            else self.validation_path)
-        if group:
+        if group or category:
-            self.log.debug('Getting the validations list by group')
+            self.log.debug(
-            try:
+                "Getting the validations list by:\n"
-                validations = v_utils.parse_all_validations_on_disk(
+                "  - groups: {}\n"
-                    validations_dir, group)
+                "  - categories: {}".format(group, category)
-                for val in validations:
+            )
-                    playbooks.append(val.get('id') + '.yaml')
+            validations = v_utils.parse_all_validations_on_disk(
-            except Exception as e:
+                path=validations_dir, groups=group, categories=category)
-                raise(e)
+            for val in validations:
+                playbooks.append(val.get('id') + '.yaml')
         elif validation_name:
             playbooks = v_utils.get_validations_playbook(validations_dir,
                                                          validation_name,
@@ -478,17 +491,24 @@ class ValidationActions(object):
         column_name = ("Groups", "Description", "Number of Validations")
         return (column_name, group_info)
 
-    def show_validations_parameters(self, validation=None, group=None,
+    def show_validations_parameters(self,
-                                    output_format='json', download_file=None):
+                                    validations=None,
+                                    groups=None,
+                                    categories=None,
+                                    output_format='json',
+                                    download_file=None):
         """
         Return Validations Parameters for one or several validations by their
-        names or their groups.
+        names, their groups or by their categories.
 
-        :param validation: List of validation name(s)
+        :param validations: List of validation name(s)
-        :type validation: `list`
+        :type validations: `list`
 
-        :param group: List of validation group(s)
+        :param groups: List of validation group(s)
-        :type group: `list`
+        :type groups: `list`
+
+        :param categories: List of validation category(ies)
+        :type categories: `list`
 
         :param output_format: Output format (Supported format are JSON or YAML)
         :type output_format: `string`
@@ -502,10 +522,12 @@ class ValidationActions(object):
                  parameters will be created in the file system.
         :exemple:
 
-        >>> validation = ['check-cpu', 'check-ram']
+        >>> validations = ['check-cpu', 'check-ram']
-        >>> group = None
+        >>> groups = None
+        >>> categories = None
         >>> output_format = 'json'
-        >>> show_validations_parameters(validation, group, output_format)
+        >>> show_validations_parameters(validations, groups,
+                                        categories, output_format)
         {
             "check-cpu": {
                 "parameters": {
@@ -519,21 +541,38 @@ class ValidationActions(object):
             }
         }
         """
-        if not validation:
+        if not validations:
-            validation = []
+            validations = []
+        elif not isinstance(validations, list):
+            raise TypeError("The 'validations' argument must be a List")
 
-        if not group:
+        if not groups:
-            group = []
+            groups = []
+        elif not isinstance(groups, list):
+            raise TypeError("The 'groups' argument must be a List")
+
+        if not categories:
+            categories = []
+        elif not isinstance(categories, list):
+            raise TypeError("The 'categories' argument must be a List")
 
         supported_format = ['json', 'yaml']
 
         if output_format not in supported_format:
             raise RuntimeError("{} output format not supported".format(output_format))
 
-        validations = v_utils.get_validations_playbook(
+        validation_playbooks = v_utils.get_validations_playbook(
-            self.validation_path, validation, group)
+            path=self.validation_path,
-        params = v_utils.get_validations_parameters(validations, validation,
+            validation_id=validations,
-                                                    group)
+            groups=groups,
+            categories=categories)
+
+        params = v_utils.get_validations_parameters(
+            validations_data=validation_playbooks,
+            validation_name=validations,
+            groups=groups,
+            categories=categories)
+
         if download_file:
             params_only = {}
             try: