Implement verify profiles as a custom action

Adds a custom action to verify the number of nodes with proper profiles
exist in the baremetal inventory.

Change-Id: If6460877188af4d006df5f75ab5a921b411088e2
Partial-Bug: #1638697

releasenotes/notes/verify-profiles-a9d075f565bc3df6.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Adds an action and workflow used to verify the profiles
+    assigned to nodes and their count.

setup.cfg

@@ -103,5 +103,6 @@ mistral.actions =
     tripleo.validations.list_groups = tripleo_common.actions.validations:ListGroupsAction
     tripleo.validations.list_validations = tripleo_common.actions.validations:ListValidationsAction
     tripleo.validations.run_validation = tripleo_common.actions.validations:RunValidationAction
+    tripleo.validations.verify_profiles = tripleo_common.actions.validations:VerifyProfilesAction
     # deprecated for pike release, will be removed in queens
     tripleo.templates.upload_default = tripleo_common.actions.templates:UploadTemplatesAction

tripleo_common/actions/validations.py

@@ -295,3 +295,92 @@ class CheckNodeBootConfigurationAction(base.TripleOAction):
             mistral_result = {'data': return_value}
 
         return mistral_workflow_utils.Result(**mistral_result)
+
+
+class VerifyProfilesAction(base.TripleOAction):
+    """Verify that the profiles have enough nodes"""
+
+    # TODO(bcrochet): The validation actions are temporary. This logic should
+    #                 move to the tripleo-validations project eventually.
+    def __init__(self, nodes, flavors):
+        super(VerifyProfilesAction, self).__init__()
+
+        self.nodes = nodes
+        self.flavors = flavors
+
+    def run(self):
+        errors = []
+        warnings = []
+
+        bm_nodes = {node['uuid']: node for node in self.nodes
+                    if node['provision_state'] in ('available', 'active')}
+        free_node_caps = {uu: self._node_get_capabilities(node)
+                          for uu, node in bm_nodes.items()}
+
+        profile_flavor_used = False
+        for flavor_name, (flavor, scale) in self.flavors.items():
+            if not scale:
+                continue
+
+            profile = flavor.get('capabilities:profile')
+
+            if not profile and len(self.flavors) > 1:
+                message = ('Error: The {flavor} flavor has no profile '
+                           'associated.\n'
+                           'Recommendation: assign a profile with openstack '
+                           'flavor set --property '
+                           '"capabilities:profile"="PROFILE_NAME" {flavor}')
+
+                errors.append(message.format(flavor=flavor_name))
+                continue
+
+            profile_flavor_used = True
+
+            assigned_nodes = [uu for uu, caps in free_node_caps.items()
+                              if caps.get('profile') == profile]
+            required_count = scale - len(assigned_nodes)
+
+            if required_count < 0:
+                warnings.append('%d nodes with profile %s won\'t be used '
+                                'for deployment now' % (-required_count,
+                                                        profile))
+                required_count = 0
+
+            for uu in assigned_nodes:
+                free_node_caps.pop(uu)
+
+            if required_count > 0:
+                message = ('Error: only {total} of {scale} requested ironic '
+                           'nodes are tagged to profile {profile} (for flavor '
+                           '{flavor}).\n'
+                           'Recommendation: tag more nodes using openstack '
+                           'baremetal node set --property "capabilities='
+                           'profile:{profile},boot_option:local" <NODE ID>')
+                errors.append(message.format(total=scale - required_count,
+                                             scale=scale,
+                                             profile=profile,
+                                             flavor=flavor_name))
+
+        nodes_without_profile = [uu for uu, caps in free_node_caps.items()
+                                 if not caps.get('profile')]
+        if nodes_without_profile and profile_flavor_used:
+            warnings.append("There are %d ironic nodes with no profile that "
+                            "will not be used: %s" % (
+                                len(nodes_without_profile),
+                                ', '.join(nodes_without_profile)))
+
+        return_value = {
+            'errors': errors,
+            'warnings': warnings,
+        }
+        if errors:
+            mistral_result = {'error': return_value}
+        else:
+            mistral_result = {'data': return_value}
+
+        return mistral_workflow_utils.Result(**mistral_result)
+
+    def _node_get_capabilities(self, node):
+        """Get node capabilities."""
+        return nodeutils.capabilities_to_dict(
+            node['properties'].get('capabilities'))

tripleo_common/tests/actions/test_validations.py

@@ -14,6 +14,7 @@
 # under the License.
 import collections
 import mock
+from uuid import uuid4
 
 from mistral.workflow import utils as mistral_workflow_utils
 from oslo_concurrency.processutils import ProcessExecutionError
@@ -22,6 +23,7 @@ from tripleo_common.actions import validations
 from tripleo_common import constants
 from tripleo_common.tests import base
 from tripleo_common.tests.utils import test_validations
+from tripleo_common.utils import nodes as nodeutils
 
 
 class GetPubkeyActionTest(base.TestCase):
@@ -455,3 +457,202 @@ class TestCheckNodeBootConfigurationAction(base.TestCase):
 
         action = validations.CheckNodeBootConfigurationAction(**action_args)
         self.assertEqual(expected, action.run())
+
+
+class TestVerifyProfilesAction(base.TestCase):
+    def setUp(self):
+        super(TestVerifyProfilesAction, self).setUp()
+
+        self.nodes = []
+        self.flavors = {name: (self._get_fake_flavor(name), 1)
+                        for name in ('compute', 'control')}
+
+    def _get_fake_node(self, profile=None, possible_profiles=[],
+                       provision_state='available'):
+        caps = {'%s_profile' % p: '1'
+                for p in possible_profiles}
+        if profile is not None:
+            caps['profile'] = profile
+        caps = nodeutils.dict_to_capabilities(caps)
+        return {
+            'uuid': str(uuid4()),
+            'properties': {'capabilities': caps},
+            'provision_state': provision_state,
+        }
+
+    def _get_fake_flavor(self, name, profile=''):
+        the_profile = profile or name
+        return {
+            'name': name,
+            'profile': the_profile,
+            'capabilities:boot_option': 'local',
+            'capabilities:profile': the_profile
+        }
+
+    def _test(self, expected_result):
+        action = validations.VerifyProfilesAction(self.nodes, self.flavors)
+        result = action.run()
+
+        self.assertEqual(expected_result, result)
+
+    def test_no_matching_without_scale(self):
+        self.flavors = {name: (object(), 0)
+                        for name in self.flavors}
+        self.nodes[:] = [self._get_fake_node(profile='fake'),
+                         self._get_fake_node(profile='fake')]
+
+        expected = mistral_workflow_utils.Result(
+            data={
+                'errors': [],
+                'warnings': [],
+            })
+        self._test(expected)
+
+    def test_exact_match(self):
+        self.nodes[:] = [self._get_fake_node(profile='compute'),
+                         self._get_fake_node(profile='control')]
+
+        expected = mistral_workflow_utils.Result(
+            data={
+                'errors': [],
+                'warnings': [],
+            })
+        self._test(expected)
+
+    def test_nodes_with_no_profiles_present(self):
+        self.nodes[:] = [self._get_fake_node(profile='compute'),
+                         self._get_fake_node(profile=None),
+                         self._get_fake_node(profile='foobar'),
+                         self._get_fake_node(profile='control')]
+
+        expected = mistral_workflow_utils.Result(
+            data={
+                'warnings': [
+                    'There are 1 ironic nodes with no profile that will not '
+                    'be used: %s' % self.nodes[1].get('uuid')
+                ],
+                'errors': [],
+            })
+        self._test(expected)
+
+    def test_more_nodes_with_profiles_present(self):
+        self.nodes[:] = [self._get_fake_node(profile='compute'),
+                         self._get_fake_node(profile='compute'),
+                         self._get_fake_node(profile='compute'),
+                         self._get_fake_node(profile='control')]
+
+        expected = mistral_workflow_utils.Result(
+            data={
+                'warnings': ["2 nodes with profile compute won't be used for "
+                             "deployment now"],
+                'errors': [],
+            })
+        self._test(expected)
+
+    def test_no_nodes(self):
+        # One error per each flavor
+        expected = mistral_workflow_utils.Result(
+            error={'errors': ['Error: only 0 of 1 requested ironic nodes are '
+                              'tagged to profile compute (for flavor '
+                              'compute)\n'
+                              'Recommendation: tag more nodes using openstack '
+                              'baremetal node set --property  '
+                              '"capabilities=profile:compute,'
+                              'boot_option:local" <NODE ID>',
+                              'Error: only 0 of 1 requested ironic nodes are '
+                              'tagged to profile control (for flavor '
+                              'control).\n'
+                              'Recommendation: tag more nodes using openstack '
+                              'baremetal node set --property '
+                              '"capabilities=profile:control,'
+                              'boot_option:local" <NODE ID>'],
+                   'warnings': []})
+
+        action = validations.VerifyProfilesAction(self.nodes, self.flavors)
+        result = action.run()
+        self.assertEqual(expected.error['errors'].sort(),
+                         result.error['errors'].sort())
+        self.assertEqual(expected.error['warnings'], result.error['warnings'])
+        self.assertEqual(None, result.data)
+
+    def test_not_enough_nodes(self):
+        self.nodes[:] = [self._get_fake_node(profile='compute')]
+        expected = mistral_workflow_utils.Result(
+            error={'errors': ['Error: only 0 of 1 requested ironic nodes are '
+                              'tagged to profile control (for flavor '
+                              'control).\n'
+                              'Recommendation: tag more nodes using openstack '
+                              'baremetal node set --property '
+                              '"capabilities=profile:control,'
+                              'boot_option:local" <NODE ID>'],
+                   'warnings': []})
+        self._test(expected)
+
+    def test_scale(self):
+        # active nodes with assigned profiles are fine
+        self.nodes[:] = [self._get_fake_node(profile='compute',
+                                             provision_state='active'),
+                         self._get_fake_node(profile='control')]
+
+        expected = mistral_workflow_utils.Result(
+            data={
+                'errors': [],
+                'warnings': [],
+            }
+        )
+        self._test(expected)
+
+    def test_assign_profiles_wrong_state(self):
+        # active nodes are not considered for assigning profiles
+        self.nodes[:] = [self._get_fake_node(possible_profiles=['compute'],
+                                             provision_state='active'),
+                         self._get_fake_node(possible_profiles=['control'],
+                                             provision_state='cleaning'),
+                         self._get_fake_node(profile='compute',
+                                             provision_state='error')]
+        expected = mistral_workflow_utils.Result(
+            error={
+                'warnings': [
+                    'There are 1 ironic nodes with no profile that will not '
+                    'be used: %s' % self.nodes[0].get('uuid')
+                ],
+                'errors': [
+                    'Error: only 0 of 1 requested ironic nodes are tagged to '
+                    'profile control (for flavor control).\n'
+                    'Recommendation: tag more nodes using openstack baremetal '
+                    'node set --property "capabilities=profile:control,'
+                    'boot_option:local" <NODE ID>',
+                    'Error: only 0 of 1 requested ironic nodes are tagged to '
+                    'profile compute (for flavor compute).\n'
+                    'Recommendation: tag more nodes using openstack baremetal '
+                    'node set --property "capabilities=profile:compute,'
+                    'boot_option:local" <NODE ID>'
+                ]
+            })
+
+        action = validations.VerifyProfilesAction(self.nodes, self.flavors)
+        result = action.run()
+        self.assertEqual(expected.error['errors'].sort(),
+                         result.error['errors'].sort())
+        self.assertEqual(expected.error['warnings'], result.error['warnings'])
+        self.assertEqual(None, result.data)
+
+    def test_no_spurious_warnings(self):
+        self.nodes[:] = [self._get_fake_node(profile=None)]
+        self.flavors = {'baremetal': (
+            self._get_fake_flavor('baremetal', None), 1)}
+        expected = mistral_workflow_utils.Result(
+            error={
+                'warnings': [
+                    'There are 1 ironic nodes with no profile that will not '
+                    'be used: %s' % self.nodes[0].get('uuid')
+                ],
+                'errors': [
+                    'Error: only 0 of 1 requested ironic nodes are tagged to '
+                    'profile baremetal (for flavor baremetal).\n'
+                    'Recommendation: tag more nodes using openstack baremetal '
+                    'node set --property "capabilities=profile:baremetal,'
+                    'boot_option:local" <NODE ID>'
+                ]
+            })
+        self._test(expected)

workbooks/validations.yaml

@@ -449,3 +449,71 @@ workflows:
                 warnings: <% $.warnings %>
         on-success:
           - fail: <% $.get('status') = "FAILED" %>
+
+  verify_profiles:
+    input:
+      - flavors: []
+      - run_validations: true
+      - queue_name: tripleo
+    output:
+      errors: <% $.errors %>
+      warnings: <% $.warnings %>
+
+    tasks:
+      check_run_validations:
+        on-complete:
+          - get_ironic_nodes: <% $.run_validations %>
+          - send_message: <% not $.run_validations %>
+
+      get_ironic_nodes:
+        action: ironic.node_list
+        on-success: verify_profiles
+        on-error: failed_get_ironic_nodes
+        input:
+          maintenance: false
+          detail: true
+        publish:
+          nodes: <% task(get_ironic_nodes).result %>
+
+      failed_get_ironic_nodes:
+        on-success: send_message
+        publish:
+          status: FAILED
+          message: <% task(get_ironic_nodes).result %>
+
+      verify_profiles:
+        action: tripleo.validations.verify_profiles
+        input:
+          nodes: <% $.nodes %>
+          flavors: <% $.flavors %>
+        on-success: send_message
+        on-error: fail_verify_profiles
+        publish:
+          errors: <% task(verify_profiles).result.errors %>
+          warnings: <% task(verify_profiles).result.warnings %>
+        publish-on-error:
+          errors: <% task(verify_profiles).result.errors %>
+          warnings: <% task(verify_profiles).result.warnings %>
+
+      fail_verify_profiles:
+        on-success: send_message
+        publish:
+          status: Failed
+          message: <% task(verify_profiles).result %>
+
+      send_message:
+        action: zaqar.queue_post
+        retry: count=5 delay=1
+        input:
+          queue_name: <% $.queue_name %>
+          messages:
+            body:
+              type: tripleo.validations.v1.verify_profiles
+              payload:
+                status: <% $.get('status', 'SUCCESS') %>
+                message: <% $.get('message', '') %>
+                execution: <% execution() %>
+                errors: <% $.errors %>
+                warnings: <% $.warnings %>
+        on-success:
+          - fail: <% $.get('status') = "FAILED" %>