Add test task coverage by lcm tests

Implements: blueprint test-granular-task-idempotency
Change-Id: I49dabcf0e9b08eec602ce2a7f5102556df10e343

doc/base_tests.rst

@@ -366,6 +366,11 @@ Test task ensurability
 .. automodule:: fuelweb_test.tests.tests_lcm.test_ensurability
   :members:
 
+Test task coverage by LCM tests
+-------------------------------
+.. automodule:: fuelweb_test.tests.tests_lcm.test_task_coverage
+  :members:
+
 Gating tests
 ============
 

fuelweb_test/tests/tests_lcm/base_lcm_test.py

@@ -41,6 +41,7 @@ def construct_ruby_sym(loader, node):
 
 
 TASKS_BLACKLIST = [
+    "pre_hiera_config",
     "reboot_provisioned_nodes",
     "hiera",
     "configure_default_route",
@@ -151,7 +152,7 @@ class LCMTestBasic(TestBasic):
                      fixture is loaded
         :return: a dictionary with loaded fixture data
         """
-        subdir = "" if idmp else "ensurability"
+        subdir = "idempotency" if idmp else "ensurability"
         fixture_path = os.path.join(
             os.path.dirname(__file__), "fixtures",
             deployment_type, subdir, "{}.yaml".format(role))

fuelweb_test/tests/tests_lcm/test_task_coverage.py

@@ -0,0 +1,123 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from proboscis import asserts
+from proboscis import test
+import yaml
+
+from fuelweb_test import logger
+from fuelweb_test.helpers.decorators import log_snapshot_after_test
+from fuelweb_test.tests.base_test_case import SetupEnvironment
+from fuelweb_test.tests.base_test_case import TestBasic
+from fuelweb_test.tests.tests_lcm.base_lcm_test import TASKS_BLACKLIST
+
+
+EXCLUDED_TASKS_FROM_COVERAGE = [
+    "top-role-cinder-vmware",
+    "top-role-compute-vmware",
+    "generate_vms"
+]
+
+
+@test
+class TaskLCMCoverage(TestBasic):
+    """Test suite for verification of task coverage by LCM tests"""
+    @staticmethod
+    def _load_from_file(path, tasks):
+        """Load fixture from the corresponding yaml file
+
+        :param path: a string, a full path to fixture file
+        :return: a list of tasks
+        """
+        with open(path) as f:
+            fixture = yaml.load(f)
+        for task in fixture['tasks']:
+            task_name, task_attr = task.items()[0]
+            if task_attr is None or 'type' not in task_attr:
+                tasks.append(task_name)
+        return tasks
+
+    def load_tasks_fixture_file(self, path, subdir, tasks=None):
+        """Load task fixtures
+
+        :param path: a string, relative path to fixture directory
+        :param subdir: a string, indicates whether idempotency or ensurability
+                       fixture is uploaded
+        :param tasks: a list of taken into consideration tasks
+        :return: a list of tasks
+        """
+        if not tasks:
+            tasks = []
+        if os.path.isdir(path) and os.path.basename(path) == subdir:
+            for fl in os.listdir(path):
+                filepath = os.path.join(path, fl)
+                tasks = list(set(tasks))
+                tasks.extend(
+                    self._load_from_file(filepath, tasks) or [])
+        elif os.path.isdir(path):
+            for fl in os.listdir(path):
+                filepath = os.path.join(path, fl)
+                tasks.extend(
+                    self.load_tasks_fixture_file(
+                        filepath, subdir, tasks) or [])
+        return tasks
+
+    @test(depends_on=[SetupEnvironment.prepare_release],
+          groups=['task_idempotency_coverage'])
+    @log_snapshot_after_test
+    def task_idempotency_coverage(self):
+        """Setup master node with custom manifests
+
+          Scenario:
+            1. Revert snapshot "ready"
+            2. Download task graph
+            3. Download task from existing fixture files
+            4. Define coverage of fuel task by idempotency tests
+
+        Duration 60m
+        """
+        self.show_step(1)
+        self.env.revert_snapshot('ready')
+
+        self.show_step(2)
+        release_id = self.fuel_web.client.get_release_id()
+        deployment_tasks = self.fuel_web.client.get_release_deployment_tasks(
+            release_id
+        )
+        puppet_tasks = [task['id']
+                        for task in deployment_tasks
+                        if task['type'] == 'puppet']
+        puppet_tasks = set(puppet_tasks)
+
+        self.show_step(3)
+        path = os.path.join(os.path.dirname(__file__), "fixtures")
+        fixture_tasks = set(self.load_tasks_fixture_file(path, 'idempotency'))
+
+        self.show_step(4)
+        task_blacklist = (set(TASKS_BLACKLIST) |
+                          set(EXCLUDED_TASKS_FROM_COVERAGE))
+        general_tasks = puppet_tasks & fixture_tasks
+        extra_deployment_tasks = puppet_tasks - general_tasks - task_blacklist
+        extra_fixtures_tasks = fixture_tasks - general_tasks
+        if extra_fixtures_tasks:
+            logger.warning('There are extra fixture tasks which are not '
+                           ' included in the current deployment graph: '
+                           'list of tasks: {}'.format(extra_fixtures_tasks))
+        asserts.assert_equal(extra_deployment_tasks, set(),
+                             'There are new deployment tasks which '
+                             'appeared in the current deployment graph and '
+                             'are not included to test LCM fixtures: list '
+                             'of tasks: {}'.format(extra_deployment_tasks))