Add tests for idempotency

Implements blueprint: test-granular-task-idempotency
Co-Authored-By: Sergey Novikov <snovikov@mirantis.com>

Change-Id: I3483221a709823b620ef87a9212c2697bf7ed4ce

doc/base_tests.rst

@@ -337,6 +337,16 @@ Test Daemon Resource Allocation Control
 .. automodule:: fuelweb_test.tests.test_cgroups
   :members:
 
+Test LCM base
+-------------
+.. automodule:: fuelweb_test.tests.tests_lcm.base_lcm_test
+  :members:
+
+Test task idempotency
+---------------------
+.. automodule:: fuelweb_test.tests.tests_lcm.test_idempotency
+  :members:
+
 Gating tests
 ============
 

fuelweb_test/helpers/fuel_actions.py

@@ -14,17 +14,17 @@
 
 import os
 import re
-import yaml
+from six.moves import cStringIO
 
 from devops.helpers.helpers import wait
 from devops.models import DiskDevice
 from devops.models import Node
 from devops.models import Volume
 from proboscis.asserts import assert_equal
+import yaml
 
 from fuelweb_test import logger
 from fuelweb_test import logwrap
-
 from fuelweb_test.helpers.regenerate_repo import regenerate_centos_repo
 from fuelweb_test.helpers.regenerate_repo import regenerate_ubuntu_repo
 from fuelweb_test.helpers import replace_repos
@@ -324,6 +324,19 @@ class AdminActions(BaseActions):
         assert_equal(result['exit_code'], 0,
                      "Saving Fuel settings failed: {0}!".format(result))
 
+    @logwrap
+    def get_tasks_description(self, release=None):
+        """Get tasks description
+
+        :param release: a string with release name
+        :return: a dictionary of tasks description
+        """
+        if not release:
+            release = ''
+        cmd = "cat `find /etc/puppet/{} -name tasks.yaml`".format(release)
+        data = self.ssh_manager.execute_on_remote(self.admin_ip, cmd)
+        return yaml.load(cStringIO(''.join(data['stdout'])))
+
 
 class NailgunActions(BaseActions):
     """NailgunActions."""  # TODO documentation

fuelweb_test/tests/tests_lcm/base_lcm_test.py

@@ -0,0 +1,509 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import fileinput
+import os
+
+from devops.helpers.helpers import TimeoutError
+from proboscis import asserts
+from proboscis import test
+import yaml
+
+from fuelweb_test import logger
+from fuelweb_test.helpers.decorators import log_snapshot_after_test
+from fuelweb_test.helpers.ssh_manager import SSHManager
+from fuelweb_test.settings import NEUTRON
+from fuelweb_test.settings import DEPLOYMENT_MODE
+from fuelweb_test.settings import NEUTRON_SEGMENT
+from fuelweb_test.tests.base_test_case import SetupEnvironment
+from fuelweb_test.tests.base_test_case import TestBasic
+
+
+# NOTE: Setup yaml to work with puppet report
+def construct_ruby_object(loader, suffix, node):
+    """Define a specific constructor"""
+    return loader.construct_yaml_map(node)
+
+
+def construct_ruby_sym(loader, node):
+    """Define a specific multi constructor"""
+    return loader.construct_yaml_str(node)
+
+
+TASKS_BLACKLIST = [
+    "reboot_provisioned_nodes",
+    "hiera",
+    "configure_default_route",
+    "netconfig"]
+
+
+class DeprecatedFixture(Exception):
+    def __init__(self):
+        msg = ('Please update fixtires in the fuel-qa repo with '
+               'according to generated fixtures')
+        super(DeprecatedFixture, self).__init__(msg)
+
+
+class LCMTestBasic(TestBasic):
+    """LCMTestBasic."""  # TODO documentation
+
+    def __init__(self):
+        super(LCMTestBasic, self).__init__()
+        yaml.add_multi_constructor(u"!ruby/object:", construct_ruby_object)
+        yaml.add_constructor(u"!ruby/sym", construct_ruby_sym)
+
+    # FIXME: after implementation of the main functional of PROD-2510
+    @staticmethod
+    def get_nodes_tasks(node_id):
+        """
+        :param node_id: an integer number of node id
+        :return: a set of deployment tasks for corresponding node
+        """
+        tasks = set()
+        ssh = SSHManager()
+
+        result = ssh.execute_on_remote(ssh.admin_ip, "ls /var/log/astute")
+        filenames = [filename.strip() for filename in result['stdout']]
+
+        for filename in filenames:
+            ssh.download_from_remote(
+                ssh.admin_ip,
+                destination="/var/log/astute/{0}".format(filename),
+                target="/tmp/{0}".format(filename))
+
+        data = fileinput.FileInput(
+            files=["/tmp/{0}".format(filename) for filename in filenames],
+            openhook=fileinput.hook_compressed)
+        for line in data:
+            if "Task time summary" in line \
+                    and "node {}".format(node_id) in line:
+                # FIXME: define an exact search of task
+                task_name = line.split("Task time summary: ")[1].split()[0]
+                check = any([excluded_task in task_name
+                             for excluded_task in TASKS_BLACKLIST])
+                if check:
+                    continue
+                tasks.add(task_name)
+        return tasks
+
+    @staticmethod
+    def get_task_type(tasks, task_id):
+        """Get task type
+
+        :param tasks: a list of dictionaries with task description
+        :param task_id: a string, name of deployment task
+        :return: a string of task type or a boolean value "False"
+        """
+        for task in tasks:
+            if task.get('id', '') == task_id:
+                return task.get('type', False)
+        return False
+
+    @staticmethod
+    def get_puppet_report(node):
+        """Get puppet run report from corresponding node
+
+        :param node: a dictionary with node description
+        :return: a dictionary with puppet report data
+        """
+        ssh = SSHManager()
+        ip = node['ip']
+        report_file = "/var/lib/puppet/state/last_run_report.yaml"
+        asserts.assert_true(ssh.isfile_on_remote(ip, report_file),
+                            'File {!r} not found on node {!r}'
+                            .format(report_file, node['id']))
+        with ssh.open_on_remote(ip, report_file) as f:
+            data = yaml.load(f)
+        ssh.rm_rf_on_remote(ip, report_file)
+        return data
+
+    @staticmethod
+    def load_fixture(deployment_type, role):
+        """Load fixture for corresponding kind of deployment
+
+        :param deployment_type: a string, name of the deployment kind
+        :param role: a string, node role
+        :return: a dictionary with loaded fixture data
+        """
+        fixture_path = os.path.join(
+            os.path.dirname(__file__), "fixtures",
+            deployment_type, "{}.yaml".format(role))
+        with open(fixture_path) as f:
+            fixture = yaml.load(f)
+
+        default_attrs = {"no_puppet_run": False,
+                         "type": "puppet",
+                         "skip": []}
+
+        # NOTE: Populate fixture with default values
+        for task in fixture['tasks']:
+            task_name, task_attrs = task.items()[0]
+            if task_attrs is None:
+                task_attrs = {}
+
+            for default_attr, default_value in default_attrs.items():
+                if default_attr not in task_attrs:
+                    task_attrs[default_attr] = default_value
+
+            task[task_name] = task_attrs
+        return fixture
+
+    def get_fixture_relevance(self, actual_tasks, fixture):
+        """Get fixture relevance between actual deployment tasks
+           and tasks from fixture files
+
+        :param actual_tasks: a list of actual tasks
+        :param fixture: a dictionary with fixture data
+        :return: a tuple of task sets
+        """
+        actual_tasks = set(actual_tasks)
+        fixture_tasks = set([i.keys()[0] for i in fixture["tasks"]])
+        tasks_description = self.env.admin_actions.get_tasks_description()
+
+        extra_actual_tasks = actual_tasks.difference(fixture_tasks)
+        extra_fixture_tasks = fixture_tasks.difference(actual_tasks)
+
+        # NOTE: in ideal case we need to avoid tasks with wrong types
+        wrong_types = {}
+        for task in fixture["tasks"]:
+            task_name, attrs = task.items()[0]
+            expected_type = self.get_task_type(tasks_description, task_name)
+            if not expected_type:
+                logger.error("No type or no such task {!r}".format(task_name))
+            else:
+                if expected_type != attrs["type"]:
+                    wrong_types.update({task_name: expected_type})
+
+        logger.info("Actual tasks {}contain extra tasks: {}"
+                    .format("" if extra_actual_tasks else "don't ",
+                            extra_actual_tasks))
+        logger.info("Fixture tasks {}contain extra tasks: {}"
+                    .format("" if extra_fixture_tasks else "don't ",
+                            extra_fixture_tasks))
+
+        return extra_actual_tasks, extra_fixture_tasks, wrong_types
+
+    def check_extra_tasks(self, slave_nodes, deployment):
+        """Check existing extra tasks regarding to fixture and actual task
+           or tasks with a wrong type
+
+        :param slave_nodes: a list of nailgun nodes
+        :param deployment: a string, name of the deployment kind
+        :return: a list with nodes for which extra tasks regarding to fixture
+                 and actual task or tasks with a wrong type were found
+        """
+        result = {'extra_actual_tasks': {},
+                  'extra_fixture_tasks': {},
+                  'wrong_types': {},
+                  'failed_tasks': {}}
+        for node in slave_nodes:
+            node_roles = "_".join(sorted(node["roles"]))
+            node_ref = "{}_{}".format(node["id"], node_roles)
+            fixture = self.load_fixture(deployment, node_roles)
+            node_tasks = self.get_nodes_tasks(node["id"])
+            extra_actual_tasks, extra_fixture_tasks, wrong_types = \
+                self.get_fixture_relevance(node_tasks, fixture)
+            result['extra_actual_tasks'][node_ref] = extra_actual_tasks
+            result['extra_fixture_tasks'][node_ref] = extra_fixture_tasks
+            result['wrong_types'][node_ref] = wrong_types
+            result['failed_tasks'][node_ref] = \
+                extra_actual_tasks | \
+                extra_fixture_tasks | \
+                set([task for task in wrong_types.keys()])
+
+        logger.warning("Uncovered deployment tasks:\n{}"
+                       .format(yaml.dump(result, default_flow_style=False)))
+        failed_nodes = [node_refs
+                        for node_refs, failed_tasks in
+                        result['failed_tasks'].items()
+                        if failed_tasks]
+        return failed_nodes
+
+    def execute_task_on_node(self, task, node, cluster_id):
+        """Execute deployment task against the corresponding node
+
+        :param task: a string of task name
+        :param node: a dictionary with node description
+        :param cluster_id: an integer, number of cluster id
+        :return: None
+        """
+        try:
+            logger.info("Trying to execute {!r} task on node {!r}"
+                        .format(task, node['id']))
+            tsk = self.fuel_web.client.put_deployment_tasks_for_cluster(
+                cluster_id=cluster_id,
+                data=[task],
+                node_id=node['id'])
+            self.fuel_web.assert_task_success(tsk, timeout=30 * 60)
+        except (AssertionError, TimeoutError) as e:
+            logger.exception("Failed to run task {!r}\n"
+                             "Exception:\n{}".format(task, e))
+
+    def generate_fixture(self, node_refs, cluster_id, slave_nodes):
+        """Generate fixture with description of task idempotency
+
+        :param node_refs: a string, refs to nailgun node
+        :param cluster_id: an integer, number of cluster id
+        :param slave_nodes: a list of nailgun nodes
+        :return: None
+        """
+        result = {}
+        for node in slave_nodes:
+            node_roles = "_".join(sorted(node["roles"]))
+            node_ref = "{}_{}".format(node["id"], node_roles)
+            if node_ref not in node_refs:
+                logger.debug('Node {!r} was skipped because the current '
+                             'fixtures are actual for deployment tasks which '
+                             'are executed on this node'.format(node_ref))
+                continue
+            node_tasks = self.get_nodes_tasks(node["id"])
+            tasks_description = self.env.admin_actions.get_tasks_description()
+            tasks = []
+
+            for task in node_tasks:
+                task_type = self.get_task_type(tasks_description, task)
+                if task_type != "puppet":
+                    logger.info("Skip checking of {!r} task,it is not puppet"
+                                .format(task))
+                    tasks.append({task: {"type": task_type}})
+                    continue
+
+                self.execute_task_on_node(task, node, cluster_id)
+
+                try:
+                    report = self.get_puppet_report(node)
+                except AssertionError:
+                    # NOTE: in ideal case we need to avoid puppet
+                    # tasks with "no_puppet_run": True
+                    tasks.append({task: {"no_puppet_run": True}})
+                    msg = ("Unexpected no_puppet_run for task: {}"
+                           .format(task))
+                    logger.info(msg)
+                    continue
+
+                failed = False
+                task_resources = []
+
+                for res_name, res_stats in report['resource_statuses'].items():
+                    if res_stats['changed']:
+                        failed = True
+                        msg = ("Non-idempotent task {!r}, resource: {}"
+                               .format(task, res_name))
+                        logger.error(msg)
+                        task_resources.append(res_name)
+
+                if failed:
+                    tasks.append({
+                        task: {"skip": task_resources}
+                    })
+                else:
+                    tasks.append({
+                        task: None
+                    })
+                    logger.info(
+                        "Task {!r} on node {!r} was executed successfully"
+                        .format(task, node['id']))
+
+            result.update(
+                {
+                    node_ref: {
+                        "role": node_roles,
+                        "tasks": tasks
+                    }
+                }
+            )
+
+        logger.info("Generated fixture:\n{}"
+                    .format(yaml.dump(result, default_flow_style=False)))
+
+
+@test(groups=['deploy_lcm_environment'])
+class SetupLCMEnvironment(LCMTestBasic):
+    @test(depends_on=[SetupEnvironment.prepare_slaves_3],
+          groups=['lcm_deploy_1_ctrl_1_cmp_1_cinder'])
+    @log_snapshot_after_test
+    def lcm_deploy_1_ctrl_1_cmp_1_cinder(self):
+        """Create cluster with cinder
+
+          Scenario:
+            1. Revert snapshot "ready_with_3_slaves"
+            2. Create cluster
+            3. Add 1 controller
+            4. Add 1 compute node
+            5. Add 1 cinder node
+            6. Deploy cluster
+            7. Check extra deployment tasks
+            8. Generate fixtures
+
+        Snapshot: "lcm_deploy_1_ctrl_1_cmp_1_cinder"
+        """
+        deployment = '1_ctrl_1_cmp_1_cinder'
+        snapshotname = 'lcm_deploy_{}'.format(deployment)
+        self.check_run(snapshotname)
+        self.show_step(1)
+        self.env.revert_snapshot("ready_with_3_slaves")
+
+        self.show_step(2)
+        segment_type = NEUTRON_SEGMENT['tun']
+        cluster_id = self.fuel_web.create_cluster(
+            name=self.__class__.__name__,
+            mode=DEPLOYMENT_MODE,
+            settings={
+                "net_provider": NEUTRON,
+                "net_segment_type": segment_type
+            }
+        )
+        self.show_step(3)
+        self.show_step(4)
+        self.show_step(5)
+        self.fuel_web.update_nodes(
+            cluster_id,
+            {
+                'slave-01': ['controller'],
+                'slave-02': ['compute'],
+                'slave-03': ['cinder']
+            }
+        )
+
+        self.show_step(6)
+        self.fuel_web.deploy_cluster_wait(cluster_id)
+        self.show_step(7)
+        slave_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
+        node_refs = self.check_extra_tasks(slave_nodes, deployment)
+        if node_refs:
+            self.show_step(8)
+            self.generate_fixture(node_refs, cluster_id, slave_nodes)
+            raise DeprecatedFixture
+        self.env.make_snapshot(snapshotname, is_make=True)
+
+    @test(depends_on=[SetupEnvironment.prepare_slaves_3],
+          groups=['lcm_deploy_1_ctrl_1_cmp_1_mongo'])
+    @log_snapshot_after_test
+    def lcm_deploy_1_ctrl_1_cmp_1_mongo(self):
+        """Create cluster with Ceilometer
+
+          Scenario:
+            1. Revert snapshot "ready_with_3_slaves"
+            2. Create cluster
+            3. Add 1 controller
+            4. Add 1 compute node
+            5. Add 1 mongo node
+            6. Deploy cluster
+            7. Check extra deployment tasks
+            8. Generate fixtures
+
+        Snapshot: "lcm_deploy_1_ctrl_1_cmp_1_mongo"
+        """
+        deployment = '1_ctrl_1_cmp_1_mongo'
+        snapshotname = 'lcm_deploy_{}'.format(deployment)
+        self.check_run(snapshotname)
+        self.show_step(1)
+        self.env.revert_snapshot("ready_with_3_slaves")
+
+        self.show_step(2)
+        segment_type = NEUTRON_SEGMENT['vlan']
+        cluster_id = self.fuel_web.create_cluster(
+            name=self.__class__.__name__,
+            mode=DEPLOYMENT_MODE,
+            settings={
+                'ceilometer': True,
+                'net_provider': NEUTRON,
+                'net_segment_type': segment_type
+            }
+        )
+        self.show_step(3)
+        self.show_step(4)
+        self.show_step(5)
+        self.fuel_web.update_nodes(
+            cluster_id,
+            {
+                'slave-01': ['controller'],
+                'slave-02': ['compute'],
+                'slave-03': ['mongo']
+            }
+        )
+
+        self.show_step(6)
+        self.fuel_web.deploy_cluster_wait(cluster_id)
+        self.show_step(7)
+        slave_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
+        node_refs = self.check_extra_tasks(slave_nodes, deployment)
+        if node_refs:
+            self.show_step(8)
+            self.generate_fixture(node_refs, cluster_id, slave_nodes)
+            raise DeprecatedFixture
+        self.env.make_snapshot(snapshotname, is_make=True)
+
+    @test(depends_on=[SetupEnvironment.prepare_slaves_5],
+          groups=['lcm_deploy_1_ctrl_1_cmp_3_ceph'])
+    @log_snapshot_after_test
+    def lcm_deploy_1_ctrl_1_cmp_3_ceph(self):
+        """Create cluster with ceph
+
+          Scenario:
+            1. Revert snapshot "ready_with_5_slaves"
+            2. Create cluster
+            3. Add 1 controller
+            4. Add 1 compute node
+            5. Add 3 ceph-osd nodes
+            6. Deploy cluster
+            7. Check extra deployment tasks
+            8. Generate fixtures
+
+        Snapshot: "lcm_deploy_1_ctrl_1_cmp_3_ceph"
+        """
+        deployment = '1_ctrl_1_cmp_3_ceph'
+        snapshotname = 'lcm_deploy_{}'.format(deployment)
+        self.check_run(snapshotname)
+        self.show_step(1)
+        self.env.revert_snapshot("ready_with_5_slaves")
+
+        self.show_step(2)
+        segment_type = NEUTRON_SEGMENT['tun']
+        cluster_id = self.fuel_web.create_cluster(
+            name=self.__class__.__name__,
+            mode=DEPLOYMENT_MODE,
+            settings={
+                'volumes_lvm': False,
+                'volumes_ceph': True,
+                'images_ceph': True,
+                'objects_ceph': True,
+                'net_provider': NEUTRON,
+                'net_segment_type': segment_type
+            }
+        )
+        self.show_step(3)
+        self.show_step(4)
+        self.show_step(5)
+        self.fuel_web.update_nodes(
+            cluster_id,
+            {
+                'slave-01': ['controller'],
+                'slave-02': ['compute'],
+                'slave-03': ['ceph-osd'],
+                'slave-04': ['ceph-osd'],
+                'slave-05': ['ceph-osd']
+            }
+        )
+
+        self.show_step(6)
+        self.fuel_web.deploy_cluster_wait(cluster_id)
+        self.show_step(7)
+        slave_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
+        node_refs = self.check_extra_tasks(slave_nodes, deployment)
+        if node_refs:
+            self.show_step(8)
+            self.generate_fixture(node_refs, cluster_id, slave_nodes)
+            raise DeprecatedFixture
+        self.env.make_snapshot(snapshotname, is_make=True)

fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/cinder.yaml

@@ -0,0 +1,47 @@
+roles:
+  cinder
+tasks:
+  - update_hosts: null
+  - clear_nodes_info:
+      type: shell
+  - copy_keys_ceph:
+      type: copy_files
+  - globals: null
+  - fuel_pkgs: null
+  - tools: null
+  - enable_cinder_volume_service: null
+  - rsync_core_puppet:
+      type: sync
+  - cgroups: null
+  - upload_nodes_info:
+      type: skipped
+  - copy_keys:
+      type: copy_files
+  - override_configuration: null
+  - setup_repositories: null
+  - dns-client: null
+  - allocate_hugepages: null
+  - plugins_setup_repositories: null
+  - upload_provision_data:
+      type: false
+  - ssl-keys-saving: null
+  - upload_configuration:
+      type: upload_file
+  - firewall: null
+  - top-role-cinder:
+      skip:
+      - Service[cinder-volume]
+  - logging: null
+  - sync_time:
+      type: shell
+  - plugins_rsync:
+      no_puppet_run: true
+  - connectivity_tests: null
+  - configuration_symlink:
+      type: shell
+  - hosts: null
+  - copy_haproxy_keys:
+      type: copy_files
+  - ntp-client: null
+  - ssl-add-trust-chain: null
+  - reserved_ports: null

fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/compute.yaml

@@ -0,0 +1,66 @@
+roles:
+  compute
+tasks:
+  - update_hosts: null
+  - openstack-network-start:
+      type: skipped
+  - openstack-network-common-config: null
+  - clear_nodes_info:
+      type: shell
+  - openstack-network-agents-sriov: null
+  - copy_keys_ceph:
+      type: copy_files
+  - globals: null
+  - fuel_pkgs: null
+  - openstack-network-agents-l3: null
+  - openstack-network-agents-metadata: null
+  - tools: null
+  - rsync_core_puppet:
+      type: sync
+  - enable_nova_compute_service: null
+  - cgroups: null
+  - upload_nodes_info:
+      type: skipped
+  - copy_keys:
+      type: copy_files
+  - override_configuration: null
+  - setup_repositories: null
+  - dns-client: null
+  - openstack-network-plugins-l2: null
+  - allocate_hugepages: null
+  - plugins_setup_repositories: null
+  - upload_provision_data:
+      type: false
+  - ceph-compute:
+      no_puppet_run: true
+  - ssl-keys-saving: null
+  - sriov_iommu_check:
+      skip:
+      - Exec[sriov_iommu_check]
+  - openstack-network-end:
+      type: skipped
+  - ceilometer-compute:
+      no_puppet_run: true
+  - upload_configuration:
+      type: upload_file
+  - firewall: null
+  - logging: null
+  - top-role-compute:
+      skip:
+      - Notify[Module openstack_tasks cannot notify service nova-compute on packages
+        update]
+      - Service[nova-compute]
+  - sync_time:
+      type: shell
+  - openstack-network-compute-nova: null
+  - plugins_rsync:
+      no_puppet_run: true
+  - connectivity_tests: null
+  - configuration_symlink:
+      type: shell
+  - hosts: null
+  - copy_haproxy_keys:
+      type: copy_files
+  - ntp-client: null
+  - ssl-add-trust-chain: null
+  - reserved_ports: null

fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/controller.yaml

@@ -0,0 +1,188 @@
+roles:
+  controller
+tasks:
+  - ironic_post_swift_key:
+      type: shell
+  - openstack-haproxy-mysqld: null
+  - cinder-db: null
+  - dump_rabbitmq_definitions: null
+  - rsync_core_puppet:
+      type: sync
+  - ssl-dns-setup: null
+  - ceilometer-controller: null
+  - override_configuration: null
+  - ceilometer-keystone:
+      no_puppet_run: true
+  - nova-db: null
+  - workloads_collector_add: null
+  - primary-openstack-network-plugins-l2: null
+  - radosgw-keystone: null
+  - virtual_ips: null
+  - primary-dns-server: null
+  - openstack-haproxy-murano: null
+  - openstack-network-end:
+      type: skipped
+  - openstack-haproxy-radosgw: null
+  - openstack-haproxy-swift: null
+  - heat-db: null
+  - openstack-haproxy-neutron: null
+  - updatedb:
+      no_puppet_run: true
+  - ironic-db:
+      no_puppet_run: true
+  - plugins_rsync:
+      no_puppet_run: true
+  - ceilometer-radosgw-user:
+      no_puppet_run: true
+  - openstack-haproxy-keystone: null
+  - hosts: null
+  - primary-rabbitmq: null
+  - primary-cluster-haproxy: null
+  - openstack-network-routers: null
+  - reserved_ports: null
+  - controller_remaining_tasks: null
+  - glance-keystone: null
+  - openstack-haproxy-aodh: null
+  - murano-cfapi:
+      no_puppet_run: true
+  - vmware-vcenter:
+      no_puppet_run: true
+  - ironic-compute:
+      no_puppet_run: true
+  - primary-openstack-network-agents-metadata: null
+  - cinder-keystone: null
+  - copy_keys:
+      type: copy_files
+  - enable_rados:
+      no_puppet_run: true
+  - ntp-check: null
+  - aodh-db:
+      no_puppet_run: true
+  - disable_keystone_service_token: null
+  - umm: null
+  - memcached: null
+  - allocate_hugepages: null
+  - openrc-delete:
+      skip:
+      - File[/root/openrc]
+  - plugins_setup_repositories:
+      no_puppet_run: true
+  - sahara-keystone:
+      no_puppet_run: true
+  - openstack-haproxy-sahara: null
+  - ssl-keys-saving: null
+  - primary-cluster: null
+  - upload_cirros:
+      type: shell
+  - primary-keystone:
+      skip:
+      - File[/root/openrc]
+  - primary-openstack-network-agents-l3: null
+  - upload_configuration:
+      type: upload_file
+  - create-cinder-types: null
+  - neutron-keystone: null
+  - logging: null
+  - nova-keystone: null
+  - update_hosts: null
+  - ironic-keystone:
+      no_puppet_run: true
+  - connectivity_tests: null
+  - swift-storage: null
+  - primary-heat: null
+  - conntrackd: null
+  - sahara-db:
+      no_puppet_run: true
+  - horizon: null
+  - openstack-haproxy-ceilometer: null
+  - openstack-network-common-config: null
+  - firewall: null
+  - apache: null
+  - globals: null
+  - aodh-keystone:
+      no_puppet_run: true
+  - glance: null
+  - tools: null
+  - openstack-haproxy: null
+  - cgroups: null
+  - murano-cfapi-keystone:
+      no_puppet_run: true
+  - aodh:
+      no_puppet_run: true
+  - ceph_create_pools:
+      no_puppet_run: true
+  - openstack-haproxy-ironic:
+      no_puppet_run: true
+  - setup_repositories: null
+  - openstack-network-routers-ha:
+      no_puppet_run: true
+  - glance-db: null
+  - neutron-db: null
+  - ironic_upload_images:
+      type: shell
+  - swift-rebalance-cron: null
+  - primary-ceph-mon: null
+  - openstack-haproxy-stats: null
+  - ironic-api:
+      no_puppet_run: true
+  - primary-ceph-radosgw: null
+  - dns-client: null
+  - cluster-vrouter: null
+  - murano-rabbitmq:
+      no_puppet_run: true
+  - api-proxy: null
+  - cluster_health: null
+  - heat-keystone: null
+  - openstack-haproxy-horizon: null
+  - openstack-network-start:
+      type: skipped
+  - clear_nodes_info:
+      type: shell
+  - murano-db:
+      no_puppet_run: true
+  - copy_keys_ceph:
+      type: copy_files
+  - sahara:
+      no_puppet_run: true
+  - fuel_pkgs: null
+  - swift-keystone: null
+  - public_vip_ping: null
+  - upload_nodes_info:
+      type: skipped
+  - openstack-haproxy-glance: null
+  - murano:
+      no_puppet_run: true
+  - ceph_ready_check:
+      type: shell
+  - enable_quorum:
+      type: shell
+  - openstack-haproxy-nova: null
+  - upload_provision_data:
+      type: false
+  - openstack-network-server-config: null
+  - primary-database:
+      skip:
+      - File[/root/.my.cnf]
+  - vcenter_compute_zones_create:
+      type: shell
+  - openstack-haproxy-cinder: null
+  - ntp-server: null
+  - murano-keystone:
+      no_puppet_run: true
+  - primary-openstack-network-agents-dhcp: null
+  - openstack-haproxy-heat: null
+  - primary-openstack-controller: null
+  - openstack-cinder: null
+  - keystone-db:
+      skip:
+      - File[/root/.my.cnf]
+  - sync_time:
+      type: shell
+  - configuration_symlink:
+      type: shell
+  - openstack-network-server-nova: null
+  - copy_haproxy_keys:
+      type: copy_files
+  - primary-swift-proxy: null
+  - openstack-network-networks: null
+  - ssl-add-trust-chain: null

fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/compute.yaml

@@ -0,0 +1,66 @@
+roles:
+  compute
+tasks:
+  - update_hosts: null
+  - openstack-network-start:
+      type: skipped
+  - openstack-network-common-config: null
+  - clear_nodes_info:
+      type: shell
+  - openstack-network-agents-sriov: null
+  - copy_keys_ceph:
+      type: copy_files
+  - globals: null
+  - fuel_pkgs: null
+  - openstack-network-agents-l3: null
+  - openstack-network-agents-metadata: null
+  - tools: null
+  - rsync_core_puppet:
+      type: sync
+  - enable_nova_compute_service: null
+  - cgroups: null
+  - upload_nodes_info:
+      type: skipped
+  - copy_keys:
+      type: copy_files
+  - override_configuration: null
+  - setup_repositories: null
+  - dns-client: null
+  - openstack-network-plugins-l2: null
+  - allocate_hugepages: null
+  - plugins_setup_repositories:
+      no_puppet_run: true
+  - upload_provision_data:
+      type: false
+  - ceph-compute:
+      no_puppet_run: true
+  - ssl-keys-saving: null
+  - sriov_iommu_check:
+      skip:
+      - Exec[sriov_iommu_check]
+  - openstack-network-end:
+      type: skipped
+  - ceilometer-compute: null
+  - upload_configuration:
+      type: upload_file
+  - firewall: null
+  - logging: null
+  - top-role-compute:
+      skip:
+      - Notify[Module openstack_tasks cannot notify service nova-compute on packages
+        update]
+      - Service[nova-compute]
+  - sync_time:
+      type: shell
+  - openstack-network-compute-nova: null
+  - plugins_rsync:
+      no_puppet_run: true
+  - connectivity_tests: null
+  - configuration_symlink:
+      type: shell
+  - hosts: null
+  - copy_haproxy_keys:
+      type: copy_files
+  - ntp-client: null
+  - ssl-add-trust-chain: null
+  - reserved_ports: null

fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/controller.yaml

@@ -0,0 +1,183 @@
+roles:
+  controller
+tasks:
+  - ironic_post_swift_key:
+      type: shell
+  - openstack-haproxy-mysqld: null
+  - cinder-db: null
+  - dump_rabbitmq_definitions: null
+  - rsync_core_puppet:
+      type: sync
+  - ssl-dns-setup: null
+  - ceilometer-controller: null
+  - override_configuration: null
+  - ceilometer-keystone: null
+  - nova-db: null
+  - workloads_collector_add: null
+  - primary-openstack-network-plugins-l2: null
+  - radosgw-keystone: null
+  - virtual_ips: null
+  - primary-dns-server: null
+  - openstack-haproxy-murano: null
+  - openstack-network-end:
+      type: skipped
+  - openstack-haproxy-radosgw: null
+  - openstack-haproxy-swift: null
+  - heat-db: null
+  - openstack-haproxy-neutron: null
+  - updatedb: null
+  - ironic-db:
+      no_puppet_run: true
+  - plugins_rsync:
+      no_puppet_run: true
+  - ceilometer-radosgw-user:
+      no_puppet_run: true
+  - openstack-haproxy-keystone: null
+  - hosts: null
+  - primary-rabbitmq: null
+  - primary-cluster-haproxy: null
+  - openstack-network-routers: null
+  - reserved_ports: null
+  - controller_remaining_tasks: null
+  - glance-keystone: null
+  - openstack-haproxy-aodh: null
+  - murano-cfapi:
+      no_puppet_run: true
+  - vmware-vcenter:
+      no_puppet_run: true
+  - ironic-compute:
+      no_puppet_run: true
+  - primary-openstack-network-agents-metadata: null
+  - cinder-keystone: null
+  - copy_keys:
+      type: copy_files
+  - enable_rados:
+      no_puppet_run: true
+  - ntp-check: null
+  - aodh-db: null
+  - disable_keystone_service_token: null
+  - umm: null
+  - memcached: null
+  - allocate_hugepages: null
+  - openrc-delete:
+      skip:
+      - File[/root/openrc]
+  - plugins_setup_repositories:
+      no_puppet_run: true
+  - sahara-keystone:
+      no_puppet_run: true
+  - openstack-haproxy-sahara: null
+  - ssl-keys-saving: null
+  - primary-cluster: null
+  - upload_cirros:
+      type: shell
+  - primary-keystone:
+      skip:
+      - File[/root/openrc]
+  - primary-openstack-network-agents-l3: null
+  - upload_configuration:
+      type: upload_file
+  - create-cinder-types: null
+  - neutron-keystone: null
+  - logging: null
+  - nova-keystone: null
+  - update_hosts: null
+  - ironic-keystone:
+      no_puppet_run: true
+  - connectivity_tests: null
+  - swift-storage: null
+  - primary-heat: null
+  - conntrackd: null
+  - sahara-db:
+      no_puppet_run: true
+  - horizon: null
+  - openstack-haproxy-ceilometer: null
+  - openstack-network-common-config: null
+  - firewall: null
+  - apache: null
+  - globals: null
+  - aodh-keystone: null
+  - glance: null
+  - tools: null
+  - openstack-haproxy: null
+  - cgroups: null
+  - murano-cfapi-keystone:
+      no_puppet_run: true
+  - aodh: null
+  - ceph_create_pools:
+      no_puppet_run: true
+  - openstack-haproxy-ironic:
+      no_puppet_run: true
+  - setup_repositories: null
+  - openstack-network-routers-ha:
+      no_puppet_run: true
+  - glance-db: null
+  - neutron-db: null
+  - ironic_upload_images:
+      type: shell
+  - swift-rebalance-cron: null
+  - primary-ceph-mon: null
+  - openstack-haproxy-stats: null
+  - ironic-api:
+      no_puppet_run: true
+  - primary-ceph-radosgw: null
+  - dns-client: null
+  - cluster-vrouter: null
+  - murano-rabbitmq:
+      no_puppet_run: true
+  - api-proxy: null
+  - cluster_health: null
+  - heat-keystone: null
+  - openstack-haproxy-horizon: null
+  - openstack-network-start:
+      type: skipped
+  - clear_nodes_info:
+      type: shell
+  - murano-db:
+      no_puppet_run: true
+  - copy_keys_ceph:
+      type: copy_files
+  - sahara:
+      no_puppet_run: true
+  - fuel_pkgs: null
+  - swift-keystone: null
+  - public_vip_ping: null
+  - upload_nodes_info:
+      type: skipped
+  - openstack-haproxy-glance: null
+  - murano:
+      no_puppet_run: true
+  - ceph_ready_check:
+      type: shell
+  - enable_quorum:
+      type: shell
+  - openstack-haproxy-nova: null
+  - upload_provision_data:
+      type: false
+  - openstack-network-server-config: null
+  - primary-database:
+      skip:
+      - File[/root/.my.cnf]
+  - vcenter_compute_zones_create:
+      type: shell
+  - openstack-haproxy-cinder: null
+  - ntp-server: null
+  - murano-keystone:
+      no_puppet_run: true
+  - primary-openstack-network-agents-dhcp: null
+  - openstack-haproxy-heat: null
+  - primary-openstack-controller: null
+  - openstack-cinder: null
+  - keystone-db:
+      skip:
+      - File[/root/.my.cnf]
+  - sync_time:
+      type: shell
+  - configuration_symlink:
+      type: shell
+  - openstack-network-server-nova: null
+  - copy_haproxy_keys:
+      type: copy_files
+  - primary-swift-proxy: null
+  - openstack-network-networks: null
+  - ssl-add-trust-chain: null

fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/mongo.yaml

@@ -0,0 +1,44 @@
+roles:
+  mongo
+tasks:
+  - update_hosts: null
+  - clear_nodes_info:
+      type: shell
+  - top-role-primary-mongo: null
+  - copy_keys_ceph:
+      type: copy_files
+  - globals: null
+  - fuel_pkgs: null
+  - tools: null
+  - rsync_core_puppet:
+      type: sync
+  - cgroups: null
+  - upload_nodes_info:
+      type: skipped
+  - copy_keys:
+      type: copy_files
+  - override_configuration: null
+  - setup_repositories: null
+  - dns-client: null
+  - allocate_hugepages: null
+  - plugins_setup_repositories: null
+  - upload_provision_data:
+      type: false
+  - ssl-keys-saving: null
+  - upload_configuration:
+      type: upload_file
+  - firewall: null
+  - logging: null
+  - sync_time:
+      type: shell
+  - plugins_rsync:
+      no_puppet_run: true
+  - connectivity_tests: null
+  - configuration_symlink:
+      type: shell
+  - hosts: null
+  - copy_haproxy_keys:
+      type: copy_files
+  - ntp-client: null
+  - ssl-add-trust-chain: null
+  - reserved_ports: null

fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/ceph-osd.yaml

@@ -0,0 +1,45 @@
+roles:
+  ceph-osd
+tasks:
+  - update_hosts: null
+  - clear_nodes_info:
+      type: shell
+  - copy_keys_ceph:
+      type: copy_files
+  - globals: null
+  - fuel_pkgs: null
+  - tools: null
+  - rsync_core_puppet:
+      type: sync
+  - cgroups: null
+  - upload_nodes_info:
+      type: skipped
+  - copy_keys:
+      type: copy_files
+  - override_configuration: null
+  - setup_repositories: null
+  - dns-client: null
+  - allocate_hugepages: null
+  - plugins_setup_repositories: null
+  - upload_provision_data:
+      type: false
+  - ssl-keys-saving: null
+  - upload_configuration:
+      type: upload_file
+  - firewall: null
+  - top-role-ceph-osd: null
+  - logging: null
+  - updatedb: null
+  - sync_time:
+      type: shell
+  - plugins_rsync:
+      no_puppet_run: true
+  - connectivity_tests: null
+  - configuration_symlink:
+      type: shell
+  - hosts: null
+  - copy_haproxy_keys:
+      type: copy_files
+  - ntp-client: null
+  - ssl-add-trust-chain: null
+  - reserved_ports: null

fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/compute.yaml

@@ -0,0 +1,63 @@
+roles:
+  compute
+tasks:
+  - update_hosts: null
+  - openstack-network-start:
+      type: skipped
+  - openstack-network-common-config: null
+  - clear_nodes_info:
+      type: shell
+  - openstack-network-agents-sriov: null
+  - copy_keys_ceph:
+      type: copy_files
+  - globals: null
+  - fuel_pkgs: null
+  - openstack-network-agents-l3: null
+  - openstack-network-agents-metadata: null
+  - tools: null
+  - rsync_core_puppet:
+      type: sync
+  - enable_nova_compute_service: null
+  - cgroups: null
+  - upload_nodes_info:
+      type: skipped
+  - copy_keys:
+      type: copy_files
+  - override_configuration: null
+  - setup_repositories: null
+  - dns-client: null
+  - openstack-network-plugins-l2: null
+  - allocate_hugepages: null
+  - plugins_setup_repositories: null
+  - upload_provision_data:
+      type: false
+  - ceph-compute: null
+  - ssl-keys-saving: null
+  - sriov_iommu_check: null
+  - openstack-network-end:
+      type: skipped
+  - ceilometer-compute:
+      no_puppet_run: true
+  - upload_configuration:
+      type: upload_file
+  - firewall: null
+  - logging: null
+  - top-role-compute:
+      skip:
+      - Notify[Module openstack_tasks cannot notify service nova-compute on packages
+        update]
+      - Service[nova-compute]
+  - sync_time:
+      type: shell
+  - openstack-network-compute-nova: null
+  - plugins_rsync:
+      no_puppet_run: true
+  - connectivity_tests: null
+  - configuration_symlink:
+      type: shell
+  - hosts: null
+  - copy_haproxy_keys:
+      type: copy_files
+  - ntp-client: null
+  - ssl-add-trust-chain: null
+  - reserved_ports: null

fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/controller.yaml

@@ -0,0 +1,189 @@
+roles:
+  controller
+tasks:
+  - ironic_post_swift_key:
+      type: shell
+  - openstack-haproxy-mysqld: null
+  - cinder-db: null
+  - dump_rabbitmq_definitions: null
+  - rsync_core_puppet:
+      type: sync
+  - ssl-dns-setup: null
+  - ceilometer-controller: null
+  - override_configuration: null
+  - ceilometer-keystone:
+      no_puppet_run: true
+  - nova-db: null
+  - workloads_collector_add: null
+  - primary-openstack-network-plugins-l2: null
+  - radosgw-keystone: null
+  - virtual_ips: null
+  - primary-dns-server: null
+  - openstack-haproxy-murano: null
+  - openstack-network-end:
+      type: skipped
+  - openstack-haproxy-radosgw: null
+  - openstack-haproxy-swift: null
+  - heat-db: null
+  - openstack-haproxy-neutron: null
+  - updatedb: null
+  - ironic-db:
+      no_puppet_run: true
+  - plugins_rsync:
+      no_puppet_run: true
+  - ceilometer-radosgw-user:
+      no_puppet_run: true
+  - openstack-haproxy-keystone: null
+  - hosts: null
+  - primary-rabbitmq: null
+  - primary-cluster-haproxy: null
+  - openstack-network-routers: null
+  - reserved_ports: null
+  - controller_remaining_tasks: null
+  - glance-keystone: null
+  - openstack-haproxy-aodh: null
+  - murano-cfapi:
+      no_puppet_run: true
+  - vmware-vcenter:
+      no_puppet_run: true
+  - ironic-compute:
+      no_puppet_run: true
+  - primary-openstack-network-agents-metadata: null
+  - cinder-keystone: null
+  - copy_keys:
+      type: copy_files
+  - enable_rados: null
+  - ntp-check: null
+  - aodh-db:
+      no_puppet_run: true
+  - disable_keystone_service_token: null
+  - umm: null
+  - memcached: null
+  - allocate_hugepages: null
+  - openrc-delete:
+      skip:
+      - File[/root/openrc]
+  - plugins_setup_repositories:
+      no_puppet_run: true
+  - sahara-keystone:
+      no_puppet_run: true
+  - openstack-haproxy-sahara: null
+  - ssl-keys-saving: null
+  - primary-cluster: null
+  - upload_cirros:
+      type: shell
+  - primary-keystone:
+      skip:
+      - File[/root/openrc]
+  - primary-openstack-network-agents-l3: null
+  - upload_configuration:
+      type: upload_file
+  - create-cinder-types: null
+  - neutron-keystone: null
+  - logging: null
+  - nova-keystone: null
+  - update_hosts: null
+  - ironic-keystone:
+      no_puppet_run: true
+  - connectivity_tests: null
+  - swift-storage:
+      no_puppet_run: true
+  - primary-heat: null
+  - conntrackd: null
+  - sahara-db:
+      no_puppet_run: true
+  - horizon: null
+  - openstack-haproxy-ceilometer: null
+  - openstack-network-common-config: null
+  - firewall: null
+  - apache: null
+  - globals: null
+  - aodh-keystone:
+      no_puppet_run: true
+  - glance: null
+  - tools: null
+  - openstack-haproxy: null
+  - cgroups: null
+  - murano-cfapi-keystone:
+      no_puppet_run: true
+  - aodh:
+      no_puppet_run: true
+  - ceph_create_pools: null
+  - openstack-haproxy-ironic:
+      no_puppet_run: true
+  - setup_repositories: null
+  - openstack-network-routers-ha:
+      no_puppet_run: true
+  - glance-db: null
+  - neutron-db: null
+  - ironic_upload_images:
+      type: shell
+  - swift-rebalance-cron:
+      no_puppet_run: true
+  - primary-ceph-mon: null
+  - openstack-haproxy-stats: null
+  - ironic-api:
+      no_puppet_run: true
+  - primary-ceph-radosgw: null
+  - dns-client: null
+  - cluster-vrouter: null
+  - murano-rabbitmq:
+      no_puppet_run: true
+  - api-proxy: null
+  - cluster_health: null
+  - heat-keystone: null
+  - openstack-haproxy-horizon: null
+  - openstack-network-start:
+      type: skipped
+  - clear_nodes_info:
+      type: shell
+  - murano-db:
+      no_puppet_run: true
+  - copy_keys_ceph:
+      type: copy_files
+  - sahara:
+      no_puppet_run: true
+  - fuel_pkgs: null
+  - swift-keystone:
+      no_puppet_run: true
+  - public_vip_ping: null
+  - upload_nodes_info:
+      type: skipped
+  - openstack-haproxy-glance: null
+  - murano:
+      no_puppet_run: true
+  - ceph_ready_check:
+      type: shell
+  - enable_quorum:
+      type: shell
+  - openstack-haproxy-nova: null
+  - upload_provision_data:
+      type: false
+  - openstack-network-server-config: null
+  - primary-database:
+      skip:
+      - File[/root/.my.cnf]
+  - vcenter_compute_zones_create:
+      type: shell
+  - openstack-haproxy-cinder: null
+  - ntp-server: null
+  - murano-keystone:
+      no_puppet_run: true
+  - primary-openstack-network-agents-dhcp: null
+  - openstack-haproxy-heat: null
+  - primary-openstack-controller: null
+  - openstack-cinder: null
+  - keystone-db:
+      skip:
+      - File[/root/.my.cnf]
+  - sync_time:
+      type: shell
+  - configuration_symlink:
+      type: shell
+  - openstack-network-server-nova: null
+  - copy_haproxy_keys:
+      type: copy_files
+  - primary-swift-proxy:
+      no_puppet_run: true
+  - openstack-network-networks: null
+  - ssl-add-trust-chain: null

fuelweb_test/tests/tests_lcm/test_idempotency.py

@@ -0,0 +1,162 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from proboscis import asserts
+from proboscis import test
+import yaml
+
+from fuelweb_test import logger
+from fuelweb_test.helpers.decorators import log_snapshot_after_test
+from fuelweb_test.tests.tests_lcm.base_lcm_test import SetupLCMEnvironment
+from fuelweb_test.tests.tests_lcm.base_lcm_test import LCMTestBasic
+
+
+@test(groups=['idempotency'])
+class TaskIdempotency(LCMTestBasic):
+    """TaskIdempotency."""  # TODO documentation
+
+    def check_idempotency(self, deployment):
+        """Check task idempotency for corresponding deployment
+
+        :param deployment: a string, name of the deployment kind
+        :return: a boolean, all tasks is idempotent - True,
+                 some task is not idempotent - False
+        """
+        idempotent = True
+        cluster_id = self.fuel_web.get_last_created_cluster()
+        slave_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
+
+        result = {'tasks_idempotency': {},
+                  'timeouterror_tasks': {}}
+
+        for node in slave_nodes:
+            node_roles = "_".join(sorted(node["roles"]))
+            node_ref = "{}_{}".format(node["id"], node_roles)
+            fixture = self.load_fixture(deployment, node_roles)
+
+            failed_tasks = {}
+            timeouterror_tasks = []
+
+            for task in fixture['tasks']:
+                task_name, fixture_task = task.items()[0]
+
+                if fixture_task['type'] != 'puppet':
+                    logger.info('Skip checking of {!r} task,it is not puppet'
+                                .format(task_name))
+                    continue
+
+                self.execute_task_on_node(task_name, node, cluster_id)
+
+                try:
+                    report = self.get_puppet_report(node)
+                except AssertionError:
+                    if not fixture_task.get('no_puppet_run'):
+                        msg = ('Unexpected no_puppet_run for task: {!r}'
+                               .format(task_name))
+                        logger.info(msg)
+                        timeouterror_tasks.append(task_name)
+                    continue
+
+                skip = fixture_task.get('skip')
+                failed = False
+                task_resources = []
+
+                for res_name, res_stats in report['resource_statuses'].items():
+                    if res_stats['changed'] and res_name not in skip:
+                        failed = True
+                        msg = ('Non-idempotent task {!r}, resource: {}'
+                               .format(task, res_name))
+                        logger.error(msg)
+                        task_resources.append(res_name)
+
+                if failed:
+                    idempotent = False
+                    failed_tasks.update({
+                        task_name: task_resources
+                    })
+                else:
+                    logger.info(
+                        'Task {!r} on node {!r} was executed successfully'
+                        .format(task_name, node['id']))
+
+            result['tasks_idempotency'][node_ref] = failed_tasks
+            result['timeouterror_tasks'][node_ref] = timeouterror_tasks
+
+        logger.warning('Non-idempotent tasks:\n{}'
+                       .format(yaml.dump(result, default_flow_style=False)))
+        return idempotent
+
+    @test(depends_on=[SetupLCMEnvironment.lcm_deploy_1_ctrl_1_cmp_1_cinder],
+          groups=['idempotency',
+                  'idempotency_1_ctrl_1_cmp_1_cinder'])
+    @log_snapshot_after_test
+    def idempotency_1_ctrl_1_cmp_1_cinder(self):
+        """Test idempotency for cluster with cinder
+
+          Scenario:
+            1. Revert snapshot "lcm_deploy_1_ctrl_1_cmp_1_cinder"
+            2. Check task idempotency
+
+        Snapshot: "idempotency_1_ctrl_1_cmp_1_cinder"
+        """
+        self.show_step(1)
+        deployment = "1_ctrl_1_cmp_1_cinder"
+        self.env.revert_snapshot('lcm_deploy_{}'.format(deployment))
+        self.show_step(2)
+        asserts.assert_true(self.check_idempotency(deployment),
+                            'There are non-idempotent tasks. '
+                            'Please take a look at the output above!')
+        self.env.make_snapshot('idempotency_{}'.format(deployment))
+
+    @test(depends_on=[SetupLCMEnvironment.lcm_deploy_1_ctrl_1_cmp_1_mongo],
+          groups=['idempotency', 'idempotency_1_ctrl_1_cmp_1_mongo'])
+    @log_snapshot_after_test
+    def idempotency_1_ctrl_1_cmp_1_mongo(self):
+        """Test idempotency for cluster with Ceilometer
+
+          Scenario:
+            1. Revert snapshot "lcm_deploy_1_ctrl_1_cmp_1_mongo"
+            2. Check task idempotency
+
+        Snapshot: "idempotency_1_ctrl_1_cmp_1_mongo"
+        """
+        self.show_step(1)
+        deployment = "1_ctrl_1_cmp_1_mongo"
+        self.env.revert_snapshot('lcm_deploy_{}'.format(deployment))
+        self.show_step(2)
+        asserts.assert_true(self.check_idempotency(deployment),
+                            'There are non-idempotent tasks. '
+                            'Please take a look at the output above!')
+        self.env.make_snapshot('idempotency_{}'.format(deployment))
+
+    @test(depends_on=[SetupLCMEnvironment.lcm_deploy_1_ctrl_1_cmp_3_ceph],
+          groups=['idempotency', 'idempotency_1_ctrl_1_cmp_3_ceph'])
+    @log_snapshot_after_test
+    def idempotency_1_ctrl_1_cmp_3_ceph(self):
+        """Test idempotency for cluster with Ceph
+
+          Scenario:
+            1. Revert snapshot "lcm_deploy_1_ctrl_1_cmp_3_ceph"
+            2. Check task idempotency
+
+        Snapshot: "idempotency_1_ctrl_1_cmp_3_ceph"
+        """
+        self.show_step(1)
+        deployment = "1_ctrl_1_cmp_3_ceph"
+        self.env.revert_snapshot('lcm_deploy_{}'.format(deployment))
+        self.show_step(2)
+        asserts.assert_true(self.check_idempotency(deployment),
+                            'There are non-idempotent tasks. '
+                            'Please take a look at the output above!')
+        self.env.make_snapshot('idempotency_{}'.format(deployment))