Add test for single invocation of gd tasks

Add support of api:
* get tasks for cluster
* get tasks for release
* get tasks ended with
* put task for cluster
* put tasks for release

Add version of base test for granular
deployment:
* Step 1 - deploy master node, create 2 node cluster,
  provision cnodes
* Step 2 Run task end with host
* Step 3 -8- run controller specific task
* Step 9 - run compute task
* Step 10 - run cinder task
* Step 11 - run post -deployment

The same in second test covers ha with 3 controller,
mongo and ceph

Change-Id: Idece933217b442bbe7353ce99eb0280c8b562952
Implements: granular-deployment-testing

fuelweb_test/helpers/granular_deployment_checkers.py

@@ -0,0 +1,103 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import time
+
+
+from proboscis.asserts import assert_equal
+from proboscis.asserts import assert_true
+
+from fuelweb_test import logger
+
+
+def check_hiera_resources(remote, file_name=None):
+    cmd_sh = 'if [ -d /etc/hiera ] ; then echo "fine" ;  fi'
+    output = ''.join(remote.execute(cmd_sh)['stdout'])
+    assert_true('fine' in output, output)
+    if not file_name:
+        output_f = ''.join(remote.execute(
+            'if [ -r /etc/hiera.yaml ] ; then echo "passed" ;  fi')[
+            'stdout'])
+        assert_true('passed' in output_f, output_f)
+    else:
+        output_f = ''.join(remote.execute(
+            'if [ -r /etc/%s ] ; then echo "passed" ;  fi' % file_name)[
+            'stdout'])
+        assert_true('passed' in output_f,
+                    'Can not find passed result in '
+                    'output {0}'.format(output_f))
+
+
+def get_hiera_data(remote, data):
+    cmd = 'hiera {}'.format(data)
+    res = remote.execute(cmd)['stdout']
+    return res
+
+
+def check_interface_status(remote, iname):
+    cmd = 'ethtools {0}| grep "Link detected"'.format(iname)
+    result = remote.execute(cmd)
+    assert_equal(0, result['exit_code'],
+                 "Non-zero exit code sderr {0}, "
+                 "stdout {1}".format(result['stderr'], result['stdout']))
+
+    assert_true('yes' in ''.join(result['stdout']),
+                "No link detected for interface {0},"
+                " Actual stdout {1}".format(iname, result['stdout']))
+
+
+def ping_remote_net(remote, ip):
+    cmd = "ping -q -c1 -w10 {0}".format(ip)
+    res = remote.execute(cmd)
+    logger.debug('Current res from ping is {0}'.format(res))
+    assert_equal(
+        res['exit_code'], 0,
+        "Ping of {0} ended with non zero exit-code. "
+        "Stdout is {1}, sderr {2}".format(
+            ip, ''.join(res['stdout']), ''.join(res['stderr'])))
+
+
+def check_logging_task(remote, conf_name):
+    cmd_sh = 'if [ -r /rsyslog.d/{0}] ; then echo "fine" ;  fi'.format(
+        conf_name)
+    output = ''.join(remote.execute(cmd_sh)['stdout'])
+    assert_true('fine' in output, output)
+
+
+def check_tools_task(remote, tool_name):
+    cmd_sh = 'pgrep {0}'.format(tool_name)
+    output = remote.execute(cmd_sh)
+    assert_equal(
+        0, output['exit_code'],
+        "Command {0} failed with non zero exit code, current output is:"
+        " stdout {1}, sdterr: {2} ".format(
+            cmd_sh, ''.join(output['stdout']), ''.join(output['stderr'])))
+
+
+def run_check_from_task(remote, path):
+    res = remote.execute('{0}'.format(path))
+    try:
+        assert_equal(
+            0, res['exit_code'],
+            "Check {0} finishes with non zero exit code, stderr is {1}, "
+            "stdout is {2} on remote".format(
+                path, res['stderr'], res['stdout']))
+    except AssertionError:
+        time.sleep(60)
+        logger.info('remoote is {0}'.format(remote))
+        res = remote.execute('{0}'.format(path))
+        assert_equal(
+            0, res['exit_code'],
+            "Check {0} finishes with non zero exit code, stderr is {1}, "
+            "stdout is {2} on remote".format(
+                path, res['stderr'], res['stdout']))

fuelweb_test/models/fuel_web_client.py

@@ -1559,13 +1559,13 @@ class FuelWebClient(object):
                 self.client.assign_nodegroup(ngroup_id, node_groups[ngroup])
 
     @logwrap
-    def get_nailgun_primary_controller(self, slave):
+    def get_nailgun_primary_node(self, slave, role='primary-controller'):
-        # returns controller that is primary in nailgun
+        # returns controller or mongo that is primary in nailgun
         remote = self.get_ssh_for_node(slave.name)
         data = yaml.load(''.join(
             remote.execute('cat /etc/astute.yaml')['stdout']))
         node_name = [node['fqdn'] for node in data['nodes']
-                     if node['role'] == 'primary-controller'][0]
+                     if node['role'] == role][0]
         logger.debug("node name is {0}".format(node_name))
         fqdn = self.get_fqdn_by_hostname(node_name)
         devops_node = self.find_devops_node_by_nailgun_fqdn(

fuelweb_test/models/nailgun_client.py

@@ -384,3 +384,56 @@ class NailgunClient(object):
         if user_email:
             settings['settings']['statistics']['email']['value'] = user_email
         self.update_settings(data=settings)
+
+    @logwrap
+    @json_parse
+    def get_cluster_deployment_tasks(self, cluster_id):
+        """ Get list of all deployment tasks for cluster."""
+        return self.client.get(
+            '/api/clusters/{}/deployment_tasks'.format(cluster_id))
+
+    @logwrap
+    @json_parse
+    def get_release_deployment_tasks(self, release_id):
+        """ Get list of all deployment tasks for release."""
+        return self.client.get(
+            '/api/releases/{}/deployment_tasks'.format(release_id))
+
+    @logwrap
+    @json_parse
+    def get_end_deployment_tasks(self, cluster_id, end, start=None):
+        """ Get list of all deployment tasks for cluster with end parameter.
+        If  end=netconfig, return all tasks from the graph included netconfig
+        """
+        if not start:
+            return self.client.get(
+                '/api/clusters/{0}/deployment_tasks?end={1}'.format(
+                    cluster_id, end))
+        return self.client.get(
+            '/api/clusters/{0}/deployment_tasks?start={1}&end={2}'.format(
+                cluster_id, start, end))
+
+    @logwrap
+    @json_parse
+    def get_orchestrator_deployment_info(self, cluster_id):
+        return self.client.get(
+            '/api/clusters/{}/orchestrator/deployment'.format(cluster_id))
+
+    @logwrap
+    @json_parse
+    def put_deployment_tasks_for_cluster(self, cluster_id, data, node_id):
+        """ Put  task to be executed on the nodes from cluster.:
+        Params:
+        cluster_id : Cluster id,
+        node_id: Node ids where task should be run, can be node_id=1,
+        or node_id =1,2,3,
+        data: tasks ids"""
+        return self.client.put(
+            '/api/clusters/{0}/deploy_tasks?nodes={1}'.format(
+                cluster_id, node_id), data)
+
+    @logwrap
+    @json_parse
+    def put_deployment_tasks_for_release(self, release_id, data):
+        return self.client.put(
+            '/api/releases/{}/deployment_tasks'.format(release_id), data)

fuelweb_test/run_tests.py

@@ -41,11 +41,15 @@ def import_tests():
     from tests.plugins.plugin_lbaas import test_plugin_lbaas  # noqa
     from tests.plugins.plugin_reboot import test_plugin_reboot_task  # noqa
     from tests import test_multiple_networks  # noqa
+    from tests.gd_based_tests import test_neutron  # noqa
+    from tests.gd_based_tests import test_neutron_vlan_ceph_mongo  # noqa
     from tests.tests_patching import test_patching  # noqa
 
 
 def run_tests():
     from proboscis import TestProgram  # noqa
+    import_tests()
+
     # Run Proboscis and exit.
     TestProgram().run_and_exit()
 

fuelweb_test/tests/gd_based_tests/test_neutron.py

@@ -0,0 +1,833 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import sys
+
+
+from proboscis.asserts import assert_true
+from proboscis import test
+from proboscis import SkipTest
+
+
+from fuelweb_test import logger
+from fuelweb_test.helpers.decorators import log_snapshot_on_error
+from fuelweb_test.helpers.decorators import upload_manifests
+from fuelweb_test.helpers import granular_deployment_checkers as gd
+from fuelweb_test.settings import DEPLOYMENT_MODE
+from fuelweb_test.settings import UPLOAD_MANIFESTS
+from fuelweb_test.tests.base_test_case import SetupEnvironment
+from fuelweb_test.tests.base_test_case import TestBasic
+import time
+
+
+@test(groups=["gd", "gd_deploy_neutron_gre"])
+class NeutronGre(TestBasic):
+    @classmethod
+    def get_pre_test(cls, tasks, task_name):
+        return [task['test_pre'] for task in tasks
+                if task['id'] == task_name and 'test_pre' in task]
+
+    @classmethod
+    def get_post_test(cls, tasks, task_name):
+        return [task['test_post'] for task in tasks
+                if task['id'] == task_name and 'test_post' in task]
+
+    @upload_manifests
+    def check_run_by_group(self, snapshot_name, expected_group):
+        test_group = sys.argv[-1]
+        try:
+            self.check_run(snapshot_name=snapshot_name)
+        except SkipTest:
+            if expected_group in test_group:
+                pass
+            else:
+                raise
+
+    def sync_manifest_to_the_slaves(self, cluster_id, node_ids):
+        if UPLOAD_MANIFESTS:
+            task_sync = self.fuel_web.client.put_deployment_tasks_for_cluster(
+                cluster_id, data=['rsync_core_puppet'],
+                node_id=str(node_ids).strip('[]'))
+            self.fuel_web.assert_task_success(task=task_sync)
+            task_hiera = self.fuel_web.client.put_deployment_tasks_for_cluster(
+                cluster_id, data=['hiera'],
+                node_id=str(node_ids).strip('[]'))
+            self.fuel_web.assert_task_success(task=task_hiera)
+
+    def sync_time_on_slaves(self, cluster_id, node_ids):
+        task_sync = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['sync_time'],
+            node_id=str(node_ids).strip('[]'))
+        self.fuel_web.assert_task_success(task=task_sync)
+
+    @test(depends_on=[SetupEnvironment.prepare_slaves_3])
+    @log_snapshot_on_error
+    def step_1_create_3_node_cluster_and_provision_nodes(self):
+        """Create cluster with 3 node, provision it and create snapshot
+          Depends:
+          "Bootstrap 3 slave nodes"
+
+          Scenario:
+            1. Revert snapshot "ready_with_3_slaves"
+            2. Create cluster with neutron
+            3. Add 1 controller
+            4. Add 2 node with compute and cinder node
+            5. Run provisioning task on all nodes, assert it is ready
+            6. Create snapshot
+
+        Snapshot: "step_1_provision_3_nodes"
+        """
+        self.check_run("step_1_create_3_node_cluster")
+        self.env.revert_snapshot("ready_with_3_slaves")
+
+        segment_type = 'gre'
+        cluster_id = self.fuel_web.create_cluster(
+            name=self.__class__.__name__,
+            mode=DEPLOYMENT_MODE,
+            settings={
+                "net_provider": 'neutron',
+                "net_segment_type": segment_type,
+                'tenant': 'gd',
+                'user': 'gd',
+                'password': 'gd'
+            }
+        )
+        self.fuel_web.update_nodes(
+            cluster_id,
+            {
+                'slave-01': ['controller'],
+                'slave-02': ['compute', 'cinder'],
+                'slave-03': ['compute', 'cinder']
+            }
+        )
+
+        self.fuel_web.provisioning_cluster_wait(cluster_id)
+
+        self.env.make_snapshot('step_1_create_3_node_cluster')
+
+    @test(depends_on=[step_1_create_3_node_cluster_and_provision_nodes],
+          groups=['run_tasks_end_with_host'])
+    @log_snapshot_on_error
+    def run_tasks_end_with_hosts(self):
+        """Run tasks end with hosts all nodes of the cluster.
+          Depends:
+          "step_1_create_3_node_cluster"
+
+          Scenario:
+            1. Revert snapshot "step 1 create_3_node_cluster_provision"
+            2. Get cluster id
+            3. Get cluster task list
+            4. Execute tasks ended with host on all nodes
+            5. Assert task hiera
+            6. Assert task globals
+            7. Assert task tools
+            8. Assert task logging
+            9. Assert task netconfig
+            10. Assert task firewall and hosts
+            11. Create snapshot
+
+        Snapshot: "run_tasks_end_with_hosts"
+        """
+        self.check_run_by_group('run_tasks_end_with_hosts',
+                                'run_tasks_end_with_host')
+        self.env.revert_snapshot("step_1_create_3_node_cluster")
+
+        cluster_id = self.fuel_web.get_last_created_cluster()
+
+        # get task list:
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, end='hosts')
+
+        logger.debug('task list is {0}'.format(tasks))
+
+        data = [task['id'] for task in tasks]
+
+        for t in ['hiera', 'globals', 'netconfig', 'hosts']:
+            assert_true(t in data,
+                        message='Can not find task {0}'
+                                ' in task list {1}'.format(t, data))
+
+        nodes_ids = [n['id'] for n in
+                     self.fuel_web.client.list_cluster_nodes(cluster_id)]
+
+        task = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=data, node_id=str(nodes_ids).strip('[]'))
+
+        logger.debug('task info is {0}'.format(task))
+        self.fuel_web.assert_task_success(task)
+
+        task_tools = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['tools'], node_id=str(nodes_ids).strip('[]'))
+
+        self.fuel_web.assert_task_success(task_tools)
+
+        task_firewall = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['firewall'], node_id=str(nodes_ids).strip('[]'))
+
+        self.fuel_web.assert_task_success(task_firewall)
+
+        task_ntp = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['ntp-client'],
+            node_id=str(nodes_ids).strip('[]'))
+
+        self.fuel_web.assert_task_success(task_ntp)
+
+        task_dns = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['dns-client'],
+            node_id=str(nodes_ids).strip('[]'))
+
+        self.fuel_web.assert_task_success(task_dns)
+
+        all_tasks = self.fuel_web.client.get_cluster_deployment_tasks(
+            cluster_id)
+
+        nodes = ['slave-0{0}'.format(slave) for slave in xrange(1, 4)]
+
+        # check hiera
+
+        if self.get_post_test(tasks, 'hiera'):
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=self.get_post_test(tasks, 'hiera')[0]['cmd'])
+             for node in nodes]
+
+        # check globals
+
+        if self.get_post_test(tasks, 'globals'):
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=self.get_post_test(tasks, 'globals')[0]['cmd'])
+             for node in nodes]
+
+        # check netcondfig
+
+        if self.get_post_test(tasks, 'netconfig'):
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=self.get_post_test(tasks, 'netconfig')[0]['cmd'])
+             for node in nodes]
+
+        # check firewall
+
+        if self.get_post_test(all_tasks, 'firewall'):
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=self.get_post_test(all_tasks, 'firewall')[0]['cmd'])
+             for node in nodes]
+
+        # check hosts
+
+        if self.get_post_test(tasks, 'hosts'):
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=self.get_post_test(tasks, 'hosts')[0]['cmd'])
+             for node in nodes]
+
+        # check tools
+
+        if self.get_post_test(all_tasks, 'tools'):
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=self.get_post_test(all_tasks, 'tools')[0]['cmd'])
+             for node in nodes]
+         # check tools
+
+        self.env.make_snapshot('run_tasks_end_with_hosts')
+
+    @test(depends_on=[run_tasks_end_with_hosts],
+          groups=['cluster_controller'])
+    @log_snapshot_on_error
+    def step_3_run_cluster_controller(self):
+        """Execute cluster task on controller, create snapshot
+          Depends:
+          "run_tasks_end_with_hosts"
+
+          Scenario:
+            1. Revert snapshot "run_tasks_end_with_hosts"
+            2. Get cluster id
+            4. Execute cluster task on controller
+            5. Verify that task was finished with success.
+            6. Assert task execution
+            7. Create snapshot
+
+        Snapshot: "step_3_run_cluster_controller"
+        """
+        self.check_run_by_group('step_3_run_cluster_controller',
+                                'cluster_controller')
+
+        self.env.revert_snapshot("run_tasks_end_with_hosts")
+        cluster_id = self.fuel_web.get_last_created_cluster()
+        controller_id = [
+            n['id'] for n in
+            self.fuel_web.client.list_cluster_nodes(cluster_id)
+            if 'controller' in n['roles']]
+
+        self.sync_manifest_to_the_slaves(
+            cluster_id=cluster_id,
+            node_ids=controller_id)
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, end='cluster')
+
+        pre_cluster = self.get_pre_test(tasks, 'cluster')
+        post_cluster = self.get_post_test(tasks, 'cluster')
+        if pre_cluster:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=pre_cluster[0]['cmd'])
+             for node in ['slave-01']]
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['cluster'],
+            node_id='{0}'.format(controller_id[0]))
+
+        self.fuel_web.assert_task_success(task=res)
+        if post_cluster:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=post_cluster[0]['cmd'])
+             for node in ['slave-01']]
+        self.env.make_snapshot("step_3_run_cluster_controller")
+
+    @test(depends_on=[step_3_run_cluster_controller],
+          groups=['virtual_ips_controller'])
+    @log_snapshot_on_error
+    def step_4_run_virtual_ips_controller(self):
+        """Execute virtual_ips task on controller, create snapshot
+          Depends:
+          "step_3_run_cluster_controller"
+
+          Scenario:
+            1. Revert snapshot "step_3_run_cluster_controller"
+            2. Get cluster id
+            4. Execute virtual ips task on controller
+            5. Verify that task was finished with success.
+            6. Assert task execution
+            7. Create snapshot
+
+        Snapshot: "step_4_run_virtual_ips_controller"
+        """
+        self.check_run_by_group('step_4_run_virtual_ips_controller',
+                                'virtual_ips_controller')
+
+        self.env.revert_snapshot("step_3_run_cluster_controller")
+        cluster_id = self.fuel_web.get_last_created_cluster()
+        controller_id = [
+            n['id'] for n in
+            self.fuel_web.client.list_cluster_nodes(cluster_id)
+            if 'controller' in n['roles']]
+
+        self.sync_manifest_to_the_slaves(
+            cluster_id=cluster_id,
+            node_ids=controller_id)
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, end='virtual_ips')
+
+        pre_virtual_ips = self.get_pre_test(tasks, 'virtual_ips')
+        post_virtual_ips = self.get_post_test(tasks, 'virtual_ips')
+        if pre_virtual_ips:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=pre_virtual_ips[0]['cmd'])
+             for node in ['slave-01']]
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['virtual_ips'],
+            node_id='{0}'.format(controller_id[0]))
+
+        self.fuel_web.assert_task_success(task=res)
+
+        if post_virtual_ips:
+            try:
+                gd.run_check_from_task(
+                    remote=self.fuel_web.get_ssh_for_node('slave-01'),
+                    path=post_virtual_ips[0]['cmd'])
+            except AssertionError:
+                import time
+                time.sleep(60)
+                gd.run_check_from_task(
+                    remote=self.fuel_web.get_ssh_for_node('slave-01'),
+                    path=post_virtual_ips[0]['cmd'])
+
+        self.env.make_snapshot("step_4_run_virtual_ips_controller")
+
+    @test(depends_on=[step_4_run_virtual_ips_controller],
+          groups=['cluster_haproxy_controller'])
+    @log_snapshot_on_error
+    def step_5_run_cluster_haproxy_controller(self):
+        """Execute cluster-haproxy task on controller, create snapshot
+          Depends:
+          "Step 4 run virtual_ips"
+
+          Scenario:
+            1. Revert snapshot "step 4 run virtual ips controller"
+            2. Get cluster id
+            4. Execute cluster-haproxy task on controller
+            5. Verify that task was finished with success.
+            6. Assert task execution
+            7. Create snapshot
+
+        Snapshot: "step_5_run_cluster_haproxy_controller"
+        """
+        self.check_run_by_group('step_5_run_cluster_haproxy_controller',
+                                'cluster_haproxy_controller')
+        self.env.revert_snapshot("step_4_run_virtual_ips_controller")
+        cluster_id = self.fuel_web.get_last_created_cluster()
+        controller_id = [
+            n['id'] for n in
+            self.fuel_web.client.list_cluster_nodes(cluster_id)
+            if 'controller' in n['roles']]
+
+        self.sync_manifest_to_the_slaves(
+            cluster_id=cluster_id,
+            node_ids=controller_id)
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, end='cluster-haproxy')
+
+        pre_cluster_haproxy = self.get_pre_test(tasks, 'cluster-haproxy')
+        post_cluster_haproxy = self.get_post_test(tasks, 'cluster-haproxy')
+        if pre_cluster_haproxy:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=pre_cluster_haproxy[0]['cmd'])
+             for node in ['slave-01']]
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['cluster-haproxy'],
+            node_id='{0}'.format(controller_id[0]))
+        logger.debug('res info is {0}'.format(res))
+
+        self.fuel_web.assert_task_success(task=res)
+        if post_cluster_haproxy:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=post_cluster_haproxy[0]['cmd'])
+             for node in ['slave-01']]
+
+        self.env.make_snapshot("step_5_run_cluster_haproxy_controller")
+
+    @test(depends_on=[step_5_run_cluster_haproxy_controller],
+          groups=['openstack_haproxy_controller'])
+    @log_snapshot_on_error
+    def step_6_run_openstack_haproxy_controller(self):
+        """Execute openstack-haproxy task on controller, create snapshot
+          Depends:
+          "Step 5 run cluster-haproxy"
+
+          Scenario:
+            1. Revert snapshot "step 5 run cluster haproxy controller"
+            2. Get cluster id
+            4. Execute openstack-haproxy task on controller
+            5. Verify that task was finished with success.
+            6. Assert task execution
+            7. Create snapshot
+
+        Snapshot: "step_6_run_openstack_haproxy_controller"
+        """
+        self.check_run_by_group('step_6_run_openstack_haproxy_controller',
+                                'openstack_haproxy_controller')
+        self.env.revert_snapshot("step_5_run_cluster_haproxy_controller")
+        cluster_id = self.fuel_web.get_last_created_cluster()
+        controller_id = [
+            n['id'] for n in
+            self.fuel_web.client.list_cluster_nodes(cluster_id)
+            if 'controller' in n['roles']]
+
+        self.sync_manifest_to_the_slaves(
+            cluster_id=cluster_id,
+            node_ids=controller_id)
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, end='openstack-haproxy')
+
+        pre_openstack_haproxy = self.get_pre_test(tasks, 'openstack-haproxy')
+        post_openstack_haproxy = self.get_post_test(tasks, 'openstack-haproxy')
+        if pre_openstack_haproxy:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=pre_openstack_haproxy[0]['cmd'])
+             for node in ['slave-01']]
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['openstack-haproxy'],
+            node_id='{0}'.format(controller_id[0]))
+        logger.debug('res info is {0}'.format(res))
+        self.fuel_web.assert_task_success(task=res)
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, start='dns-server',
+            end='ntp-server')
+        logger.debug("task list for ntp {0}".format(tasks))
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=[task['id'] for task in tasks],
+            node_id='{0}'.format(controller_id[0]))
+        logger.debug('res info is {0}'.format(res))
+        self.fuel_web.assert_task_success(task=res)
+
+        for service in [task['id'] for task in tasks]:
+            if self.get_post_test(tasks, service):
+                [gd.run_check_from_task(
+                    remote=self.fuel_web.get_ssh_for_node(node),
+                    path=self.get_post_test(tasks, service)[0]['cmd'])
+                 for node in ['slave-01']]
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, end='openstack-controller')
+
+        logger.debug("task list for services {0}".format(tasks))
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['memcached', 'database', 'rabbitmq',
+                              'keystone', 'glance', 'openstack-cinder'],
+            node_id='{0}'.format(controller_id[0]))
+        logger.debug('res info is {0}'.format(res))
+        self.fuel_web.assert_task_success(task=res)
+        if post_openstack_haproxy:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=post_openstack_haproxy[0]['cmd'])
+             for node in ['slave-01']]
+        for service in ['memcached', 'openstack-cinder', 'database'
+                        'rabbitmq', 'keystone', 'glance']:
+            if self.get_post_test(tasks, service):
+                [gd.run_check_from_task(
+                    remote=self.fuel_web.get_ssh_for_node(node),
+                    path=self.get_post_test(tasks, service)[0]['cmd'])
+                 for node in ['slave-01']]
+
+        self.env.make_snapshot("step_6_run_openstack_haproxy_controller")
+
+    @test(depends_on=[step_6_run_openstack_haproxy_controller],
+          groups=['openstack_controller'])
+    @log_snapshot_on_error
+    def step_7_run_openstack_controller(self):
+        """Execute openstack-controller task on controller, create snapshot
+          Depends:
+          "Step 6 run openstack haproxy controller
+
+          Scenario:
+            1. Revert snapshot "step_6_run_openstack_haproxy_controller"
+            2. Get cluster id
+            4. Execute openstack-controller task on controller
+            5. Verify that task was finished with success.
+            6. Assert task execution
+            7. Create snapshot
+
+        Snapshot: "step_7_run_openstack_controller"
+        """
+        self.check_run_by_group('step_7_run_openstack_controller',
+                                'openstack_controller')
+        self.env.revert_snapshot("step_6_run_openstack_haproxy_controller")
+
+        cluster_id = self.fuel_web.get_last_created_cluster()
+        controller_id = [
+            n['id'] for n in
+            self.fuel_web.client.list_cluster_nodes(cluster_id)
+            if 'controller' in n['roles']]
+
+        self.sync_manifest_to_the_slaves(
+            cluster_id=cluster_id,
+            node_ids=controller_id)
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, end='openstack-controller')
+
+        pre_openstack_ctr = self.get_pre_test(tasks, 'openstack-controller')
+        post_openstack_ctr = self.get_post_test(tasks, 'openstack-controller')
+        if pre_openstack_ctr:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=pre_openstack_ctr[0]['cmd'])
+             for node in ['slave-01']]
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['openstack-controller'],
+            node_id='{0}'.format(controller_id[0]))
+        logger.debug('res info is {0}'.format(res))
+
+        self.fuel_web.assert_task_success(task=res)
+        if post_openstack_ctr:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=post_openstack_ctr[0]['cmd'])
+             for node in ['slave-01']]
+
+        self.env.make_snapshot("step_7_run_openstack_controller")
+
+    @test(depends_on=[step_7_run_openstack_controller],
+          groups=['controller_remaining_tasks'])
+    @log_snapshot_on_error
+    def step_8_run_controller_remaining_tasks(self):
+        """Execute controller_remaining_task task on controller
+          Depends:
+          "Step 7 run openstack controller
+
+          Scenario:
+            1. Revert snapshot "step_7_run_openstack_controller"
+            2. Get cluster id
+            4. Executecontroller_remaining_tasks on controller
+            5. Verify that task was finished with success.
+            6. Assert task execution
+            7. Create snapshot
+
+        Snapshot: "step_8_run_controller_remaining_tasks"
+        """
+        self.check_run_by_group('step_8_run_controller_remaining_tasks',
+                                'controller_remaining_tasks')
+        self.env.revert_snapshot("step_7_run_openstack_controller")
+        cluster_id = self.fuel_web.get_last_created_cluster()
+        controller_id = [
+            n['id'] for n in
+            self.fuel_web.client.list_cluster_nodes(cluster_id)
+            if 'controller' in n['roles']]
+
+        self.sync_manifest_to_the_slaves(
+            cluster_id=cluster_id,
+            node_ids=controller_id)
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, start='openstack-controller',
+            end='controller_remaining_tasks')
+        expected_task_list = ['heat', 'horizon', 'api-proxy', 'swift',
+                              'controller_remaining_tasks']
+
+        for task in expected_task_list:
+            assert_true(task in [t['id'] for t in tasks],
+                        'Can not find task {0}, '
+                        'current list {1}'.format(task, tasks))
+
+        pre_net = self.get_pre_test(tasks, 'openstack-network')
+        if pre_net:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=pre_net[0]['cmd'])
+             for node in ['slave-01']]
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=[task['id'] for task in tasks],
+            node_id='{0}'.format(controller_id[0]))
+        logger.debug('res info is {0}'.format(res))
+
+        self.fuel_web.assert_task_success(task=res)
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['openstack-network'],
+            node_id='{0}'.format(controller_id[0]))
+        logger.debug('res info is {0}'.format(res))
+
+        self.fuel_web.assert_task_success(task=res)
+
+        expected_task_list.append('opensstack-network')
+
+        for task in expected_task_list:
+            if self.get_post_test(tasks, task):
+                [gd.run_check_from_task(
+                    remote=self.fuel_web.get_ssh_for_node(node),
+                    path=self.get_post_test(tasks, task)[0]['cmd'])
+                 for node in ['slave-01']]
+        try:
+            self.fuel_web.run_ostf(cluster_id=cluster_id,
+                                   test_sets=['sanity'],
+                                   should_fail=1)
+        except AssertionError:
+            time.sleep(60)
+            self.fuel_web.run_ostf(cluster_id, test_sets=['sanity'],
+                                   should_fail=1)
+
+        self.env.make_snapshot("step_8_run_controller_remaining_tasks")
+
+    @test(depends_on=[step_8_run_controller_remaining_tasks],
+          groups=['top_role_compute'])
+    @log_snapshot_on_error
+    def step_9_run_top_role_compute(self):
+        """Execute top-role-compute task on computes, create snapshot
+          Depends:
+          "step_8_run_controller_remaining_task
+
+          Scenario:
+            1. Revert snapshot "step_8_run_controller_remaining_task"
+            2. Get cluster id
+            4. Execute top-role-compute task on computes
+            5. Verify that task was finished with success.
+            6. Assert task execution
+            7. Create snapshot
+
+        Snapshot: "step_9_run_top_role_compute"
+        """
+        self.check_run_by_group('step_9_run_top_role_compute',
+                                'top_role_compute')
+
+        self.env.revert_snapshot("step_8_run_controller_remaining_tasks")
+        cluster_id = self.fuel_web.get_last_created_cluster()
+        compute_ids = [
+            n['id'] for n in
+            self.fuel_web.client.list_cluster_nodes(cluster_id)
+            if 'compute' in n['roles']]
+
+        self.sync_manifest_to_the_slaves(
+            cluster_id=cluster_id,
+            node_ids=compute_ids)
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, end='post_deployment_end')
+
+        pre_top_compute = self.get_pre_test(tasks, 'top-role-compute')
+        post_top_compute = self.get_post_test(tasks, 'top-role-compute')
+        if pre_top_compute:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=pre_top_compute[0]['cmd'])
+             for node in ['slave-02', 'slave-03']]
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['top-role-compute'],
+            node_id='{0},{1}'.format(compute_ids[0], compute_ids[1]))
+        logger.debug('res info is {0}'.format(res))
+
+        self.fuel_web.assert_task_success(task=res)
+        if post_top_compute:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=post_top_compute[0]['cmd'])
+             for node in ['slave-02', 'slave-03']]
+
+        pre_net = self.get_pre_test(tasks, 'openstack-network-compute')
+        post_net = self.get_post_test(tasks, 'openstack-network-compute')
+        if pre_net:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=pre_net[0]['cmd'])
+             for node in ['slave-02', 'slave-03']]
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['openstack-network-compute'],
+            node_id='{0},{1}'.format(compute_ids[0], compute_ids[1]))
+        logger.debug('res info is {0}'.format(res))
+
+        self.fuel_web.assert_task_success(task=res)
+
+        if post_net:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=post_net[0]['cmd'])
+             for node in ['slave-02', 'slave-03']]
+
+        self.env.make_snapshot("step_9_run_top_role_compute")
+
+    @test(depends_on=[step_9_run_top_role_compute],
+          groups=['top_role_cinder'])
+    @log_snapshot_on_error
+    def step_10_run_top_role_cinder(self):
+        """Execute top-role-cinder task on cinders, create snapshot
+          Depends:
+          "Step 9 run_top_role_compute
+
+          Scenario:
+            1. Revert snapshot "step_9_run_top_role_compute"
+            2. Get cluster id
+            4. Execute top-role-cinder task on cinder nodes
+            5. Verify that task was finished with success.
+            6. Assert task execution
+            7. Create snapshot
+
+        Snapshot: "step_10_run_top_role_cinder"
+        """
+        self.check_run_by_group('step_10_run_top_role_cinder',
+                                'top_role_cinder')
+
+        self.env.revert_snapshot('step_9_run_top_role_compute')
+        cluster_id = self.fuel_web.get_last_created_cluster()
+        cinder_ids = [
+            n['id'] for n in
+            self.fuel_web.client.list_cluster_nodes(cluster_id)
+            if 'cinder' in n['roles']]
+
+        self.sync_manifest_to_the_slaves(
+            cluster_id=cluster_id,
+            node_ids=cinder_ids)
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, end='top-role-cinder')
+
+        pre_top_cinder = self.get_pre_test(tasks, 'top-role-cinder')
+        post_top_cinder = self.get_post_test(tasks, 'top-role-cinder')
+        if pre_top_cinder:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=pre_top_cinder[0]['cmd'])
+             for node in ['slave-02', 'slave-03']]
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=['top-role-cinder'],
+            node_id='{0},{1}'.format(cinder_ids[0], cinder_ids[1]))
+        logger.debug('res info is {0}'.format(res))
+
+        self.fuel_web.assert_task_success(task=res)
+        if post_top_cinder:
+            [gd.run_check_from_task(
+                remote=self.fuel_web.get_ssh_for_node(node),
+                path=post_top_cinder[0]['cmd'])
+             for node in ['slave-02', 'slave-03']]
+
+        self.env.make_snapshot("step_10_run_top_role_cinder")
+
+    @test(depends_on=[step_10_run_top_role_cinder],
+          groups=['post_deployment'])
+    @log_snapshot_on_error
+    def step_11_run_post_deployment(self):
+        """Execute post_deployment tasks on all nodes
+          Depends:
+          "Step 10 run top-role-cinder
+
+          Scenario:
+            1. Revert snapshot "step_10_run_top_role_cinder"
+            2. Get cluster id
+            3. Execute post_deployment tasks
+            4. Verify that task was finished with success.
+            5. Run ostf
+        """
+        self.check_run_by_group('step_11_run_post_deployment',
+                                'post_deployment')
+
+        self.env.revert_snapshot("step_10_run_top_role_cinder")
+        cluster_id = self.fuel_web.get_last_created_cluster()
+
+        tasks = self.fuel_web.client.get_end_deployment_tasks(
+            cluster_id, start='post_deployment_start',
+            end='post_deployment_end')
+        data = [task['id'] for task in tasks]
+        nodes_ids = [n['id'] for n in
+                     self.fuel_web.client.list_cluster_nodes(cluster_id)]
+        self.sync_manifest_to_the_slaves(
+            cluster_id=cluster_id,
+            node_ids=nodes_ids)
+
+        contr_ids = [n['id'] for n in
+                     self.fuel_web.client.list_cluster_nodes(cluster_id)
+                     if 'controller' in n['roles']]
+
+        res = self.fuel_web.client.put_deployment_tasks_for_cluster(
+            cluster_id, data=data,
+            node_id=str(contr_ids).strip('[]'))
+        logger.debug('res info is {0}'.format(res))
+
+        self.fuel_web.assert_task_success(task=res)
+        time.sleep(100)
+        self.fuel_web.run_ostf(cluster_id=cluster_id)
+
+        self.env.make_snapshot("step_11_run_post_deployment")

fuelweb_test/tests/test_ha.py

@@ -93,7 +93,7 @@ class TestHaVLAN(TestBasic):
             self.env.d_env.get_ssh_to_remote(_ip))
 
         self.fuel_web.verify_network(cluster_id)
-        devops_node = self.fuel_web.get_nailgun_primary_controller(
+        devops_node = self.fuel_web.get_nailgun_primary_node(
             self.env.d_env.nodes().slaves[0])
         logger.debug("devops node name is {0}".format(devops_node.name))
 
@@ -163,7 +163,7 @@ class TestHaFlat(TestBasic):
             os_conn, smiles_count=16, networks_count=1, timeout=300)
 
         self.fuel_web.verify_network(cluster_id)
-        devops_node = self.fuel_web.get_nailgun_primary_controller(
+        devops_node = self.fuel_web.get_nailgun_primary_node(
             self.env.d_env.nodes().slaves[0])
         logger.debug("devops node name is {0}".format(devops_node.name))
 
@@ -281,7 +281,7 @@ class TestHaFlatScalability(TestBasic):
             }
         )
         self.fuel_web.deploy_cluster_wait(cluster_id)
-        devops_node = self.fuel_web.get_nailgun_primary_controller(
+        devops_node = self.fuel_web.get_nailgun_primary_node(
             self.env.d_env.nodes().slaves[0])
         logger.debug("devops node name is {0}".format(devops_node.name))
 
@@ -300,7 +300,7 @@ class TestHaFlatScalability(TestBasic):
                 devops_node.name,
                 self.env.d_env.nodes().slaves[:3], [])
 
-        devops_node = self.fuel_web.get_nailgun_primary_controller(
+        devops_node = self.fuel_web.get_nailgun_primary_node(
             self.env.d_env.nodes().slaves[0])
         logger.debug("devops node name is {0}".format(devops_node.name))
 
@@ -327,7 +327,7 @@ class TestHaFlatScalability(TestBasic):
                           '\s+Started node', ret), 'vip public started')
 
         self.fuel_web.security.verify_firewall(cluster_id)
-        devops_node = self.fuel_web.get_nailgun_primary_controller(
+        devops_node = self.fuel_web.get_nailgun_primary_node(
             self.env.d_env.nodes().slaves[0])
         logger.debug("devops node name is {0}".format(devops_node.name))\
 

fuelweb_test/tests/test_neutron.py

@@ -193,7 +193,7 @@ class NeutronGreHa(TestBasic):
         assert_equal(str(cluster['net_provider']), 'neutron')
 
         self.fuel_web.verify_network(cluster_id)
-        devops_node = self.fuel_web.get_nailgun_primary_controller(
+        devops_node = self.fuel_web.get_nailgun_primary_node(
             self.env.d_env.nodes().slaves[0])
         logger.debug("devops node name is {0}".format(devops_node.name))
         _ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
@@ -324,7 +324,7 @@ class NeutronVlanHa(TestBasic):
             cluster_id, self.env.d_env.get_ssh_to_remote(_ip))
 
         self.fuel_web.verify_network(cluster_id)
-        devops_node = self.fuel_web.get_nailgun_primary_controller(
+        devops_node = self.fuel_web.get_nailgun_primary_node(
             self.env.d_env.nodes().slaves[0])
         logger.debug("devops node name is {0}".format(devops_node.name))
         _ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']

fuelweb_test/tests/tests_strength/test_huge_environments.py

@@ -169,7 +169,6 @@ class HugeHaNeutron(base_test_case.TestBasic):
             'images_ceph': True,
             'objects_ceph': True,
             'ceilometer': True,
-            'objects_ceph': True,
             'net_provider': 'neutron',
             'net_segment_type': 'gre',
             'tenant': 'haGreCephHugeScale',
@@ -248,7 +247,6 @@ class HugeHaNeutron(base_test_case.TestBasic):
             'volumes_ceph': True,
             'images_ceph': True,
             'volumes_lvm': False,
-            'ceilometer': True,
             'objects_ceph': True,
             'net_provider': 'neutron',
             'net_segment_type': 'vlan',