diff --git a/fuelweb_test/models/fuel_web_client.py b/fuelweb_test/models/fuel_web_client.py index 2470fd38d..60922ffc9 100644 --- a/fuelweb_test/models/fuel_web_client.py +++ b/fuelweb_test/models/fuel_web_client.py @@ -121,12 +121,13 @@ class FuelWebClient(object): assert_true(result) @logwrap - def assert_ostf_run(self, cluster_id, should_fail=0, timeout=15 * 60): + def assert_ostf_run(self, cluster_id, should_fail=0, + failed_test_name=None, timeout=15 * 60): set_result_list = self._ostf_test_wait(cluster_id, timeout) - + failed_tests_res = [] failed = 0 - failed_tests_names = [] + actual_failed_names = [] test_result = {} for set_result in set_result_list: @@ -137,19 +138,30 @@ class FuelWebClient(object): set_result['tests'] ) ) + + [actual_failed_names.append(test['name']) + for test in set_result['tests'] if test['status'] != 'success'] + [test_result.update({test['name']:test['status']}) for test in set_result['tests']] - [failed_tests_names.append({test['name']:test['message']}) + [failed_tests_res.append({test['name']:test['message']}) for test in set_result['tests'] if test['status'] != 'success'] logger.info('OSTF test statuses are : {0}'.format(test_result)) + if failed_test_name: + for test_name in failed_test_name: + assert_true(test_name in actual_failed_names, + 'WARNINg unexpected fail,' + 'expected {0} actual {1}'.format( + failed_test_name, actual_failed_names)) + assert_true( failed <= should_fail, 'Failed tests, fails: {} should fail:' ' {} failed tests name: {}' ''.format(failed, should_fail, - failed_tests_names)) + failed_tests_res)) def assert_release_state(self, release_name, state='available'): for release in self.client.get_releases(): @@ -397,8 +409,9 @@ class FuelWebClient(object): cluster_id, self.client.get_networks(cluster_id)['networks']) @logwrap - def run_ostf(self, cluster_id, test_sets=None, should_fail=0, - tests_must_be_passed=None, timeout=None): + def run_ostf(self, cluster_id, test_sets=None, + should_fail=0, tests_must_be_passed=None, + timeout=None, failed_test_name=None): test_sets = test_sets or ['smoke', 'sanity'] timeout = timeout or 30 * 60 self.client.ostf_run_tests(cluster_id, test_sets) @@ -408,9 +421,12 @@ class FuelWebClient(object): tests_must_be_passed, timeout) else: + logger.info('Try to run assert ostf with ' + 'expected fail name {0}'.format(failed_test_name)) self.assert_ostf_run( cluster_id, - should_fail=should_fail, timeout=timeout) + should_fail=should_fail, timeout=timeout, + failed_test_name=failed_test_name) @logwrap def run_single_ostf_test(self, cluster_id, diff --git a/fuelweb_test/tests/test_ceph.py b/fuelweb_test/tests/test_ceph.py index aae143ba1..4500c45f2 100644 --- a/fuelweb_test/tests/test_ceph.py +++ b/fuelweb_test/tests/test_ceph.py @@ -74,9 +74,7 @@ class CephCompact(TestBasic): check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) # Run ostf - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ceph_multinode_compact") @@ -129,9 +127,7 @@ class CephCompactWithCinder(TestBasic): check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) # Run ostf - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ceph_multinode_with_cinder") @@ -186,9 +182,7 @@ class CephHA(TestBasic): check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) # Run ostf - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ceph_ha") @@ -250,7 +244,8 @@ class CephRadosGW(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['smoke', 'sanity', 'platform_tests'], - should_fail=1) + should_fail=1, + failed_test_name=['Check stack autoscaling']) # Check the radosqw daemon is started radosgw_started = lambda: len(remote.check_call( diff --git a/fuelweb_test/tests/test_ha.py b/fuelweb_test/tests/test_ha.py index 46c51e28d..513c94b52 100644 --- a/fuelweb_test/tests/test_ha.py +++ b/fuelweb_test/tests/test_ha.py @@ -79,7 +79,8 @@ class TestHaVLAN(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity'], - should_fail=4) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_ha_vlan") @@ -132,7 +133,8 @@ class TestHaFlat(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity'], - should_fail=4) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_ha_flat") @@ -191,6 +193,7 @@ class TestHaFlatAddCompute(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - test_sets=['ha', 'smoke', 'sanity'], should_fail=4) + test_sets=['ha', 'smoke', 'sanity'], should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("ha_flat_add_compute") diff --git a/fuelweb_test/tests/test_neutron.py b/fuelweb_test/tests/test_neutron.py index 3e7a4b902..8362a63f1 100644 --- a/fuelweb_test/tests/test_neutron.py +++ b/fuelweb_test/tests/test_neutron.py @@ -86,7 +86,8 @@ class NeutronGre(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=2) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_neutron_gre") @@ -144,7 +145,8 @@ class NeutronVlan(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=2) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_neutron_vlan") @@ -204,7 +206,8 @@ class NeutronGreHa(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=4) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_neutron_gre_ha") @@ -264,6 +267,7 @@ class NeutronVlanHa(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=4) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_neutron_vlan_ha") diff --git a/fuelweb_test/tests/test_pullrequest.py b/fuelweb_test/tests/test_pullrequest.py index ef2fd5261..a29007361 100644 --- a/fuelweb_test/tests/test_pullrequest.py +++ b/fuelweb_test/tests/test_pullrequest.py @@ -73,5 +73,5 @@ class TestPullRequest(TestBasic): self.fuel_web.run_ostf( cluster_id=self.fuel_web.get_last_created_cluster(), - should_fail=4 + should_fail=1 ) diff --git a/fuelweb_test/tests/test_services.py b/fuelweb_test/tests/test_services.py index 39fc96efa..4556f1bee 100644 --- a/fuelweb_test/tests/test_services.py +++ b/fuelweb_test/tests/test_services.py @@ -279,7 +279,8 @@ class CeilometerSimple(TestBasic): # run ostf smoke and sanity self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=1) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) # verify if needed image exists LOGGER.debug('Check MD5 of image') @@ -308,7 +309,8 @@ class CeilometerSimple(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['platform_tests'], - should_fail=1, timeout=3500) + should_fail=1, timeout=3500, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_ceilometer_simple") @@ -440,7 +442,9 @@ class CeilometerSimpleMongo(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['platform_tests', 'ha'], - should_fail=2, timeout=3500) + should_fail=2, timeout=3500, + failed_test_name=['Create volume and attach it to instance', + 'Check stack autoscaling']) self.env.make_snapshot("deploy_ceilometer_ha_with_mongo") @@ -492,7 +496,9 @@ class CeilometerSimpleMongo(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['platform_tests'], - should_fail=2, timeout=3500) + should_fail=2, timeout=3500, + failed_test_name=['Create volume and attach it to instance', + 'Check stack autoscaling']) self.env.make_snapshot("deploy_ceilometer_simple_mulirole") @@ -545,6 +551,8 @@ class CeilometerSimpleMongo(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['platform_tests', 'ha'], - should_fail=2, timeout=3500) + should_fail=2, timeout=3500, + failed_test_name=['Create volume and attach it to instance', + 'Check stack autoscaling']) self.env.make_snapshot("deploy_ceilometer_ha_mulirole") diff --git a/fuelweb_test/tests/test_simple.py b/fuelweb_test/tests/test_simple.py index 329f3e346..8c8319f6d 100644 --- a/fuelweb_test/tests/test_simple.py +++ b/fuelweb_test/tests/test_simple.py @@ -117,7 +117,8 @@ class SimpleFlat(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=5) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_simple_flat") @@ -268,7 +269,8 @@ class SimpleFlat(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=5) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("simple_flat_add_compute") @@ -320,7 +322,8 @@ class SimpleVlan(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=5) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_simple_vlan") @@ -362,9 +365,7 @@ class MultiroleControllerCinder(TestBasic): self.fuel_web.verify_network(cluster_id) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("deploy_multirole_controller_cinder") @@ -407,9 +408,7 @@ class MultiroleComputeCinder(TestBasic): self.fuel_web.verify_network(cluster_id) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("deploy_multirole_compute_cinder") @@ -536,7 +535,8 @@ class FloatingIPs(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=5) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_floating_ips") @@ -581,9 +581,7 @@ class SimpleCinder(TestBasic): self.fuel_web.assert_cluster_ready( 'slave-01', smiles_count=6, networks_count=1, timeout=300) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("deploy_simple_cinder") diff --git a/fuelweb_test/tests/tests_strength/test_huge_environments.py b/fuelweb_test/tests/tests_strength/test_huge_environments.py index 6e5c751b6..eac1b670e 100644 --- a/fuelweb_test/tests/tests_strength/test_huge_environments.py +++ b/fuelweb_test/tests/tests_strength/test_huge_environments.py @@ -83,7 +83,8 @@ class HugeEnvironments(base_test_case.TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=2) + should_fail=1, + failed_test_name=['Check stack autoscaling']) @test(depends_on=[base_test_case.SetupEnvironment.prepare_release], groups=["nine_nodes_separate_roles"]) @@ -143,4 +144,5 @@ class HugeEnvironments(base_test_case.TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=2) + should_fail=1, + failed_test_name=['Check stack autoscaling']) diff --git a/fuelweb_test/tests/tests_strength/test_master_node_failover.py b/fuelweb_test/tests/tests_strength/test_master_node_failover.py index 125f4bbfb..59c58dc68 100644 --- a/fuelweb_test/tests/tests_strength/test_master_node_failover.py +++ b/fuelweb_test/tests/tests_strength/test_master_node_failover.py @@ -71,12 +71,13 @@ class DeploySimpleMasterNodeFail(base_test_case.TestBasic): logger.info('PASS DEPLOYMENT') self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=1) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) logger.info('PASS OSTF') logger.info('Get controller ip') controller_ip = self.fuel_web.get_nailgun_node_by_name( 'slave-01')['ip'] - logger.info('Destoy admin node...') + logger.info('Destroy admin node...') self.env.nodes().admin.destroy() logger.info('Admin node destroyed') diff --git a/fuelweb_test/tests/tests_strength/test_restart.py b/fuelweb_test/tests/tests_strength/test_restart.py index a7ebd0d0b..99b7ca65c 100644 --- a/fuelweb_test/tests/tests_strength/test_restart.py +++ b/fuelweb_test/tests/tests_strength/test_restart.py @@ -81,9 +81,7 @@ class CephRestart(TestBasic): self.fuel_web.warm_restart_nodes(self.env.nodes().slaves[:4]) check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) @test(groups=["thread_5", "ceph"]) @@ -138,9 +136,7 @@ class CephHARestart(TestBasic): check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) # Run ostf - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) # Destroy osd-node self.env.nodes().slaves[5].destroy() @@ -148,23 +144,19 @@ class CephHARestart(TestBasic): recovery_timeout=True) self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=4) + should_fail=0) # Destroy compute node self.env.nodes().slaves[4].destroy() check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01'), recovery_timeout=True) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) # Cold restart self.fuel_web.cold_restart_nodes(self.env.nodes().slaves[:4]) check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ceph_ha") @@ -212,4 +204,5 @@ class SimpleFlatRestart(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=5) + should_fail=1, + failed_test_name=['Create volume and attach it to instance'])