From 94e2a4bc823f347d64d7b50ff83fd2d3e9686ef7 Mon Sep 17 00:00:00 2001 From: TatyanaLeontovich Date: Mon, 17 Feb 2014 16:13:40 +0200 Subject: [PATCH] Change ostf expectation failure count According to we have connectivity on the slaves on CI now, change OSTF expected fail in the tests Also add verification that failed test has the same name as we exdpected Change-Id: Iea4cb8e49a5e55d237548c769ae701fadb793185 --- fuelweb_test/models/fuel_web_client.py | 32 ++++++++++++++----- fuelweb_test/tests/test_ceph.py | 15 +++------ fuelweb_test/tests/test_ha.py | 9 ++++-- fuelweb_test/tests/test_neutron.py | 12 ++++--- fuelweb_test/tests/test_pullrequest.py | 2 +- fuelweb_test/tests/test_services.py | 18 ++++++++--- fuelweb_test/tests/test_simple.py | 24 +++++++------- .../tests_strength/test_huge_environments.py | 6 ++-- .../test_master_node_failover.py | 5 +-- .../tests/tests_strength/test_restart.py | 21 ++++-------- 10 files changed, 82 insertions(+), 62 deletions(-) diff --git a/fuelweb_test/models/fuel_web_client.py b/fuelweb_test/models/fuel_web_client.py index 2470fd38d..60922ffc9 100644 --- a/fuelweb_test/models/fuel_web_client.py +++ b/fuelweb_test/models/fuel_web_client.py @@ -121,12 +121,13 @@ class FuelWebClient(object): assert_true(result) @logwrap - def assert_ostf_run(self, cluster_id, should_fail=0, timeout=15 * 60): + def assert_ostf_run(self, cluster_id, should_fail=0, + failed_test_name=None, timeout=15 * 60): set_result_list = self._ostf_test_wait(cluster_id, timeout) - + failed_tests_res = [] failed = 0 - failed_tests_names = [] + actual_failed_names = [] test_result = {} for set_result in set_result_list: @@ -137,19 +138,30 @@ class FuelWebClient(object): set_result['tests'] ) ) + + [actual_failed_names.append(test['name']) + for test in set_result['tests'] if test['status'] != 'success'] + [test_result.update({test['name']:test['status']}) for test in set_result['tests']] - [failed_tests_names.append({test['name']:test['message']}) + [failed_tests_res.append({test['name']:test['message']}) for test in set_result['tests'] if test['status'] != 'success'] logger.info('OSTF test statuses are : {0}'.format(test_result)) + if failed_test_name: + for test_name in failed_test_name: + assert_true(test_name in actual_failed_names, + 'WARNINg unexpected fail,' + 'expected {0} actual {1}'.format( + failed_test_name, actual_failed_names)) + assert_true( failed <= should_fail, 'Failed tests, fails: {} should fail:' ' {} failed tests name: {}' ''.format(failed, should_fail, - failed_tests_names)) + failed_tests_res)) def assert_release_state(self, release_name, state='available'): for release in self.client.get_releases(): @@ -397,8 +409,9 @@ class FuelWebClient(object): cluster_id, self.client.get_networks(cluster_id)['networks']) @logwrap - def run_ostf(self, cluster_id, test_sets=None, should_fail=0, - tests_must_be_passed=None, timeout=None): + def run_ostf(self, cluster_id, test_sets=None, + should_fail=0, tests_must_be_passed=None, + timeout=None, failed_test_name=None): test_sets = test_sets or ['smoke', 'sanity'] timeout = timeout or 30 * 60 self.client.ostf_run_tests(cluster_id, test_sets) @@ -408,9 +421,12 @@ class FuelWebClient(object): tests_must_be_passed, timeout) else: + logger.info('Try to run assert ostf with ' + 'expected fail name {0}'.format(failed_test_name)) self.assert_ostf_run( cluster_id, - should_fail=should_fail, timeout=timeout) + should_fail=should_fail, timeout=timeout, + failed_test_name=failed_test_name) @logwrap def run_single_ostf_test(self, cluster_id, diff --git a/fuelweb_test/tests/test_ceph.py b/fuelweb_test/tests/test_ceph.py index aae143ba1..4500c45f2 100644 --- a/fuelweb_test/tests/test_ceph.py +++ b/fuelweb_test/tests/test_ceph.py @@ -74,9 +74,7 @@ class CephCompact(TestBasic): check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) # Run ostf - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ceph_multinode_compact") @@ -129,9 +127,7 @@ class CephCompactWithCinder(TestBasic): check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) # Run ostf - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ceph_multinode_with_cinder") @@ -186,9 +182,7 @@ class CephHA(TestBasic): check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) # Run ostf - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ceph_ha") @@ -250,7 +244,8 @@ class CephRadosGW(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['smoke', 'sanity', 'platform_tests'], - should_fail=1) + should_fail=1, + failed_test_name=['Check stack autoscaling']) # Check the radosqw daemon is started radosgw_started = lambda: len(remote.check_call( diff --git a/fuelweb_test/tests/test_ha.py b/fuelweb_test/tests/test_ha.py index 46c51e28d..513c94b52 100644 --- a/fuelweb_test/tests/test_ha.py +++ b/fuelweb_test/tests/test_ha.py @@ -79,7 +79,8 @@ class TestHaVLAN(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity'], - should_fail=4) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_ha_vlan") @@ -132,7 +133,8 @@ class TestHaFlat(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity'], - should_fail=4) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_ha_flat") @@ -191,6 +193,7 @@ class TestHaFlatAddCompute(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - test_sets=['ha', 'smoke', 'sanity'], should_fail=4) + test_sets=['ha', 'smoke', 'sanity'], should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("ha_flat_add_compute") diff --git a/fuelweb_test/tests/test_neutron.py b/fuelweb_test/tests/test_neutron.py index 3e7a4b902..8362a63f1 100644 --- a/fuelweb_test/tests/test_neutron.py +++ b/fuelweb_test/tests/test_neutron.py @@ -86,7 +86,8 @@ class NeutronGre(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=2) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_neutron_gre") @@ -144,7 +145,8 @@ class NeutronVlan(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=2) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_neutron_vlan") @@ -204,7 +206,8 @@ class NeutronGreHa(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=4) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_neutron_gre_ha") @@ -264,6 +267,7 @@ class NeutronVlanHa(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=4) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_neutron_vlan_ha") diff --git a/fuelweb_test/tests/test_pullrequest.py b/fuelweb_test/tests/test_pullrequest.py index ef2fd5261..a29007361 100644 --- a/fuelweb_test/tests/test_pullrequest.py +++ b/fuelweb_test/tests/test_pullrequest.py @@ -73,5 +73,5 @@ class TestPullRequest(TestBasic): self.fuel_web.run_ostf( cluster_id=self.fuel_web.get_last_created_cluster(), - should_fail=4 + should_fail=1 ) diff --git a/fuelweb_test/tests/test_services.py b/fuelweb_test/tests/test_services.py index 39fc96efa..4556f1bee 100644 --- a/fuelweb_test/tests/test_services.py +++ b/fuelweb_test/tests/test_services.py @@ -279,7 +279,8 @@ class CeilometerSimple(TestBasic): # run ostf smoke and sanity self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=1) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) # verify if needed image exists LOGGER.debug('Check MD5 of image') @@ -308,7 +309,8 @@ class CeilometerSimple(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['platform_tests'], - should_fail=1, timeout=3500) + should_fail=1, timeout=3500, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_ceilometer_simple") @@ -440,7 +442,9 @@ class CeilometerSimpleMongo(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['platform_tests', 'ha'], - should_fail=2, timeout=3500) + should_fail=2, timeout=3500, + failed_test_name=['Create volume and attach it to instance', + 'Check stack autoscaling']) self.env.make_snapshot("deploy_ceilometer_ha_with_mongo") @@ -492,7 +496,9 @@ class CeilometerSimpleMongo(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['platform_tests'], - should_fail=2, timeout=3500) + should_fail=2, timeout=3500, + failed_test_name=['Create volume and attach it to instance', + 'Check stack autoscaling']) self.env.make_snapshot("deploy_ceilometer_simple_mulirole") @@ -545,6 +551,8 @@ class CeilometerSimpleMongo(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['platform_tests', 'ha'], - should_fail=2, timeout=3500) + should_fail=2, timeout=3500, + failed_test_name=['Create volume and attach it to instance', + 'Check stack autoscaling']) self.env.make_snapshot("deploy_ceilometer_ha_mulirole") diff --git a/fuelweb_test/tests/test_simple.py b/fuelweb_test/tests/test_simple.py index 329f3e346..8c8319f6d 100644 --- a/fuelweb_test/tests/test_simple.py +++ b/fuelweb_test/tests/test_simple.py @@ -117,7 +117,8 @@ class SimpleFlat(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=5) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_simple_flat") @@ -268,7 +269,8 @@ class SimpleFlat(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=5) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("simple_flat_add_compute") @@ -320,7 +322,8 @@ class SimpleVlan(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=5) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_simple_vlan") @@ -362,9 +365,7 @@ class MultiroleControllerCinder(TestBasic): self.fuel_web.verify_network(cluster_id) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("deploy_multirole_controller_cinder") @@ -407,9 +408,7 @@ class MultiroleComputeCinder(TestBasic): self.fuel_web.verify_network(cluster_id) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("deploy_multirole_compute_cinder") @@ -536,7 +535,8 @@ class FloatingIPs(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=5) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) self.env.make_snapshot("deploy_floating_ips") @@ -581,9 +581,7 @@ class SimpleCinder(TestBasic): self.fuel_web.assert_cluster_ready( 'slave-01', smiles_count=6, networks_count=1, timeout=300) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("deploy_simple_cinder") diff --git a/fuelweb_test/tests/tests_strength/test_huge_environments.py b/fuelweb_test/tests/tests_strength/test_huge_environments.py index 6e5c751b6..eac1b670e 100644 --- a/fuelweb_test/tests/tests_strength/test_huge_environments.py +++ b/fuelweb_test/tests/tests_strength/test_huge_environments.py @@ -83,7 +83,8 @@ class HugeEnvironments(base_test_case.TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=2) + should_fail=1, + failed_test_name=['Check stack autoscaling']) @test(depends_on=[base_test_case.SetupEnvironment.prepare_release], groups=["nine_nodes_separate_roles"]) @@ -143,4 +144,5 @@ class HugeEnvironments(base_test_case.TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=2) + should_fail=1, + failed_test_name=['Check stack autoscaling']) diff --git a/fuelweb_test/tests/tests_strength/test_master_node_failover.py b/fuelweb_test/tests/tests_strength/test_master_node_failover.py index 125f4bbfb..59c58dc68 100644 --- a/fuelweb_test/tests/tests_strength/test_master_node_failover.py +++ b/fuelweb_test/tests/tests_strength/test_master_node_failover.py @@ -71,12 +71,13 @@ class DeploySimpleMasterNodeFail(base_test_case.TestBasic): logger.info('PASS DEPLOYMENT') self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=1) + should_fail=1, + failed_test_name=['Create volume and attach it to instance']) logger.info('PASS OSTF') logger.info('Get controller ip') controller_ip = self.fuel_web.get_nailgun_node_by_name( 'slave-01')['ip'] - logger.info('Destoy admin node...') + logger.info('Destroy admin node...') self.env.nodes().admin.destroy() logger.info('Admin node destroyed') diff --git a/fuelweb_test/tests/tests_strength/test_restart.py b/fuelweb_test/tests/tests_strength/test_restart.py index a7ebd0d0b..99b7ca65c 100644 --- a/fuelweb_test/tests/tests_strength/test_restart.py +++ b/fuelweb_test/tests/tests_strength/test_restart.py @@ -81,9 +81,7 @@ class CephRestart(TestBasic): self.fuel_web.warm_restart_nodes(self.env.nodes().slaves[:4]) check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) @test(groups=["thread_5", "ceph"]) @@ -138,9 +136,7 @@ class CephHARestart(TestBasic): check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) # Run ostf - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) # Destroy osd-node self.env.nodes().slaves[5].destroy() @@ -148,23 +144,19 @@ class CephHARestart(TestBasic): recovery_timeout=True) self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=4) + should_fail=0) # Destroy compute node self.env.nodes().slaves[4].destroy() check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01'), recovery_timeout=True) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) # Cold restart self.fuel_web.cold_restart_nodes(self.env.nodes().slaves[:4]) check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) - self.fuel_web.run_ostf( - cluster_id=cluster_id, - should_fail=4) + self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ceph_ha") @@ -212,4 +204,5 @@ class SimpleFlatRestart(TestBasic): self.fuel_web.run_ostf( cluster_id=cluster_id, - should_fail=5) + should_fail=1, + failed_test_name=['Create volume and attach it to instance'])