openstack
/
fuel-main
Code Issues Proposed changes

Merge "Change ostf expectation failure count"

This commit is contained in:
Jenkins 2014-02-18 18:40:33 +00:00 committed by Gerrit Code Review
parent 211daadd43 94e2a4bc82
commit 0b9ba969d1
10 changed files with 82 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
fuelweb_test/models/fuel_web_client.py
View File

@ -121,12 +121,13 @@ class FuelWebClient(object):
assert_true(result)
@logwrap
def assert_ostf_run(self, cluster_id, should_fail=0, timeout=15 * 60):
def assert_ostf_run(self, cluster_id, should_fail=0,
failed_test_name=None, timeout=15 * 60):
set_result_list = self._ostf_test_wait(cluster_id, timeout)
failed_tests_res = []
failed = 0
failed_tests_names = []
actual_failed_names = []
test_result = {}
for set_result in set_result_list:
@ -137,19 +138,30 @@ class FuelWebClient(object):
set_result['tests']
)
)
[actual_failed_names.append(test['name'])
for test in set_result['tests'] if test['status'] != 'success']
[test_result.update({test['name']:test['status']})
for test in set_result['tests']]
[failed_tests_names.append({test['name']:test['message']})
[failed_tests_res.append({test['name']:test['message']})
for test in set_result['tests'] if test['status'] != 'success']
logger.info('OSTF test statuses are : {0}'.format(test_result))
if failed_test_name:
for test_name in failed_test_name:
assert_true(test_name in actual_failed_names,
'WARNINg unexpected fail,'
'expected {0} actual {1}'.format(
failed_test_name, actual_failed_names))
assert_true(
failed <= should_fail, 'Failed tests, fails: {} should fail:'
' {} failed tests name: {}'
''.format(failed, should_fail,
failed_tests_names))
failed_tests_res))
def assert_release_state(self, release_name, state='available'):
for release in self.client.get_releases():
@ -397,8 +409,9 @@ class FuelWebClient(object):
cluster_id, self.client.get_networks(cluster_id)['networks'])
@logwrap
def run_ostf(self, cluster_id, test_sets=None, should_fail=0,
tests_must_be_passed=None, timeout=None):
def run_ostf(self, cluster_id, test_sets=None,
should_fail=0, tests_must_be_passed=None,
timeout=None, failed_test_name=None):
test_sets = test_sets or ['smoke', 'sanity']
timeout = timeout or 30 * 60
self.client.ostf_run_tests(cluster_id, test_sets)
@ -408,9 +421,12 @@ class FuelWebClient(object):
tests_must_be_passed,
timeout)
else:
logger.info('Try to run assert ostf with '
'expected fail name {0}'.format(failed_test_name))
self.assert_ostf_run(
cluster_id,
should_fail=should_fail, timeout=timeout)
should_fail=should_fail, timeout=timeout,
failed_test_name=failed_test_name)
@logwrap
def run_single_ostf_test(self, cluster_id,

15
fuelweb_test/tests/test_ceph.py
View File

@ -74,9 +74,7 @@ class CephCompact(TestBasic):
check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01'))
# Run ostf
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("ceph_multinode_compact")
@ -129,9 +127,7 @@ class CephCompactWithCinder(TestBasic):
check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01'))
# Run ostf
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("ceph_multinode_with_cinder")
@ -186,9 +182,7 @@ class CephHA(TestBasic):
check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01'))
# Run ostf
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("ceph_ha")
@ -250,7 +244,8 @@ class CephRadosGW(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
test_sets=['smoke', 'sanity', 'platform_tests'],
should_fail=1)
should_fail=1,
failed_test_name=['Check stack autoscaling'])
# Check the radosqw daemon is started
radosgw_started = lambda: len(remote.check_call(

9
fuelweb_test/tests/test_ha.py
View File

@ -79,7 +79,8 @@ class TestHaVLAN(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'],
should_fail=4)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("deploy_ha_vlan")
@ -132,7 +133,8 @@ class TestHaFlat(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'],
should_fail=4)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("deploy_ha_flat")
@ -191,6 +193,7 @@ class TestHaFlatAddCompute(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'], should_fail=4)
test_sets=['ha', 'smoke', 'sanity'], should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("ha_flat_add_compute")

12
fuelweb_test/tests/test_neutron.py
View File

@ -86,7 +86,8 @@ class NeutronGre(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=2)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("deploy_neutron_gre")
@ -144,7 +145,8 @@ class NeutronVlan(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=2)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("deploy_neutron_vlan")
@ -204,7 +206,8 @@ class NeutronGreHa(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("deploy_neutron_gre_ha")
@ -264,6 +267,7 @@ class NeutronVlanHa(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("deploy_neutron_vlan_ha")

2
fuelweb_test/tests/test_pullrequest.py
View File

@ -73,5 +73,5 @@ class TestPullRequest(TestBasic):
self.fuel_web.run_ostf(
cluster_id=self.fuel_web.get_last_created_cluster(),
should_fail=4
should_fail=1
)

18
fuelweb_test/tests/test_services.py
View File

@ -279,7 +279,8 @@ class CeilometerSimple(TestBasic):
# run ostf smoke and sanity
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=1)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
# verify if needed image exists
LOGGER.debug('Check MD5 of image')
@ -308,7 +309,8 @@ class CeilometerSimple(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id, test_sets=['platform_tests'],
should_fail=1, timeout=3500)
should_fail=1, timeout=3500,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("deploy_ceilometer_simple")
@ -440,7 +442,9 @@ class CeilometerSimpleMongo(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id, test_sets=['platform_tests', 'ha'],
should_fail=2, timeout=3500)
should_fail=2, timeout=3500,
failed_test_name=['Create volume and attach it to instance',
'Check stack autoscaling'])
self.env.make_snapshot("deploy_ceilometer_ha_with_mongo")
@ -492,7 +496,9 @@ class CeilometerSimpleMongo(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id, test_sets=['platform_tests'],
should_fail=2, timeout=3500)
should_fail=2, timeout=3500,
failed_test_name=['Create volume and attach it to instance',
'Check stack autoscaling'])
self.env.make_snapshot("deploy_ceilometer_simple_mulirole")
@ -545,6 +551,8 @@ class CeilometerSimpleMongo(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id, test_sets=['platform_tests', 'ha'],
should_fail=2, timeout=3500)
should_fail=2, timeout=3500,
failed_test_name=['Create volume and attach it to instance',
'Check stack autoscaling'])
self.env.make_snapshot("deploy_ceilometer_ha_mulirole")

24
fuelweb_test/tests/test_simple.py
View File

@ -117,7 +117,8 @@ class SimpleFlat(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=5)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("deploy_simple_flat")
@ -268,7 +269,8 @@ class SimpleFlat(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=5)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("simple_flat_add_compute")
@ -320,7 +322,8 @@ class SimpleVlan(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=5)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("deploy_simple_vlan")
@ -362,9 +365,7 @@ class MultiroleControllerCinder(TestBasic):
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("deploy_multirole_controller_cinder")
@ -407,9 +408,7 @@ class MultiroleComputeCinder(TestBasic):
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("deploy_multirole_compute_cinder")
@ -536,7 +535,8 @@ class FloatingIPs(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=5)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
self.env.make_snapshot("deploy_floating_ips")
@ -581,9 +581,7 @@ class SimpleCinder(TestBasic):
self.fuel_web.assert_cluster_ready(
'slave-01', smiles_count=6, networks_count=1, timeout=300)
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("deploy_simple_cinder")

6
fuelweb_test/tests/tests_strength/test_huge_environments.py
View File

@ -83,7 +83,8 @@ class HugeEnvironments(base_test_case.TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=2)
should_fail=1,
failed_test_name=['Check stack autoscaling'])
@test(depends_on=[base_test_case.SetupEnvironment.prepare_release],
groups=["nine_nodes_separate_roles"])
@ -143,4 +144,5 @@ class HugeEnvironments(base_test_case.TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=2)
should_fail=1,
failed_test_name=['Check stack autoscaling'])

5
fuelweb_test/tests/tests_strength/test_master_node_failover.py
View File

@ -71,12 +71,13 @@ class DeploySimpleMasterNodeFail(base_test_case.TestBasic):
logger.info('PASS DEPLOYMENT')
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=1)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])
logger.info('PASS OSTF')
logger.info('Get controller ip')
controller_ip = self.fuel_web.get_nailgun_node_by_name(
'slave-01')['ip']
logger.info('Destoy admin node...')
logger.info('Destroy admin node...')
self.env.nodes().admin.destroy()
logger.info('Admin node destroyed')

21
fuelweb_test/tests/tests_strength/test_restart.py
View File

@ -81,9 +81,7 @@ class CephRestart(TestBasic):
self.fuel_web.warm_restart_nodes(self.env.nodes().slaves[:4])
check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01'))
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
self.fuel_web.run_ostf(cluster_id=cluster_id)
@test(groups=["thread_5", "ceph"])
@ -138,9 +136,7 @@ class CephHARestart(TestBasic):
check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01'))
# Run ostf
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
self.fuel_web.run_ostf(cluster_id=cluster_id)
# Destroy osd-node
self.env.nodes().slaves[5].destroy()
@ -148,23 +144,19 @@ class CephHARestart(TestBasic):
recovery_timeout=True)
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
should_fail=0)
# Destroy compute node
self.env.nodes().slaves[4].destroy()
check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01'),
recovery_timeout=True)
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
self.fuel_web.run_ostf(cluster_id=cluster_id)
# Cold restart
self.fuel_web.cold_restart_nodes(self.env.nodes().slaves[:4])
check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01'))
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=4)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("ceph_ha")
@ -212,4 +204,5 @@ class SimpleFlatRestart(TestBasic):
self.fuel_web.run_ostf(
cluster_id=cluster_id,
should_fail=5)
should_fail=1,
failed_test_name=['Create volume and attach it to instance'])