From f98d8d02fa6a6aadecb666e6693eeef7be30716d Mon Sep 17 00:00:00 2001 From: Sergey Novikov Date: Mon, 22 Aug 2016 12:21:26 +0300 Subject: [PATCH] Add specified OSTF runner into upgrade tests Sometimes OSTF tests fail with non-related cluster health and upgrade reasons. For stabilization of upgrade tests we need to ignore these failures. Change-Id: Ib0f6f58576dd71b5556a7685623abe2a68f6a456 --- fuelweb_test/tests/tests_upgrade/__init__.py | 4 +- .../tests_upgrade/test_data_driven_upgrade.py | 14 ++-- .../test_data_driven_upgrade_base.py | 74 ++++++++++++++----- .../test_data_driven_upgrade_ceph_ha.py | 6 +- .../test_data_driven_upgrade_net_tmpl.py | 26 ++++--- .../test_data_driven_upgrade_plugin.py | 4 +- 6 files changed, 86 insertions(+), 42 deletions(-) diff --git a/fuelweb_test/tests/tests_upgrade/__init__.py b/fuelweb_test/tests/tests_upgrade/__init__.py index 556985627..b867731f1 100644 --- a/fuelweb_test/tests/tests_upgrade/__init__.py +++ b/fuelweb_test/tests/tests_upgrade/__init__.py @@ -14,12 +14,12 @@ # pylint: disable=line-too-long -from fuelweb_test.tests.tests_upgrade import test_clone_env # noqa +from fuelweb_test.tests.tests_upgrade import test_clone_env # noqa from fuelweb_test.tests.tests_upgrade import test_data_driven_upgrade # noqa from fuelweb_test.tests.tests_upgrade import test_data_driven_upgrade_ceph_ha # noqa from fuelweb_test.tests.tests_upgrade import test_data_driven_upgrade_net_tmpl # noqa from fuelweb_test.tests.tests_upgrade import test_data_driven_upgrade_plugin # noqa -from fuelweb_test.tests.tests_upgrade import test_node_reassignment # noqa +from fuelweb_test.tests.tests_upgrade import test_node_reassignment # noqa from fuelweb_test.tests.tests_upgrade import upgrader_tool # noqa from fuelweb_test.tests.tests_upgrade import test_os_upgrade # noqa diff --git a/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade.py b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade.py index 7a5fdd9f4..f441e13bd 100644 --- a/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade.py +++ b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade.py @@ -153,7 +153,7 @@ class UpgradeSmoke(DataDrivenUpgradeBase): # Live migration test could fail # https://bugs.launchpad.net/fuel/+bug/1471172 # https://bugs.launchpad.net/fuel/+bug/1604749 - self.fuel_web.run_ostf(cluster_id, should_fail=1) + self.check_ostf(cluster_id, ignore_known_issues=True) self.env.make_snapshot("upgrade_smoke_restore", is_make=True) self.cleanup() @@ -200,7 +200,7 @@ class UpgradeSmoke(DataDrivenUpgradeBase): # LP 1562736 get_devops_node_by_nailgun_node is not working self.fuel_web.deploy_cluster_wait(cluster_id) self.show_step(6) - self.fuel_web.run_ostf(cluster_id) + self.check_ostf(cluster_id, ignore_known_issues=True) cluster_id = self.fuel_web.get_last_created_cluster() self.show_step(7) @@ -225,7 +225,7 @@ class UpgradeSmoke(DataDrivenUpgradeBase): self.show_step(12) self.fuel_web.verify_network(cluster_id) self.show_step(13) - self.fuel_web.run_ostf(cluster_id) + self.check_ostf(cluster_id, ignore_known_issues=True) self.env.make_snapshot("upgrade_smoke_scale") @test(groups=['upgrade_smoke_reset_deploy'], @@ -282,7 +282,7 @@ class UpgradeSmoke(DataDrivenUpgradeBase): self.show_step(7) self.fuel_web.deploy_cluster_wait(cluster_id) self.show_step(8) - self.fuel_web.run_ostf(cluster_id) + self.check_ostf(cluster_id, ignore_known_issues=True) @test(groups=['upgrade_smoke_new_deployment'], depends_on_groups=['upgrade_smoke_restore']) @@ -347,7 +347,7 @@ class UpgradeSmoke(DataDrivenUpgradeBase): self.show_step(7) self.fuel_web.deploy_cluster_wait(cluster_id) self.show_step(8) - self.fuel_web.run_ostf(cluster_id) + self.check_ostf(cluster_id) @test(groups=['upgrade_no_cluster_tests']) @@ -451,7 +451,7 @@ class UpgradeNoCluster(DataDrivenUpgradeBase): self.show_step(6) self.fuel_web.verify_network(cluster_id) self.show_step(7) - self.fuel_web.run_ostf(cluster_id) + self.check_ostf(cluster_id) self.env.make_snapshot("upgrade_no_cluster_deploy", is_make=True) @test(groups=['upgrade_no_cluster_deploy_old_cluster'], @@ -506,4 +506,4 @@ class UpgradeNoCluster(DataDrivenUpgradeBase): self.show_step(7) self.fuel_web.deploy_cluster_wait(cluster_id) self.show_step(8) - self.fuel_web.run_ostf(cluster_id) + self.check_ostf(cluster_id, ignore_known_issues=True) diff --git a/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_base.py b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_base.py index a9362abe8..c4c120524 100644 --- a/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_base.py +++ b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_base.py @@ -14,9 +14,11 @@ from __future__ import unicode_literals +import itertools import os # pylint: disable=import-error # pylint: disable=no-name-in-module +from distutils.version import LooseVersion from distutils.version import StrictVersion # pylint: enable=no-name-in-module # pylint: enable=import-error @@ -29,11 +31,27 @@ from proboscis.asserts import assert_not_equal from proboscis.asserts import assert_true from fuelweb_test import logger -from fuelweb_test import settings +from fuelweb_test.settings import DEPLOYMENT_MODE +from fuelweb_test.settings import FUEL_PROPOSED_REPO_URL +from fuelweb_test.settings import LOGS_DIR +from fuelweb_test.settings import KEYSTONE_CREDS +from fuelweb_test.settings import NEUTRON +from fuelweb_test.settings import NEUTRON_SEGMENT +from fuelweb_test.settings import UPGRADE_FUEL_FROM +from fuelweb_test.settings import UPGRADE_BACKUP_FILES_LOCAL_DIR +from fuelweb_test.settings import UPGRADE_BACKUP_FILES_REMOTE_DIR +from fuelweb_test.settings import UPGRADE_FUEL_TO +from fuelweb_test.settings import OCTANE_PATCHES from fuelweb_test.tests.base_test_case import TestBasic class DataDrivenUpgradeBase(TestBasic): + + IGNORED_OSTF_TESTS = { + '7.0': ['Instance live migration'], + '8.0': ['Launch instance with file injection'] + } + OCTANE_COMMANDS = { 'backup': 'octane -v --debug fuel-backup --to {path}', 'repo-backup': 'octane -v --debug fuel-repo-backup --to {path} --full', @@ -47,10 +65,10 @@ class DataDrivenUpgradeBase(TestBasic): def __init__(self): super(DataDrivenUpgradeBase, self).__init__() - self.local_dir_for_backups = settings.UPGRADE_BACKUP_FILES_LOCAL_DIR + self.local_dir_for_backups = UPGRADE_BACKUP_FILES_LOCAL_DIR if not os.path.exists(self.local_dir_for_backups): os.makedirs(self.local_dir_for_backups) - self.remote_dir_for_backups = settings.UPGRADE_BACKUP_FILES_REMOTE_DIR + self.remote_dir_for_backups = UPGRADE_BACKUP_FILES_REMOTE_DIR self.cluster_creds = { 'tenant': 'upgrade', 'user': 'upgrade', @@ -145,17 +163,15 @@ class DataDrivenUpgradeBase(TestBasic): def install_octane(self): """ Install fuel-octane package to master node""" conf_file = None - if settings.FUEL_PROPOSED_REPO_URL: + if FUEL_PROPOSED_REPO_URL: conf_file = '/etc/yum.repos.d/fuel-proposed.repo' - settings.FUEL_PROPOSED_REPO_URL = os.environ.get( - 'FUEL_PROPOSED_REPO_URL') cmd = ("echo -e " "'[fuel-proposed]\n" "name=fuel-proposed\n" "baseurl={}/\n" "gpgcheck=0\n" "priority=1' > {}").format( - settings.FUEL_PROPOSED_REPO_URL, + FUEL_PROPOSED_REPO_URL, conf_file) # pylint: disable=no-member @@ -179,9 +195,9 @@ class DataDrivenUpgradeBase(TestBasic): logger.info("Octane changes:") logger.info(octane_log) - if settings.OCTANE_PATCHES: + if OCTANE_PATCHES: logger.info("Patching octane with CR: {!r}".format( - settings.OCTANE_PATCHES)) + OCTANE_PATCHES)) # pylint: disable=no-member self.admin_remote.upload( os.path.join( @@ -191,10 +207,10 @@ class DataDrivenUpgradeBase(TestBasic): self.admin_remote.check_call( "bash /tmp/octane_patcher.sh {}".format( - settings.OCTANE_PATCHES)) + OCTANE_PATCHES)) # pylint: enable=no-member - if settings.FUEL_PROPOSED_REPO_URL: + if FUEL_PROPOSED_REPO_URL: # pylint: disable=no-member self.admin_remote.rm_rf(conf_file) # pylint: enable=no-member @@ -204,7 +220,7 @@ class DataDrivenUpgradeBase(TestBasic): "Unknown octane action '{}', aborting".format(action)) octane_cli_args = { 'path': path, - 'pwd': settings.KEYSTONE_CREDS['password'] + 'pwd': KEYSTONE_CREDS['password'] } admin_remote = self.env.d_env.get_admin_remote() if 'backup' in action: @@ -224,7 +240,7 @@ class DataDrivenUpgradeBase(TestBasic): # snapshot generating procedure can be broken admin_remote.download( "/var/log/octane.log", - os.path.join(settings.LOGS_DIR, + os.path.join(LOGS_DIR, "octane_{}_.log".format(os.path.basename(path)))) raise @@ -348,7 +364,7 @@ class DataDrivenUpgradeBase(TestBasic): self.env.sync_time() cluster_id = self.fuel_web.create_cluster( name=cluster_settings['name'], - mode=settings.DEPLOYMENT_MODE, + mode=DEPLOYMENT_MODE, settings=cluster_settings['settings'] ) if cluster_settings.get('plugin'): @@ -384,12 +400,12 @@ class DataDrivenUpgradeBase(TestBasic): intermediate_snapshot = "prepare_upgrade_smoke_before_backup" assert_not_equal( - settings.KEYSTONE_CREDS['password'], 'admin', + KEYSTONE_CREDS['password'], 'admin', "Admin password was not changed, aborting execution") cluster_settings = { - 'net_provider': settings.NEUTRON, - 'net_segment_type': settings.NEUTRON_SEGMENT['vlan'] + 'net_provider': NEUTRON, + 'net_segment_type': NEUTRON_SEGMENT['vlan'] } cluster_settings.update(self.cluster_creds) @@ -452,3 +468,27 @@ class DataDrivenUpgradeBase(TestBasic): if self.fuel_version <= StrictVersion('8.0'): cmd = "dockerctl shell cobbler {}".format(cmd) admin_remote.check_call(cmd) + + def check_ostf(self, cluster_id, test_sets=None, timeout=30 * 60, + ignore_known_issues=False, additional_ignored_issues=None): + """Run OSTF tests with the ignoring some test result + """ + if additional_ignored_issues: + ignr_tests = additional_ignored_issues + else: + ignr_tests = [] + + if ignore_known_issues: + if ( + LooseVersion(UPGRADE_FUEL_TO) >= LooseVersion('9.0') and + LooseVersion(UPGRADE_FUEL_FROM) == LooseVersion('7.0') + ): + mrg_set = set(ignr_tests) | set( + itertools.chain(*self.IGNORED_OSTF_TESTS.values())) + ignr_tests = list(mrg_set) + else: + ignr_tests.extend(self.IGNORED_OSTF_TESTS[UPGRADE_FUEL_FROM]) + + self.fuel_web.run_ostf(cluster_id, test_sets=test_sets, + should_fail=len(ignr_tests), + failed_test_name=ignr_tests, timeout=timeout) diff --git a/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_ceph_ha.py b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_ceph_ha.py index 922ec163f..30b01a638 100644 --- a/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_ceph_ha.py +++ b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_ceph_ha.py @@ -170,7 +170,7 @@ class UpgradeCephHA(DataDrivenUpgradeBase): # Live migration test could fail # https://bugs.launchpad.net/fuel/+bug/1471172 # https://bugs.launchpad.net/fuel/+bug/1604749 - self.fuel_web.run_ostf(cluster_id, should_fail=1) + self.check_ostf(cluster_id, ignore_known_issues=True) self.env.make_snapshot(self.snapshot_name, is_make=True) self.cleanup() @@ -203,7 +203,7 @@ class UpgradeCephHA(DataDrivenUpgradeBase): self.fuel_web.assert_ha_services_ready(cluster_id) self.fuel_web.assert_os_services_ready(cluster_id) self.show_step(4) - self.fuel_web.run_ostf(cluster_id) + self.check_ostf(cluster_id, ignore_known_issues=True) @test(groups=['upgrade_ceph_ha_tests', 'upgrade_ceph_ha_scale_ceph'], depends_on_groups=['upgrade_ceph_ha_restore']) @@ -232,4 +232,4 @@ class UpgradeCephHA(DataDrivenUpgradeBase): # LP 1562736 get_devops_node_by_nailgun_node is not working self.fuel_web.deploy_cluster_wait(cluster_id) self.show_step(5) - self.fuel_web.run_ostf(cluster_id) + self.check_ostf(cluster_id, ignore_known_issues=True) diff --git a/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_net_tmpl.py b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_net_tmpl.py index e9c22ace8..75cf6f74b 100644 --- a/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_net_tmpl.py +++ b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_net_tmpl.py @@ -134,9 +134,9 @@ class TestUpgradeNetworkTemplates(TestNetworkTemplatesBase, # Live migration test could fail # https://bugs.launchpad.net/fuel/+bug/1471172 # https://bugs.launchpad.net/fuel/+bug/1604749 - self.fuel_web.run_ostf(cluster_id=cluster_id, - test_sets=['smoke', 'sanity', 'ha'], - should_fail=1) + self.check_ostf(cluster_id=cluster_id, + test_sets=['smoke', 'sanity', 'ha'], + ignore_known_issues=True) self.show_step(11) self.check_ipconfig_for_template(cluster_id, network_template, networks) @@ -195,8 +195,9 @@ class TestUpgradeNetworkTemplates(TestNetworkTemplatesBase, self.show_step(7) self.fuel_web.verify_network(cluster_id) self.show_step(8) - self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1, - test_sets=['smoke', 'sanity', 'ha']) + self.check_ostf(cluster_id=cluster_id, + test_sets=['smoke', 'sanity', 'ha'], + ignore_known_issues=True) self.env.make_snapshot("upgrade_net_tmpl_restore", is_make=True) @test(depends_on_groups=["upgrade_net_tmpl_restore"], @@ -229,8 +230,9 @@ class TestUpgradeNetworkTemplates(TestNetworkTemplatesBase, self.fuel_web.verify_network(cluster_id) self.show_step(5) - self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1, - test_sets=['smoke', 'sanity', 'ha']) + self.check_ostf(cluster_id=cluster_id, + test_sets=['smoke', 'sanity', 'ha'], + ignore_known_issues=True) self.env.make_snapshot("reset_deploy_net_tmpl", is_make=True) @test(depends_on_groups=["upgrade_net_tmpl_restore"], @@ -283,8 +285,9 @@ class TestUpgradeNetworkTemplates(TestNetworkTemplatesBase, self.fuel_web.verify_network(cluster_id) self.show_step(7) - self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1, - test_sets=['smoke', 'sanity', 'ha']) + self.check_ostf(cluster_id=cluster_id, + test_sets=['smoke', 'sanity', 'ha'], + ignore_known_issues=True) self.env.make_snapshot("replace_controller_net_tmpl", is_make=True) @test(depends_on_groups=["replace_controller_net_tmpl"], @@ -318,6 +321,7 @@ class TestUpgradeNetworkTemplates(TestNetworkTemplatesBase, self.fuel_web.verify_network(cluster_id) self.show_step(5) - self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1, - test_sets=['smoke', 'sanity', 'ha']) + self.check_ostf(cluster_id=cluster_id, + test_sets=['smoke', 'sanity', 'ha'], + ignore_known_issues=True) self.env.make_snapshot("restart_node_net_tmpl", is_make=True) diff --git a/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_plugin.py b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_plugin.py index d52619402..79117f320 100644 --- a/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_plugin.py +++ b/fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_plugin.py @@ -176,7 +176,7 @@ class UpgradePlugin(DataDrivenUpgradeBase): # Live migration test could fail # https://bugs.launchpad.net/fuel/+bug/1471172 # https://bugs.launchpad.net/fuel/+bug/1604749 - self.fuel_web.run_ostf(cluster_id, should_fail=1) + self.check_ostf(cluster_id, ignore_known_issues=True) self.env.make_snapshot(self.snapshot_name, is_make=True) self.cleanup() @@ -212,4 +212,4 @@ class UpgradePlugin(DataDrivenUpgradeBase): self.show_step(4) self.fuel_web.deploy_cluster_wait(cluster_id) self.show_step(5) - self.fuel_web.run_ostf(cluster_id) + self.check_ostf(cluster_id, ignore_known_issues=True)