Merge "Add specified OSTF runner into upgrade tests" into stable/mitaka

fuelweb_test/tests/tests_upgrade/__init__.py

@@ -14,12 +14,12 @@
 
 # pylint: disable=line-too-long
 
-from fuelweb_test.tests.tests_upgrade import test_clone_env # noqa
+from fuelweb_test.tests.tests_upgrade import test_clone_env  # noqa
 from fuelweb_test.tests.tests_upgrade import test_data_driven_upgrade  # noqa
 from fuelweb_test.tests.tests_upgrade import test_data_driven_upgrade_ceph_ha  # noqa
 from fuelweb_test.tests.tests_upgrade import test_data_driven_upgrade_net_tmpl  # noqa
 from fuelweb_test.tests.tests_upgrade import test_data_driven_upgrade_plugin  # noqa
-from fuelweb_test.tests.tests_upgrade import test_node_reassignment # noqa
+from fuelweb_test.tests.tests_upgrade import test_node_reassignment  # noqa
 from fuelweb_test.tests.tests_upgrade import upgrader_tool  # noqa
 from fuelweb_test.tests.tests_upgrade import test_os_upgrade  # noqa
 

fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade.py

@@ -153,7 +153,7 @@ class UpgradeSmoke(DataDrivenUpgradeBase):
         # Live migration test could fail
         # https://bugs.launchpad.net/fuel/+bug/1471172
         # https://bugs.launchpad.net/fuel/+bug/1604749
-        self.fuel_web.run_ostf(cluster_id, should_fail=1)
+        self.check_ostf(cluster_id, ignore_known_issues=True)
 
         self.env.make_snapshot("upgrade_smoke_restore", is_make=True)
         self.cleanup()
@@ -200,7 +200,7 @@ class UpgradeSmoke(DataDrivenUpgradeBase):
         # LP 1562736 get_devops_node_by_nailgun_node is not working
         self.fuel_web.deploy_cluster_wait(cluster_id)
         self.show_step(6)
-        self.fuel_web.run_ostf(cluster_id)
+        self.check_ostf(cluster_id, ignore_known_issues=True)
 
         cluster_id = self.fuel_web.get_last_created_cluster()
         self.show_step(7)
@@ -225,7 +225,7 @@ class UpgradeSmoke(DataDrivenUpgradeBase):
         self.show_step(12)
         self.fuel_web.verify_network(cluster_id)
         self.show_step(13)
-        self.fuel_web.run_ostf(cluster_id)
+        self.check_ostf(cluster_id, ignore_known_issues=True)
         self.env.make_snapshot("upgrade_smoke_scale")
 
     @test(groups=['upgrade_smoke_reset_deploy'],
@@ -282,7 +282,7 @@ class UpgradeSmoke(DataDrivenUpgradeBase):
         self.show_step(7)
         self.fuel_web.deploy_cluster_wait(cluster_id)
         self.show_step(8)
-        self.fuel_web.run_ostf(cluster_id)
+        self.check_ostf(cluster_id, ignore_known_issues=True)
 
     @test(groups=['upgrade_smoke_new_deployment'],
           depends_on_groups=['upgrade_smoke_restore'])
@@ -347,7 +347,7 @@ class UpgradeSmoke(DataDrivenUpgradeBase):
         self.show_step(7)
         self.fuel_web.deploy_cluster_wait(cluster_id)
         self.show_step(8)
-        self.fuel_web.run_ostf(cluster_id)
+        self.check_ostf(cluster_id)
 
 
 @test(groups=['upgrade_no_cluster_tests'])
@@ -451,7 +451,7 @@ class UpgradeNoCluster(DataDrivenUpgradeBase):
         self.show_step(6)
         self.fuel_web.verify_network(cluster_id)
         self.show_step(7)
-        self.fuel_web.run_ostf(cluster_id)
+        self.check_ostf(cluster_id)
         self.env.make_snapshot("upgrade_no_cluster_deploy", is_make=True)
 
     @test(groups=['upgrade_no_cluster_deploy_old_cluster'],
@@ -506,4 +506,4 @@ class UpgradeNoCluster(DataDrivenUpgradeBase):
         self.show_step(7)
         self.fuel_web.deploy_cluster_wait(cluster_id)
         self.show_step(8)
-        self.fuel_web.run_ostf(cluster_id)
+        self.check_ostf(cluster_id, ignore_known_issues=True)

fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_base.py

@@ -14,9 +14,11 @@
 
 from __future__ import unicode_literals
 
+import itertools
 import os
 # pylint: disable=import-error
 # pylint: disable=no-name-in-module
+from distutils.version import LooseVersion
 from distutils.version import StrictVersion
 # pylint: enable=no-name-in-module
 # pylint: enable=import-error
@@ -29,11 +31,27 @@ from proboscis.asserts import assert_not_equal
 from proboscis.asserts import assert_true
 
 from fuelweb_test import logger
-from fuelweb_test import settings
+from fuelweb_test.settings import DEPLOYMENT_MODE
+from fuelweb_test.settings import FUEL_PROPOSED_REPO_URL
+from fuelweb_test.settings import LOGS_DIR
+from fuelweb_test.settings import KEYSTONE_CREDS
+from fuelweb_test.settings import NEUTRON
+from fuelweb_test.settings import NEUTRON_SEGMENT
+from fuelweb_test.settings import UPGRADE_FUEL_FROM
+from fuelweb_test.settings import UPGRADE_BACKUP_FILES_LOCAL_DIR
+from fuelweb_test.settings import UPGRADE_BACKUP_FILES_REMOTE_DIR
+from fuelweb_test.settings import UPGRADE_FUEL_TO
+from fuelweb_test.settings import OCTANE_PATCHES
 from fuelweb_test.tests.base_test_case import TestBasic
 
 
 class DataDrivenUpgradeBase(TestBasic):
+
+    IGNORED_OSTF_TESTS = {
+        '7.0': ['Instance live migration'],
+        '8.0': ['Launch instance with file injection']
+    }
+
     OCTANE_COMMANDS = {
         'backup': 'octane -v --debug fuel-backup --to {path}',
         'repo-backup': 'octane -v --debug fuel-repo-backup --to {path} --full',
@@ -47,10 +65,10 @@ class DataDrivenUpgradeBase(TestBasic):
 
     def __init__(self):
         super(DataDrivenUpgradeBase, self).__init__()
-        self.local_dir_for_backups = settings.UPGRADE_BACKUP_FILES_LOCAL_DIR
+        self.local_dir_for_backups = UPGRADE_BACKUP_FILES_LOCAL_DIR
         if not os.path.exists(self.local_dir_for_backups):
             os.makedirs(self.local_dir_for_backups)
-        self.remote_dir_for_backups = settings.UPGRADE_BACKUP_FILES_REMOTE_DIR
+        self.remote_dir_for_backups = UPGRADE_BACKUP_FILES_REMOTE_DIR
         self.cluster_creds = {
             'tenant': 'upgrade',
             'user': 'upgrade',
@@ -145,17 +163,15 @@ class DataDrivenUpgradeBase(TestBasic):
     def install_octane(self):
         """ Install fuel-octane package to master node"""
         conf_file = None
-        if settings.FUEL_PROPOSED_REPO_URL:
+        if FUEL_PROPOSED_REPO_URL:
             conf_file = '/etc/yum.repos.d/fuel-proposed.repo'
-            settings.FUEL_PROPOSED_REPO_URL = os.environ.get(
-                'FUEL_PROPOSED_REPO_URL')
             cmd = ("echo -e "
                    "'[fuel-proposed]\n"
                    "name=fuel-proposed\n"
                    "baseurl={}/\n"
                    "gpgcheck=0\n"
                    "priority=1' > {}").format(
-                       settings.FUEL_PROPOSED_REPO_URL,
+                       FUEL_PROPOSED_REPO_URL,
                        conf_file)
 
             # pylint: disable=no-member
@@ -179,9 +195,9 @@ class DataDrivenUpgradeBase(TestBasic):
         logger.info("Octane changes:")
         logger.info(octane_log)
 
-        if settings.OCTANE_PATCHES:
+        if OCTANE_PATCHES:
             logger.info("Patching octane with CR: {!r}".format(
-                settings.OCTANE_PATCHES))
+                OCTANE_PATCHES))
             # pylint: disable=no-member
             self.admin_remote.upload(
                 os.path.join(
@@ -191,10 +207,10 @@ class DataDrivenUpgradeBase(TestBasic):
 
             self.admin_remote.check_call(
                 "bash /tmp/octane_patcher.sh {}".format(
-                    settings.OCTANE_PATCHES))
+                    OCTANE_PATCHES))
             # pylint: enable=no-member
 
-        if settings.FUEL_PROPOSED_REPO_URL:
+        if FUEL_PROPOSED_REPO_URL:
             # pylint: disable=no-member
             self.admin_remote.rm_rf(conf_file)
             # pylint: enable=no-member
@@ -204,7 +220,7 @@ class DataDrivenUpgradeBase(TestBasic):
                     "Unknown octane action '{}', aborting".format(action))
         octane_cli_args = {
             'path': path,
-            'pwd': settings.KEYSTONE_CREDS['password']
+            'pwd': KEYSTONE_CREDS['password']
         }
         admin_remote = self.env.d_env.get_admin_remote()
         if 'backup' in action:
@@ -224,7 +240,7 @@ class DataDrivenUpgradeBase(TestBasic):
             # snapshot generating procedure can be broken
             admin_remote.download(
                 "/var/log/octane.log",
-                os.path.join(settings.LOGS_DIR,
+                os.path.join(LOGS_DIR,
                              "octane_{}_.log".format(os.path.basename(path))))
             raise
 
@@ -348,7 +364,7 @@ class DataDrivenUpgradeBase(TestBasic):
         self.env.sync_time()
         cluster_id = self.fuel_web.create_cluster(
             name=cluster_settings['name'],
-            mode=settings.DEPLOYMENT_MODE,
+            mode=DEPLOYMENT_MODE,
             settings=cluster_settings['settings']
         )
         if cluster_settings.get('plugin'):
@@ -384,12 +400,12 @@ class DataDrivenUpgradeBase(TestBasic):
         intermediate_snapshot = "prepare_upgrade_smoke_before_backup"
 
         assert_not_equal(
-            settings.KEYSTONE_CREDS['password'], 'admin',
+            KEYSTONE_CREDS['password'], 'admin',
             "Admin password was not changed, aborting execution")
 
         cluster_settings = {
-            'net_provider': settings.NEUTRON,
+            'net_provider': NEUTRON,
-            'net_segment_type': settings.NEUTRON_SEGMENT['vlan']
+            'net_segment_type': NEUTRON_SEGMENT['vlan']
         }
         cluster_settings.update(self.cluster_creds)
 
@@ -452,3 +468,27 @@ class DataDrivenUpgradeBase(TestBasic):
         if self.fuel_version <= StrictVersion('8.0'):
             cmd = "dockerctl shell cobbler {}".format(cmd)
         admin_remote.check_call(cmd)
+
+    def check_ostf(self, cluster_id, test_sets=None, timeout=30 * 60,
+                   ignore_known_issues=False, additional_ignored_issues=None):
+        """Run OSTF tests with the ignoring some test result
+        """
+        if additional_ignored_issues:
+            ignr_tests = additional_ignored_issues
+        else:
+            ignr_tests = []
+
+        if ignore_known_issues:
+            if (
+                LooseVersion(UPGRADE_FUEL_TO) >= LooseVersion('9.0') and
+                LooseVersion(UPGRADE_FUEL_FROM) == LooseVersion('7.0')
+            ):
+                mrg_set = set(ignr_tests) | set(
+                    itertools.chain(*self.IGNORED_OSTF_TESTS.values()))
+                ignr_tests = list(mrg_set)
+            else:
+                ignr_tests.extend(self.IGNORED_OSTF_TESTS[UPGRADE_FUEL_FROM])
+
+        self.fuel_web.run_ostf(cluster_id, test_sets=test_sets,
+                               should_fail=len(ignr_tests),
+                               failed_test_name=ignr_tests, timeout=timeout)

fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_ceph_ha.py

@@ -170,7 +170,7 @@ class UpgradeCephHA(DataDrivenUpgradeBase):
         # Live migration test could fail
         # https://bugs.launchpad.net/fuel/+bug/1471172
         # https://bugs.launchpad.net/fuel/+bug/1604749
-        self.fuel_web.run_ostf(cluster_id, should_fail=1)
+        self.check_ostf(cluster_id, ignore_known_issues=True)
 
         self.env.make_snapshot(self.snapshot_name, is_make=True)
         self.cleanup()
@@ -203,7 +203,7 @@ class UpgradeCephHA(DataDrivenUpgradeBase):
         self.fuel_web.assert_ha_services_ready(cluster_id)
         self.fuel_web.assert_os_services_ready(cluster_id)
         self.show_step(4)
-        self.fuel_web.run_ostf(cluster_id)
+        self.check_ostf(cluster_id, ignore_known_issues=True)
 
     @test(groups=['upgrade_ceph_ha_tests', 'upgrade_ceph_ha_scale_ceph'],
           depends_on_groups=['upgrade_ceph_ha_restore'])
@@ -232,4 +232,4 @@ class UpgradeCephHA(DataDrivenUpgradeBase):
         # LP 1562736 get_devops_node_by_nailgun_node is not working
         self.fuel_web.deploy_cluster_wait(cluster_id)
         self.show_step(5)
-        self.fuel_web.run_ostf(cluster_id)
+        self.check_ostf(cluster_id, ignore_known_issues=True)

fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_net_tmpl.py

@@ -134,9 +134,9 @@ class TestUpgradeNetworkTemplates(TestNetworkTemplatesBase,
             # Live migration test could fail
             # https://bugs.launchpad.net/fuel/+bug/1471172
             # https://bugs.launchpad.net/fuel/+bug/1604749
-            self.fuel_web.run_ostf(cluster_id=cluster_id,
+            self.check_ostf(cluster_id=cluster_id,
-                                   test_sets=['smoke', 'sanity', 'ha'],
+                            test_sets=['smoke', 'sanity', 'ha'],
-                                   should_fail=1)
+                            ignore_known_issues=True)
             self.show_step(11)
             self.check_ipconfig_for_template(cluster_id, network_template,
                                              networks)
@@ -195,8 +195,9 @@ class TestUpgradeNetworkTemplates(TestNetworkTemplatesBase,
         self.show_step(7)
         self.fuel_web.verify_network(cluster_id)
         self.show_step(8)
-        self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1,
+        self.check_ostf(cluster_id=cluster_id,
-                               test_sets=['smoke', 'sanity', 'ha'])
+                        test_sets=['smoke', 'sanity', 'ha'],
+                        ignore_known_issues=True)
         self.env.make_snapshot("upgrade_net_tmpl_restore", is_make=True)
 
     @test(depends_on_groups=["upgrade_net_tmpl_restore"],
@@ -229,8 +230,9 @@ class TestUpgradeNetworkTemplates(TestNetworkTemplatesBase,
         self.fuel_web.verify_network(cluster_id)
 
         self.show_step(5)
-        self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1,
+        self.check_ostf(cluster_id=cluster_id,
-                               test_sets=['smoke', 'sanity', 'ha'])
+                        test_sets=['smoke', 'sanity', 'ha'],
+                        ignore_known_issues=True)
         self.env.make_snapshot("reset_deploy_net_tmpl", is_make=True)
 
     @test(depends_on_groups=["upgrade_net_tmpl_restore"],
@@ -283,8 +285,9 @@ class TestUpgradeNetworkTemplates(TestNetworkTemplatesBase,
         self.fuel_web.verify_network(cluster_id)
 
         self.show_step(7)
-        self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1,
+        self.check_ostf(cluster_id=cluster_id,
-                               test_sets=['smoke', 'sanity', 'ha'])
+                        test_sets=['smoke', 'sanity', 'ha'],
+                        ignore_known_issues=True)
         self.env.make_snapshot("replace_controller_net_tmpl", is_make=True)
 
     @test(depends_on_groups=["replace_controller_net_tmpl"],
@@ -318,6 +321,7 @@ class TestUpgradeNetworkTemplates(TestNetworkTemplatesBase,
         self.fuel_web.verify_network(cluster_id)
 
         self.show_step(5)
-        self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1,
+        self.check_ostf(cluster_id=cluster_id,
-                               test_sets=['smoke', 'sanity', 'ha'])
+                        test_sets=['smoke', 'sanity', 'ha'],
+                        ignore_known_issues=True)
         self.env.make_snapshot("restart_node_net_tmpl", is_make=True)

fuelweb_test/tests/tests_upgrade/test_data_driven_upgrade_plugin.py

@@ -176,7 +176,7 @@ class UpgradePlugin(DataDrivenUpgradeBase):
         # Live migration test could fail
         # https://bugs.launchpad.net/fuel/+bug/1471172
         # https://bugs.launchpad.net/fuel/+bug/1604749
-        self.fuel_web.run_ostf(cluster_id, should_fail=1)
+        self.check_ostf(cluster_id, ignore_known_issues=True)
 
         self.env.make_snapshot(self.snapshot_name, is_make=True)
         self.cleanup()
@@ -212,4 +212,4 @@ class UpgradePlugin(DataDrivenUpgradeBase):
         self.show_step(4)
         self.fuel_web.deploy_cluster_wait(cluster_id)
         self.show_step(5)
-        self.fuel_web.run_ostf(cluster_id)
+        self.check_ostf(cluster_id, ignore_known_issues=True)