From 8b255a648c381bfcff0623ba97880855c7fa21eb Mon Sep 17 00:00:00 2001 From: Slawek Kaplonski Date: Thu, 14 Feb 2019 11:08:19 +0100 Subject: [PATCH] Add new test decorator skip_if_timeout In some cases our db migration tests which run on MySQL are failing with timeout and it happens due to slow VMs on which job is running. Sometimes it may also happen that timeout exception is raised in the middle of some sqlalchemy operations and sqlalchemy.InterfaceError is raised as last one. Details about this exception can be found in [1]. To avoid many rechecks because of this reason this patch introduces new decorator which is very similar to "unstable_test" but will skip test only if one of exceptions mentioned above will be raised. In all other cases it will fail test. That should be a bit more safe for us because we will not miss some other failures raised in those tests and will avoid rechecks because of this "well-known" reason described in related bug. [1] http://sqlalche.me/e/rvf5 Conflicts: neutron/tests/functional/db/test_migrations.py neutron/tests/base.py Change-Id: Ie291fda7d23a696aaa1160d126a3cf72b08c522f Related-Bug: #1687027 (cherry picked from commit c0fec676723649a0516cf3d4af0dccc0fe832095) (cherry picked from commit e6f22ce81c0a1130d45d290b185b736501e6dd1e) --- neutron/tests/base.py | 23 +++++++++++++++ .../tests/functional/db/test_migrations.py | 28 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/neutron/tests/base.py b/neutron/tests/base.py index 89e3f81a285..5f46d797be0 100644 --- a/neutron/tests/base.py +++ b/neutron/tests/base.py @@ -37,6 +37,7 @@ from oslo_utils import fileutils from oslo_utils import strutils from oslotest import base import six +from sqlalchemy import exc as sqlalchemy_exc import testtools from neutron._i18n import _ @@ -98,6 +99,28 @@ def sanitize_log_path(path): return path +def skip_if_timeout(reason): + def decor(f): + @functools.wraps(f) + def inner(self, *args, **kwargs): + try: + return f(self, *args, **kwargs) + except fixtures.TimeoutException: + msg = ("Timeout raised for test %s, skipping it " + "because of: %s") % (self.id(), reason) + raise self.skipTest(msg) + except sqlalchemy_exc.InterfaceError: + # In case of db tests very often TimeoutException is reason of + # some sqlalchemy InterfaceError exception and that is final + # raised exception which needs to be handled + msg = ("DB connection broken in test %s. It is very likely " + "that this happend because of test timeout. " + "Skipping test because of: %s") % (self.id(), reason) + raise self.skipTest(msg) + return inner + return decor + + def set_timeout(timeout): """Timeout decorator for test methods. diff --git a/neutron/tests/functional/db/test_migrations.py b/neutron/tests/functional/db/test_migrations.py index 2e4fd50630a..ce89607c8b4 100644 --- a/neutron/tests/functional/db/test_migrations.py +++ b/neutron/tests/functional/db/test_migrations.py @@ -350,6 +350,7 @@ class TestModelsMigrationsMysql(testlib_api.MySQLTestCaseMixin, _TestModelsMigrations, testlib_api.SqlTestCaseLight): + @test_base.skip_if_timeout("bug 1687027") def test_check_mysql_engine(self): engine = self.get_engine() cfg.CONF.set_override('connection', engine.url, group='database') @@ -367,6 +368,32 @@ class TestModelsMigrationsMysql(testlib_api.MySQLTestCaseMixin, and table != 'alembic_version'] self.assertEqual(0, len(res), "%s non InnoDB tables created" % res) + @test_base.skip_if_timeout("bug 1687027") + def test_upgrade_expand_branch(self): + super(TestModelsMigrationsMysql, self).test_upgrade_expand_branch() + + @test_base.skip_if_timeout("bug 1687027") + def test_upgrade_contract_branch(self): + super(TestModelsMigrationsMysql, self).test_upgrade_contract_branch() + + @test_base.skip_if_timeout("bug 1687027") + def test_branches(self): + super(TestModelsMigrationsMysql, self).test_branches() + + @test_base.skip_if_timeout("bug 1687027") + def test_has_offline_migrations_pending_contract_scripts(self): + super(TestModelsMigrationsMysql, + self).test_has_offline_migrations_pending_contract_scripts() + + @test_base.skip_if_timeout("bug 1687027") + def test_has_offline_migrations_all_heads_upgraded(self): + super(TestModelsMigrationsMysql, + self).test_has_offline_migrations_all_heads_upgraded() + + @test_base.skip_if_timeout("bug 1687027") + def test_models_sync(self): + super(TestModelsMigrationsMysql, self).test_models_sync() + class TestModelsMigrationsPsql(testlib_api.PostgreSQLTestCaseMixin, _TestModelsMigrations, @@ -574,6 +601,7 @@ class TestWalkMigrationsMysql(testlib_api.MySQLTestCaseMixin, # on slow nodes than 'psycopg2' and because of that this increased # timeout is required only when for testing with 'mysql' backend. @test_base.set_timeout(600) + @test_base.skip_if_timeout("bug 1687027") def test_walk_versions(self): super(TestWalkMigrationsMysql, self).test_walk_versions()