From 83181a97032652f24918c0db39aa0de1bd180a0a Mon Sep 17 00:00:00 2001 From: Giulio Fidente Date: Tue, 1 Oct 2013 06:02:24 +0200 Subject: [PATCH] introduces skip_because decorator this change introduces a skip_because decorator which accepts two args a bug and a condition; also updates the skip tracker accordingly Change-Id: If53f2ef81d6bddbce284267216254b467046855f --- HACKING.rst | 23 +++++++++++-------- tempest/api/compute/admin/test_flavors.py | 5 ++-- .../compute/images/test_images_oneserver.py | 3 ++- .../test_security_group_rules.py | 11 ++++----- .../security_groups/test_security_groups.py | 17 +++++++------- .../servers/test_list_server_filters.py | 9 ++++---- .../compute/servers/test_server_actions.py | 5 ++-- .../servers/test_virtual_interfaces.py | 6 ++--- .../api/object_storage/test_container_sync.py | 5 ++-- .../api/object_storage/test_object_expiry.py | 5 ++-- tempest/hacking/checks.py | 18 --------------- tempest/test.py | 15 ++++++++++++ .../thirdparty/boto/test_ec2_instance_run.py | 6 ++--- tempest/thirdparty/boto/test_ec2_keys.py | 5 ++-- tempest/thirdparty/boto/test_ec2_network.py | 7 +++--- tempest/thirdparty/boto/test_s3_buckets.py | 5 ++-- tools/skip_tracker.py | 4 ++-- 17 files changed, 72 insertions(+), 77 deletions(-) diff --git a/HACKING.rst b/HACKING.rst index 499b43665f..7871f604ae 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -8,16 +8,6 @@ Tempest Coding Guide Tempest Specific Commandments ------------------------------ -[T101] If a test is broken because of a bug it is appropriate to skip the test until -bug has been fixed. However, the skip message should be formatted so that -Tempest's skip tracking tool can watch the bug status. The skip message should -contain the string 'Bug' immediately followed by a space. Then the bug number -should be included in the message '#' in front of the number. - -Example:: - - @testtools.skip("Skipped until the Bug #980688 is resolved") - - [T102] Cannot import OpenStack python clients in tempest/api tests - [T103] tempest/tests is deprecated - [T104] Scenario tests require a services decorator @@ -115,6 +105,19 @@ name. For example, any test that make an api call to a service other than nova in tempest.api.compute would require a service tag for those services, however they do not need to be tagged as compute. +Test skips because of Known Bugs +-------------------------------- + +If a test is broken because of a bug it is appropriate to skip the test until +bug has been fixed. You should use the skip_because decorator so that +Tempest's skip tracking tool can watch the bug status. + +Example:: + + @skip_because(bug="980688") + def test_this_and_that(self): + ... + Guidelines ---------- - Do not submit changesets with only testcases which are skipped as diff --git a/tempest/api/compute/admin/test_flavors.py b/tempest/api/compute/admin/test_flavors.py index 3ef75bae7b..004268ed3b 100644 --- a/tempest/api/compute/admin/test_flavors.py +++ b/tempest/api/compute/admin/test_flavors.py @@ -15,14 +15,13 @@ # License for the specific language governing permissions and limitations # under the License. -import testtools - from tempest.api import compute from tempest.api.compute import base from tempest.common.utils.data_utils import rand_int_id from tempest.common.utils.data_utils import rand_name from tempest import exceptions from tempest.test import attr +from tempest.test import skip_because class FlavorsAdminTestJSON(base.BaseComputeAdminTest): @@ -195,7 +194,7 @@ class FlavorsAdminTestJSON(base.BaseComputeAdminTest): flag = True self.assertTrue(flag) - @testtools.skip("Skipped until the Bug #1209101 is resolved") + @skip_because(bug="1209101") @attr(type='gate') def test_list_non_public_flavor(self): # Create a flavor with os-flavor-access:is_public false should diff --git a/tempest/api/compute/images/test_images_oneserver.py b/tempest/api/compute/images/test_images_oneserver.py index 7df90105f4..800b2de5d4 100644 --- a/tempest/api/compute/images/test_images_oneserver.py +++ b/tempest/api/compute/images/test_images_oneserver.py @@ -24,6 +24,7 @@ from tempest.common.utils.data_utils import parse_image_id from tempest.common.utils.data_utils import rand_name from tempest import exceptions from tempest.test import attr +from tempest.test import skip_because class ImagesOneServerTestJSON(base.BaseComputeTest): @@ -64,7 +65,7 @@ class ImagesOneServerTestJSON(base.BaseComputeTest): cls.alt_manager = clients.AltManager() cls.alt_client = cls.alt_manager.images_client - @testtools.skip("Skipped until the Bug #1006725 is resolved.") + @skip_because(bug="1006725") @attr(type=['negative', 'gate']) def test_create_image_specify_multibyte_character_image_name(self): # Return an error if the image name has multi-byte characters diff --git a/tempest/api/compute/security_groups/test_security_group_rules.py b/tempest/api/compute/security_groups/test_security_group_rules.py index 61db61db36..cbc00806db 100644 --- a/tempest/api/compute/security_groups/test_security_group_rules.py +++ b/tempest/api/compute/security_groups/test_security_group_rules.py @@ -15,13 +15,12 @@ # License for the specific language governing permissions and limitations # under the License. -import testtools - from tempest.api.compute import base from tempest.common.utils.data_utils import rand_name from tempest import config from tempest import exceptions from tempest.test import attr +from tempest.test import skip_because class SecurityGroupRulesTestJSON(base.BaseComputeTest): @@ -94,8 +93,8 @@ class SecurityGroupRulesTestJSON(base.BaseComputeTest): self.addCleanup(self.client.delete_security_group_rule, rule['id']) self.assertEqual(200, resp.status) - @testtools.skipIf(config.TempestConfig().service_available.neutron, - "Skipped until the Bug #1182384 is resolved") + @skip_because(bug="1182384", + condition=config.TempestConfig().service_available.neutron) @attr(type=['negative', 'gate']) def test_security_group_rules_create_with_invalid_id(self): # Negative test: Creation of Security Group rule should FAIL @@ -186,8 +185,8 @@ class SecurityGroupRulesTestJSON(base.BaseComputeTest): self.client.create_security_group_rule, secgroup_id, ip_protocol, from_port, to_port) - @testtools.skipIf(config.TempestConfig().service_available.neutron, - "Skipped until the Bug #1182384 is resolved") + @skip_because(bug="1182384", + condition=config.TempestConfig().service_available.neutron) @attr(type=['negative', 'gate']) def test_security_group_rules_delete_with_invalid_id(self): # Negative test: Deletion of Security Group rule should be FAIL diff --git a/tempest/api/compute/security_groups/test_security_groups.py b/tempest/api/compute/security_groups/test_security_groups.py index 5cca3b2596..fba2f538da 100644 --- a/tempest/api/compute/security_groups/test_security_groups.py +++ b/tempest/api/compute/security_groups/test_security_groups.py @@ -22,6 +22,7 @@ from tempest.common.utils.data_utils import rand_name from tempest import config from tempest import exceptions from tempest.test import attr +from tempest.test import skip_because class SecurityGroupsTestJSON(base.BaseComputeTest): @@ -107,8 +108,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest): "The fetched Security Group is different " "from the created Group") - @testtools.skipIf(config.TempestConfig().service_available.neutron, - "Skipped until the Bug #1182384 is resolved") + @skip_because(bug="1182384", + condition=config.TempestConfig().service_available.neutron) @attr(type=['negative', 'gate']) def test_security_group_get_nonexistant_group(self): # Negative test:Should not be able to GET the details @@ -125,8 +126,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest): self.assertRaises(exceptions.NotFound, self.client.get_security_group, non_exist_id) - @testtools.skipIf(config.TempestConfig().service_available.neutron, - "Skipped until the Bug #1161411 is resolved") + @skip_because(bug="1161411", + condition=config.TempestConfig().service_available.neutron) @attr(type=['negative', 'gate']) def test_security_group_create_with_invalid_group_name(self): # Negative test: Security Group should not be created with group name @@ -145,8 +146,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest): self.client.create_security_group, s_name, s_description) - @testtools.skipIf(config.TempestConfig().service_available.neutron, - "Skipped until the Bug #1161411 is resolved") + @skip_because(bug="1161411", + condition=config.TempestConfig().service_available.neutron) @attr(type=['negative', 'gate']) def test_security_group_create_with_invalid_group_description(self): # Negative test:Security Group should not be created with description @@ -197,8 +198,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest): self.client.delete_security_group, default_security_group_id) - @testtools.skipIf(config.TempestConfig().service_available.neutron, - "Skipped until the Bug #1182384 is resolved") + @skip_because(bug="1182384", + condition=config.TempestConfig().service_available.neutron) @attr(type=['negative', 'gate']) def test_delete_nonexistant_security_group(self): # Negative test:Deletion of a non-existent Security Group should Fail diff --git a/tempest/api/compute/servers/test_list_server_filters.py b/tempest/api/compute/servers/test_list_server_filters.py index a56bdf3699..c469827bc1 100644 --- a/tempest/api/compute/servers/test_list_server_filters.py +++ b/tempest/api/compute/servers/test_list_server_filters.py @@ -15,14 +15,13 @@ # License for the specific language governing permissions and limitations # under the License. -import testtools - from tempest.api.compute import base from tempest.api import utils from tempest.common.utils.data_utils import rand_name from tempest import config from tempest import exceptions from tempest.test import attr +from tempest.test import skip_because class ListServerFiltersTestJSON(base.BaseComputeTest): @@ -205,7 +204,7 @@ class ListServerFiltersTestJSON(base.BaseComputeTest): self.assertNotIn(self.s2_name, map(lambda x: x['name'], servers)) self.assertNotIn(self.s3_name, map(lambda x: x['name'], servers)) - @testtools.skip('Skipped until the Bug #1170718 is resolved.') + @skip_because(bug="1170718") @attr(type='gate') def test_list_servers_filtered_by_ip(self): # Filter servers by ip @@ -219,8 +218,8 @@ class ListServerFiltersTestJSON(base.BaseComputeTest): self.assertNotIn(self.s2_name, map(lambda x: x['name'], servers)) self.assertNotIn(self.s3_name, map(lambda x: x['name'], servers)) - @testtools.skipIf(config.TempestConfig().service_available.neutron, - "Skipped until the Bug #1182883 is resolved") + @skip_because(bug="1182883", + condition=config.TempestConfig().service_available.neutron) @attr(type='gate') def test_list_servers_filtered_by_ip_regex(self): # Filter servers by regex ip diff --git a/tempest/api/compute/servers/test_server_actions.py b/tempest/api/compute/servers/test_server_actions.py index 0d5a8fa85b..e9defe5933 100644 --- a/tempest/api/compute/servers/test_server_actions.py +++ b/tempest/api/compute/servers/test_server_actions.py @@ -27,6 +27,7 @@ from tempest.common.utils.linux.remote_client import RemoteClient import tempest.config from tempest import exceptions from tempest.test import attr +from tempest.test import skip_because class ServerActionsTestJSON(base.BaseComputeTest): @@ -86,7 +87,7 @@ class ServerActionsTestJSON(base.BaseComputeTest): new_boot_time = linux_client.get_boot_time() self.assertGreater(new_boot_time, boot_time) - @testtools.skip('Skipped until the Bug #1014647 is resolved.') + @skip_because(bug="1014647") @attr(type='smoke') def test_reboot_server_soft(self): # The server should be signaled to reboot gracefully @@ -250,7 +251,7 @@ class ServerActionsTestJSON(base.BaseComputeTest): self.servers_client.get_console_output, '!@#$%^&*()', 10) - @testtools.skip('Skipped until the Bug #1014683 is resolved.') + @skip_because(bug="1014683") @attr(type='gate') def test_get_console_output_server_id_in_reboot_status(self): # Positive test:Should be able to GET the console output diff --git a/tempest/api/compute/servers/test_virtual_interfaces.py b/tempest/api/compute/servers/test_virtual_interfaces.py index b743a858ee..2c7ff32470 100644 --- a/tempest/api/compute/servers/test_virtual_interfaces.py +++ b/tempest/api/compute/servers/test_virtual_interfaces.py @@ -16,13 +16,13 @@ # under the License. import netaddr -import testtools from tempest.api.compute import base from tempest.common.utils.data_utils import rand_name from tempest import config from tempest import exceptions from tempest.test import attr +from tempest.test import skip_because class VirtualInterfacesTestJSON(base.BaseComputeTest): @@ -37,8 +37,8 @@ class VirtualInterfacesTestJSON(base.BaseComputeTest): resp, server = cls.create_server(wait_until='ACTIVE') cls.server_id = server['id'] - @testtools.skipIf(CONF.service_available.neutron, "Not implemented by " + - "Neutron. Skipped until the Bug #1183436 is resolved.") + @skip_because(bug="1183436", + condition=CONF.service_available.neutron) @attr(type='gate') def test_list_virtual_interfaces(self): # Positive test:Should be able to GET the virtual interfaces list diff --git a/tempest/api/object_storage/test_container_sync.py b/tempest/api/object_storage/test_container_sync.py index a43b2b584b..ff9f7bfa3e 100644 --- a/tempest/api/object_storage/test_container_sync.py +++ b/tempest/api/object_storage/test_container_sync.py @@ -17,11 +17,10 @@ import time -import testtools - from tempest.api.object_storage import base from tempest.common.utils.data_utils import rand_name from tempest.test import attr +from tempest.test import skip_because class ContainerSyncTest(base.BaseObjectTest): @@ -52,7 +51,7 @@ class ContainerSyncTest(base.BaseObjectTest): cls.delete_containers(cls.containers, client[0], client[1]) super(ContainerSyncTest, cls).tearDownClass() - @testtools.skip('Skipped until the Bug #1093743 is resolved.') + @skip_because(bug="1093743") @attr(type='gate') def test_container_synchronization(self): # container to container synchronization diff --git a/tempest/api/object_storage/test_object_expiry.py b/tempest/api/object_storage/test_object_expiry.py index db38401670..cb52d88c4c 100644 --- a/tempest/api/object_storage/test_object_expiry.py +++ b/tempest/api/object_storage/test_object_expiry.py @@ -17,13 +17,12 @@ import time -import testtools - from tempest.api.object_storage import base from tempest.common.utils.data_utils import arbitrary_string from tempest.common.utils.data_utils import rand_name from tempest import exceptions from tempest.test import attr +from tempest.test import skip_because class ObjectExpiryTest(base.BaseObjectTest): @@ -43,7 +42,7 @@ class ObjectExpiryTest(base.BaseObjectTest): cls.delete_containers([cls.container_name]) super(ObjectExpiryTest, cls).tearDownClass() - @testtools.skip('Skipped until the Bug #1069849 is resolved.') + @skip_because(bug="1069849") @attr(type='gate') def test_get_object_after_expiry_time(self): # TODO(harika-vakadi): similar test case has to be created for diff --git a/tempest/hacking/checks.py b/tempest/hacking/checks.py index aa972114d6..4c1c107651 100644 --- a/tempest/hacking/checks.py +++ b/tempest/hacking/checks.py @@ -19,28 +19,11 @@ import re PYTHON_CLIENTS = ['cinder', 'glance', 'keystone', 'nova', 'swift', 'neutron'] -SKIP_DECORATOR_RE = re.compile(r'\s*@testtools.skip\((.*)\)') -SKIP_STR_RE = re.compile(r'.*Bug #\d+.*') PYTHON_CLIENT_RE = re.compile('import (%s)client' % '|'.join(PYTHON_CLIENTS)) TEST_DEFINITION = re.compile(r'^\s*def test.*') SCENARIO_DECORATOR = re.compile(r'\s*@.*services\(') -def skip_bugs(physical_line): - """Check skip lines for proper bug entries - - T101: skips must contain "Bug #" - """ - - res = SKIP_DECORATOR_RE.match(physical_line) - if res: - content = res.group(1) - res = SKIP_STR_RE.match(content) - if not res: - return (physical_line.find(content), - 'T101: skips must contain "Bug #"') - - def import_no_clients_in_api(physical_line, filename): """Check for client imports from tempest/api tests @@ -70,6 +53,5 @@ def scenario_tests_need_service_tags(physical_line, filename, def factory(register): - register(skip_bugs) register(import_no_clients_in_api) register(scenario_tests_need_service_tags) diff --git a/tempest/test.py b/tempest/test.py index 6acb1c902d..0c7c9167df 100644 --- a/tempest/test.py +++ b/tempest/test.py @@ -103,6 +103,21 @@ def stresstest(*args, **kwargs): return decorator +def skip_because(*args, **kwargs): + """A decorator useful to skip tests hitting known bugs + + @param bug: bug number causing the test to skip + @param condition: optional condition to be True for the skip to have place + """ + def decorator(f): + if "bug" in kwargs: + if "condition" not in kwargs or kwargs["condition"] is True: + msg = "Skipped until Bug: %s is resolved." % kwargs["bug"] + raise testtools.TestCase.skipException(msg) + return f + return decorator + + # there is a mis-match between nose and testtools for older pythons. # testtools will set skipException to be either # unittest.case.SkipTest, unittest2.case.SkipTest or an internal skip diff --git a/tempest/thirdparty/boto/test_ec2_instance_run.py b/tempest/thirdparty/boto/test_ec2_instance_run.py index bce544ad07..0f455e1ae8 100644 --- a/tempest/thirdparty/boto/test_ec2_instance_run.py +++ b/tempest/thirdparty/boto/test_ec2_instance_run.py @@ -16,7 +16,6 @@ # under the License. from boto import exception -import testtools from tempest import clients from tempest.common.utils.data_utils import rand_name @@ -24,6 +23,7 @@ from tempest.common.utils.linux.remote_client import RemoteClient from tempest import exceptions from tempest.openstack.common import log as logging from tempest.test import attr +from tempest.test import skip_because from tempest.thirdparty.boto.test import BotoTestCase from tempest.thirdparty.boto.utils.s3 import s3_upload_dir from tempest.thirdparty.boto.utils.wait import re_search_wait @@ -206,8 +206,8 @@ class InstanceRunTest(BotoTestCase): instance.terminate() self.cancelResourceCleanUp(rcuk) + @skip_because(bug="1098891") @attr(type='smoke') - @testtools.skip("Skipped until the Bug #1098891 is resolved") def test_run_terminate_instance(self): # EC2 run, terminate immediately image_ami = self.ec2_client.get_image(self.images["ami"] @@ -233,7 +233,7 @@ class InstanceRunTest(BotoTestCase): # NOTE(afazekas): doctored test case, # with normal validation it would fail - @testtools.skip("Skipped until the Bug #1182679 is resolved.") + @skip_because(bug="1182679") @attr(type='smoke') def test_integration_1(self): # EC2 1. integration test (not strict) diff --git a/tempest/thirdparty/boto/test_ec2_keys.py b/tempest/thirdparty/boto/test_ec2_keys.py index 85a99c0c1e..5592d8cd1b 100644 --- a/tempest/thirdparty/boto/test_ec2_keys.py +++ b/tempest/thirdparty/boto/test_ec2_keys.py @@ -15,11 +15,10 @@ # License for the specific language governing permissions and limitations # under the License. -import testtools - from tempest import clients from tempest.common.utils.data_utils import rand_name from tempest.test import attr +from tempest.test import skip_because from tempest.thirdparty.boto.test import BotoTestCase @@ -47,8 +46,8 @@ class EC2KeysTest(BotoTestCase): self.assertTrue(compare_key_pairs(keypair, self.client.get_key_pair(key_name))) + @skip_because(bug="1072318") @attr(type='smoke') - @testtools.skip("Skipped until the Bug #1072318 is resolved") def test_delete_ec2_keypair(self): # EC2 delete KeyPair key_name = rand_name("keypair-") diff --git a/tempest/thirdparty/boto/test_ec2_network.py b/tempest/thirdparty/boto/test_ec2_network.py index ae8c3c20c2..b4949c8387 100644 --- a/tempest/thirdparty/boto/test_ec2_network.py +++ b/tempest/thirdparty/boto/test_ec2_network.py @@ -15,10 +15,9 @@ # License for the specific language governing permissions and limitations # under the License. -import testtools - from tempest import clients from tempest.test import attr +from tempest.test import skip_because from tempest.thirdparty.boto.test import BotoTestCase @@ -30,8 +29,8 @@ class EC2NetworkTest(BotoTestCase): cls.os = clients.Manager() cls.client = cls.os.ec2api_client -# Note(afazekas): these tests for things duable without an instance - @testtools.skip("Skipped until the Bug #1080406 is resolved") + # Note(afazekas): these tests for things duable without an instance + @skip_because(bug="1080406") @attr(type='smoke') def test_disassociate_not_associated_floating_ip(self): # EC2 disassociate not associated floating ip diff --git a/tempest/thirdparty/boto/test_s3_buckets.py b/tempest/thirdparty/boto/test_s3_buckets.py index e43cbaa882..1a8fbe0fd3 100644 --- a/tempest/thirdparty/boto/test_s3_buckets.py +++ b/tempest/thirdparty/boto/test_s3_buckets.py @@ -15,11 +15,10 @@ # License for the specific language governing permissions and limitations # under the License. -import testtools - from tempest import clients from tempest.common.utils.data_utils import rand_name from tempest.test import attr +from tempest.test import skip_because from tempest.thirdparty.boto.test import BotoTestCase @@ -31,7 +30,7 @@ class S3BucketsTest(BotoTestCase): cls.os = clients.Manager() cls.client = cls.os.s3_client - @testtools.skip("Skipped until the Bug #1076965 is resolved") + @skip_because(bug="1076965") @attr(type='smoke') def test_create_and_get_delete_bucket(self): # S3 Create, get and delete bucket diff --git a/tools/skip_tracker.py b/tools/skip_tracker.py index c38ccdb023..ffaf134eb0 100755 --- a/tools/skip_tracker.py +++ b/tools/skip_tracker.py @@ -61,8 +61,8 @@ def find_skips_in_file(path): """ Return the skip tuples in a test file """ - BUG_RE = re.compile(r'.*skip.*bug:*\s*\#*(\d+)', re.IGNORECASE) - DEF_RE = re.compile(r'.*def (\w+)\(') + BUG_RE = re.compile(r'\s*@.*skip_because\(bug=[\'"](\d+)[\'"]') + DEF_RE = re.compile(r'\s*def (\w+)\(') bug_found = False results = [] lines = open(path, 'rb').readlines()