introduces skip_because decorator

this change introduces a skip_because decorator which accepts two args
a bug and a condition; also updates the skip tracker accordingly

Change-Id: If53f2ef81d6bddbce284267216254b467046855f

HACKING.rst

@@ -8,16 +8,6 @@ Tempest Coding Guide
 Tempest Specific Commandments
 ------------------------------
 
-[T101] If a test is broken because of a bug it is appropriate to skip the test until
-bug has been fixed. However, the skip message should be formatted so that
-Tempest's skip tracking tool can watch the bug status. The skip message should
-contain the string 'Bug' immediately followed by a space. Then the bug number
-should be included in the message '#' in front of the number.
-
-Example::
-
-  @testtools.skip("Skipped until the Bug #980688 is resolved")
-
 - [T102] Cannot import OpenStack python clients in tempest/api tests
 - [T103] tempest/tests is deprecated
 - [T104] Scenario tests require a services decorator
@@ -115,6 +105,19 @@ name. For example, any test that make an api call to a service other than nova
 in tempest.api.compute would require a service tag for those services, however
 they do not need to be tagged as compute.
 
+Test skips because of Known Bugs
+--------------------------------
+
+If a test is broken because of a bug it is appropriate to skip the test until
+bug has been fixed. You should use the skip_because decorator so that
+Tempest's skip tracking tool can watch the bug status.
+
+Example::
+
+  @skip_because(bug="980688")
+  def test_this_and_that(self):
+    ...
+
 Guidelines
 ----------
 - Do not submit changesets with only testcases which are skipped as

tempest/api/compute/admin/test_flavors.py

@@ -15,14 +15,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import testtools
-
 from tempest.api import compute
 from tempest.api.compute import base
 from tempest.common.utils.data_utils import rand_int_id
 from tempest.common.utils.data_utils import rand_name
 from tempest import exceptions
 from tempest.test import attr
+from tempest.test import skip_because
 
 
 class FlavorsAdminTestJSON(base.BaseComputeAdminTest):
@@ -195,7 +194,7 @@ class FlavorsAdminTestJSON(base.BaseComputeAdminTest):
                 flag = True
         self.assertTrue(flag)
 
-    @testtools.skip("Skipped until the Bug #1209101 is resolved")
+    @skip_because(bug="1209101")
     @attr(type='gate')
     def test_list_non_public_flavor(self):
         # Create a flavor with os-flavor-access:is_public false should

tempest/api/compute/images/test_images_oneserver.py

@@ -24,6 +24,7 @@ from tempest.common.utils.data_utils import parse_image_id
 from tempest.common.utils.data_utils import rand_name
 from tempest import exceptions
 from tempest.test import attr
+from tempest.test import skip_because
 
 
 class ImagesOneServerTestJSON(base.BaseComputeTest):
@@ -64,7 +65,7 @@ class ImagesOneServerTestJSON(base.BaseComputeTest):
                 cls.alt_manager = clients.AltManager()
             cls.alt_client = cls.alt_manager.images_client
 
-    @testtools.skip("Skipped until the Bug #1006725 is resolved.")
+    @skip_because(bug="1006725")
     @attr(type=['negative', 'gate'])
     def test_create_image_specify_multibyte_character_image_name(self):
         # Return an error if the image name has multi-byte characters

tempest/api/compute/security_groups/test_security_group_rules.py

@@ -15,13 +15,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import testtools
-
 from tempest.api.compute import base
 from tempest.common.utils.data_utils import rand_name
 from tempest import config
 from tempest import exceptions
 from tempest.test import attr
+from tempest.test import skip_because
 
 
 class SecurityGroupRulesTestJSON(base.BaseComputeTest):
@@ -94,8 +93,8 @@ class SecurityGroupRulesTestJSON(base.BaseComputeTest):
         self.addCleanup(self.client.delete_security_group_rule, rule['id'])
         self.assertEqual(200, resp.status)
 
-    @testtools.skipIf(config.TempestConfig().service_available.neutron,
+    @skip_because(bug="1182384",
-                      "Skipped until the Bug #1182384 is resolved")
+                  condition=config.TempestConfig().service_available.neutron)
     @attr(type=['negative', 'gate'])
     def test_security_group_rules_create_with_invalid_id(self):
         # Negative test: Creation of Security Group rule should FAIL
@@ -186,8 +185,8 @@ class SecurityGroupRulesTestJSON(base.BaseComputeTest):
                           self.client.create_security_group_rule,
                           secgroup_id, ip_protocol, from_port, to_port)
 
-    @testtools.skipIf(config.TempestConfig().service_available.neutron,
+    @skip_because(bug="1182384",
-                      "Skipped until the Bug #1182384 is resolved")
+                  condition=config.TempestConfig().service_available.neutron)
     @attr(type=['negative', 'gate'])
     def test_security_group_rules_delete_with_invalid_id(self):
         # Negative test: Deletion of Security Group rule should be FAIL

tempest/api/compute/security_groups/test_security_groups.py

@@ -22,6 +22,7 @@ from tempest.common.utils.data_utils import rand_name
 from tempest import config
 from tempest import exceptions
 from tempest.test import attr
+from tempest.test import skip_because
 
 
 class SecurityGroupsTestJSON(base.BaseComputeTest):
@@ -107,8 +108,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest):
                          "The fetched Security Group is different "
                          "from the created Group")
 
-    @testtools.skipIf(config.TempestConfig().service_available.neutron,
+    @skip_because(bug="1182384",
-                      "Skipped until the Bug #1182384 is resolved")
+                  condition=config.TempestConfig().service_available.neutron)
     @attr(type=['negative', 'gate'])
     def test_security_group_get_nonexistant_group(self):
         # Negative test:Should not be able to GET the details
@@ -125,8 +126,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest):
         self.assertRaises(exceptions.NotFound, self.client.get_security_group,
                           non_exist_id)
 
-    @testtools.skipIf(config.TempestConfig().service_available.neutron,
+    @skip_because(bug="1161411",
-                      "Skipped until the Bug #1161411 is resolved")
+                  condition=config.TempestConfig().service_available.neutron)
     @attr(type=['negative', 'gate'])
     def test_security_group_create_with_invalid_group_name(self):
         # Negative test: Security Group should not be created with group name
@@ -145,8 +146,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest):
                           self.client.create_security_group, s_name,
                           s_description)
 
-    @testtools.skipIf(config.TempestConfig().service_available.neutron,
+    @skip_because(bug="1161411",
-                      "Skipped until the Bug #1161411 is resolved")
+                  condition=config.TempestConfig().service_available.neutron)
     @attr(type=['negative', 'gate'])
     def test_security_group_create_with_invalid_group_description(self):
         # Negative test:Security Group should not be created with description
@@ -197,8 +198,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest):
                           self.client.delete_security_group,
                           default_security_group_id)
 
-    @testtools.skipIf(config.TempestConfig().service_available.neutron,
+    @skip_because(bug="1182384",
-                      "Skipped until the Bug #1182384 is resolved")
+                  condition=config.TempestConfig().service_available.neutron)
     @attr(type=['negative', 'gate'])
     def test_delete_nonexistant_security_group(self):
         # Negative test:Deletion of a non-existent Security Group should Fail

tempest/api/compute/servers/test_list_server_filters.py

@@ -15,14 +15,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import testtools
-
 from tempest.api.compute import base
 from tempest.api import utils
 from tempest.common.utils.data_utils import rand_name
 from tempest import config
 from tempest import exceptions
 from tempest.test import attr
+from tempest.test import skip_because
 
 
 class ListServerFiltersTestJSON(base.BaseComputeTest):
@@ -205,7 +204,7 @@ class ListServerFiltersTestJSON(base.BaseComputeTest):
         self.assertNotIn(self.s2_name, map(lambda x: x['name'], servers))
         self.assertNotIn(self.s3_name, map(lambda x: x['name'], servers))
 
-    @testtools.skip('Skipped until the Bug #1170718 is resolved.')
+    @skip_because(bug="1170718")
     @attr(type='gate')
     def test_list_servers_filtered_by_ip(self):
         # Filter servers by ip
@@ -219,8 +218,8 @@ class ListServerFiltersTestJSON(base.BaseComputeTest):
         self.assertNotIn(self.s2_name, map(lambda x: x['name'], servers))
         self.assertNotIn(self.s3_name, map(lambda x: x['name'], servers))
 
-    @testtools.skipIf(config.TempestConfig().service_available.neutron,
+    @skip_because(bug="1182883",
-                      "Skipped until the Bug #1182883 is resolved")
+                  condition=config.TempestConfig().service_available.neutron)
     @attr(type='gate')
     def test_list_servers_filtered_by_ip_regex(self):
         # Filter servers by regex ip

tempest/api/compute/servers/test_server_actions.py

@@ -27,6 +27,7 @@ from tempest.common.utils.linux.remote_client import RemoteClient
 import tempest.config
 from tempest import exceptions
 from tempest.test import attr
+from tempest.test import skip_because
 
 
 class ServerActionsTestJSON(base.BaseComputeTest):
@@ -86,7 +87,7 @@ class ServerActionsTestJSON(base.BaseComputeTest):
             new_boot_time = linux_client.get_boot_time()
             self.assertGreater(new_boot_time, boot_time)
 
-    @testtools.skip('Skipped until the Bug #1014647 is resolved.')
+    @skip_because(bug="1014647")
     @attr(type='smoke')
     def test_reboot_server_soft(self):
         # The server should be signaled to reboot gracefully
@@ -250,7 +251,7 @@ class ServerActionsTestJSON(base.BaseComputeTest):
                           self.servers_client.get_console_output,
                           '!@#$%^&*()', 10)
 
-    @testtools.skip('Skipped until the Bug #1014683 is resolved.')
+    @skip_because(bug="1014683")
     @attr(type='gate')
     def test_get_console_output_server_id_in_reboot_status(self):
         # Positive test:Should be able to GET the console output

tempest/api/compute/servers/test_virtual_interfaces.py

@@ -16,13 +16,13 @@
 #    under the License.
 
 import netaddr
-import testtools
 
 from tempest.api.compute import base
 from tempest.common.utils.data_utils import rand_name
 from tempest import config
 from tempest import exceptions
 from tempest.test import attr
+from tempest.test import skip_because
 
 
 class VirtualInterfacesTestJSON(base.BaseComputeTest):
@@ -37,8 +37,8 @@ class VirtualInterfacesTestJSON(base.BaseComputeTest):
         resp, server = cls.create_server(wait_until='ACTIVE')
         cls.server_id = server['id']
 
-    @testtools.skipIf(CONF.service_available.neutron, "Not implemented by " +
+    @skip_because(bug="1183436",
-                      "Neutron. Skipped until the Bug #1183436 is resolved.")
+                  condition=CONF.service_available.neutron)
     @attr(type='gate')
     def test_list_virtual_interfaces(self):
         # Positive test:Should be able to GET the virtual interfaces list

tempest/api/object_storage/test_container_sync.py

@@ -17,11 +17,10 @@
 
 import time
 
-import testtools
-
 from tempest.api.object_storage import base
 from tempest.common.utils.data_utils import rand_name
 from tempest.test import attr
+from tempest.test import skip_because
 
 
 class ContainerSyncTest(base.BaseObjectTest):
@@ -52,7 +51,7 @@ class ContainerSyncTest(base.BaseObjectTest):
             cls.delete_containers(cls.containers, client[0], client[1])
         super(ContainerSyncTest, cls).tearDownClass()
 
-    @testtools.skip('Skipped until the Bug #1093743 is resolved.')
+    @skip_because(bug="1093743")
     @attr(type='gate')
     def test_container_synchronization(self):
         # container to container synchronization

tempest/api/object_storage/test_object_expiry.py

@@ -17,13 +17,12 @@
 
 import time
 
-import testtools
-
 from tempest.api.object_storage import base
 from tempest.common.utils.data_utils import arbitrary_string
 from tempest.common.utils.data_utils import rand_name
 from tempest import exceptions
 from tempest.test import attr
+from tempest.test import skip_because
 
 
 class ObjectExpiryTest(base.BaseObjectTest):
@@ -43,7 +42,7 @@ class ObjectExpiryTest(base.BaseObjectTest):
         cls.delete_containers([cls.container_name])
         super(ObjectExpiryTest, cls).tearDownClass()
 
-    @testtools.skip('Skipped until the Bug #1069849 is resolved.')
+    @skip_because(bug="1069849")
     @attr(type='gate')
     def test_get_object_after_expiry_time(self):
         # TODO(harika-vakadi): similar test case has to be created for

tempest/hacking/checks.py

@@ -19,28 +19,11 @@ import re
 
 PYTHON_CLIENTS = ['cinder', 'glance', 'keystone', 'nova', 'swift', 'neutron']
 
-SKIP_DECORATOR_RE = re.compile(r'\s*@testtools.skip\((.*)\)')
-SKIP_STR_RE = re.compile(r'.*Bug #\d+.*')
 PYTHON_CLIENT_RE = re.compile('import (%s)client' % '|'.join(PYTHON_CLIENTS))
 TEST_DEFINITION = re.compile(r'^\s*def test.*')
 SCENARIO_DECORATOR = re.compile(r'\s*@.*services\(')
 
 
-def skip_bugs(physical_line):
-    """Check skip lines for proper bug entries
-
-    T101: skips must contain "Bug #<bug_number>"
-    """
-
-    res = SKIP_DECORATOR_RE.match(physical_line)
-    if res:
-        content = res.group(1)
-        res = SKIP_STR_RE.match(content)
-        if not res:
-            return (physical_line.find(content),
-                    'T101: skips must contain "Bug #<bug_number>"')
-
-
 def import_no_clients_in_api(physical_line, filename):
     """Check for client imports from tempest/api tests
 
@@ -70,6 +53,5 @@ def scenario_tests_need_service_tags(physical_line, filename,
 
 
 def factory(register):
-    register(skip_bugs)
     register(import_no_clients_in_api)
     register(scenario_tests_need_service_tags)

tempest/test.py

@@ -103,6 +103,21 @@ def stresstest(*args, **kwargs):
     return decorator
 
 
+def skip_because(*args, **kwargs):
+    """A decorator useful to skip tests hitting known bugs
+
+    @param bug: bug number causing the test to skip
+    @param condition: optional condition to be True for the skip to have place
+    """
+    def decorator(f):
+        if "bug" in kwargs:
+            if "condition" not in kwargs or kwargs["condition"] is True:
+                msg = "Skipped until Bug: %s is resolved." % kwargs["bug"]
+                raise testtools.TestCase.skipException(msg)
+        return f
+    return decorator
+
+
 # there is a mis-match between nose and testtools for older pythons.
 # testtools will set skipException to be either
 # unittest.case.SkipTest, unittest2.case.SkipTest or an internal skip

tempest/thirdparty/boto/test_ec2_instance_run.py

@@ -16,7 +16,6 @@
 #    under the License.
 
 from boto import exception
-import testtools
 
 from tempest import clients
 from tempest.common.utils.data_utils import rand_name
@@ -24,6 +23,7 @@ from tempest.common.utils.linux.remote_client import RemoteClient
 from tempest import exceptions
 from tempest.openstack.common import log as logging
 from tempest.test import attr
+from tempest.test import skip_because
 from tempest.thirdparty.boto.test import BotoTestCase
 from tempest.thirdparty.boto.utils.s3 import s3_upload_dir
 from tempest.thirdparty.boto.utils.wait import re_search_wait
@@ -206,8 +206,8 @@ class InstanceRunTest(BotoTestCase):
             instance.terminate()
         self.cancelResourceCleanUp(rcuk)
 
+    @skip_because(bug="1098891")
     @attr(type='smoke')
-    @testtools.skip("Skipped until the Bug #1098891 is resolved")
     def test_run_terminate_instance(self):
         # EC2 run, terminate immediately
         image_ami = self.ec2_client.get_image(self.images["ami"]
@@ -233,7 +233,7 @@ class InstanceRunTest(BotoTestCase):
 
     # NOTE(afazekas): doctored test case,
     # with normal validation it would fail
-    @testtools.skip("Skipped until the Bug #1182679 is resolved.")
+    @skip_because(bug="1182679")
     @attr(type='smoke')
     def test_integration_1(self):
         # EC2 1. integration test (not strict)

tempest/thirdparty/boto/test_ec2_keys.py

@@ -15,11 +15,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import testtools
-
 from tempest import clients
 from tempest.common.utils.data_utils import rand_name
 from tempest.test import attr
+from tempest.test import skip_because
 from tempest.thirdparty.boto.test import BotoTestCase
 
 
@@ -47,8 +46,8 @@ class EC2KeysTest(BotoTestCase):
         self.assertTrue(compare_key_pairs(keypair,
                         self.client.get_key_pair(key_name)))
 
+    @skip_because(bug="1072318")
     @attr(type='smoke')
-    @testtools.skip("Skipped until the Bug #1072318 is resolved")
     def test_delete_ec2_keypair(self):
         # EC2 delete KeyPair
         key_name = rand_name("keypair-")

tempest/thirdparty/boto/test_ec2_network.py

@@ -15,10 +15,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import testtools
-
 from tempest import clients
 from tempest.test import attr
+from tempest.test import skip_because
 from tempest.thirdparty.boto.test import BotoTestCase
 
 
@@ -31,7 +30,7 @@ class EC2NetworkTest(BotoTestCase):
         cls.client = cls.os.ec2api_client
 
     # Note(afazekas): these tests for things duable without an instance
-    @testtools.skip("Skipped until the Bug #1080406 is resolved")
+    @skip_because(bug="1080406")
     @attr(type='smoke')
     def test_disassociate_not_associated_floating_ip(self):
         # EC2 disassociate not associated floating ip

tempest/thirdparty/boto/test_s3_buckets.py

@@ -15,11 +15,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import testtools
-
 from tempest import clients
 from tempest.common.utils.data_utils import rand_name
 from tempest.test import attr
+from tempest.test import skip_because
 from tempest.thirdparty.boto.test import BotoTestCase
 
 
@@ -31,7 +30,7 @@ class S3BucketsTest(BotoTestCase):
         cls.os = clients.Manager()
         cls.client = cls.os.s3_client
 
-    @testtools.skip("Skipped until the Bug #1076965 is resolved")
+    @skip_because(bug="1076965")
     @attr(type='smoke')
     def test_create_and_get_delete_bucket(self):
         # S3 Create, get and delete bucket

tools/skip_tracker.py

@@ -61,8 +61,8 @@ def find_skips_in_file(path):
     """
     Return the skip tuples in a test file
     """
-    BUG_RE = re.compile(r'.*skip.*bug:*\s*\#*(\d+)', re.IGNORECASE)
+    BUG_RE = re.compile(r'\s*@.*skip_because\(bug=[\'"](\d+)[\'"]')
-    DEF_RE = re.compile(r'.*def (\w+)\(')
+    DEF_RE = re.compile(r'\s*def (\w+)\(')
     bug_found = False
     results = []
     lines = open(path, 'rb').readlines()