openstack
/
tempest
Code Issues Proposed changes

introduces skip_because decorator 

this change introduces a skip_because decorator which accepts two args
a bug and a condition; also updates the skip tracker accordingly

Change-Id: If53f2ef81d6bddbce284267216254b467046855f
This commit is contained in:
Giulio Fidente 2013-10-01 06:02:24 +02:00
parent 385f0b116e
commit 83181a9703
17 changed files with 72 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
HACKING.rst
View File

@ -8,16 +8,6 @@ Tempest Coding Guide
Tempest Specific Commandments
------------------------------
[T101] If a test is broken because of a bug it is appropriate to skip the test until
bug has been fixed. However, the skip message should be formatted so that
Tempest's skip tracking tool can watch the bug status. The skip message should
contain the string 'Bug' immediately followed by a space. Then the bug number
should be included in the message '#' in front of the number.
Example::
@testtools.skip("Skipped until the Bug #980688 is resolved")
- [T102] Cannot import OpenStack python clients in tempest/api tests
- [T103] tempest/tests is deprecated
- [T104] Scenario tests require a services decorator
@ -115,6 +105,19 @@ name. For example, any test that make an api call to a service other than nova
in tempest.api.compute would require a service tag for those services, however
they do not need to be tagged as compute.
Test skips because of Known Bugs
--------------------------------
If a test is broken because of a bug it is appropriate to skip the test until
bug has been fixed. You should use the skip_because decorator so that
Tempest's skip tracking tool can watch the bug status.
Example::
@skip_because(bug="980688")
def test_this_and_that(self):
...
Guidelines
----------
- Do not submit changesets with only testcases which are skipped as

5
tempest/api/compute/admin/test_flavors.py
View File

@ -15,14 +15,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import testtools
from tempest.api import compute
from tempest.api.compute import base
from tempest.common.utils.data_utils import rand_int_id
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.test import attr
from tempest.test import skip_because
class FlavorsAdminTestJSON(base.BaseComputeAdminTest):
@ -195,7 +194,7 @@ class FlavorsAdminTestJSON(base.BaseComputeAdminTest):
flag = True
self.assertTrue(flag)
@testtools.skip("Skipped until the Bug #1209101 is resolved")
@skip_because(bug="1209101")
@attr(type='gate')
def test_list_non_public_flavor(self):
# Create a flavor with os-flavor-access:is_public false should

3
tempest/api/compute/images/test_images_oneserver.py
View File

@ -24,6 +24,7 @@ from tempest.common.utils.data_utils import parse_image_id
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.test import attr
from tempest.test import skip_because
class ImagesOneServerTestJSON(base.BaseComputeTest):
@ -64,7 +65,7 @@ class ImagesOneServerTestJSON(base.BaseComputeTest):
cls.alt_manager = clients.AltManager()
cls.alt_client = cls.alt_manager.images_client
@testtools.skip("Skipped until the Bug #1006725 is resolved.")
@skip_because(bug="1006725")
@attr(type=['negative', 'gate'])
def test_create_image_specify_multibyte_character_image_name(self):
# Return an error if the image name has multi-byte characters

11
tempest/api/compute/security_groups/test_security_group_rules.py
View File

@ -15,13 +15,12 @@
# License for the specific language governing permissions and limitations
# under the License.
import testtools
from tempest.api.compute import base
from tempest.common.utils.data_utils import rand_name
from tempest import config
from tempest import exceptions
from tempest.test import attr
from tempest.test import skip_because
class SecurityGroupRulesTestJSON(base.BaseComputeTest):
@ -94,8 +93,8 @@ class SecurityGroupRulesTestJSON(base.BaseComputeTest):
self.addCleanup(self.client.delete_security_group_rule, rule['id'])
self.assertEqual(200, resp.status)
@testtools.skipIf(config.TempestConfig().service_available.neutron,
"Skipped until the Bug #1182384 is resolved")
@skip_because(bug="1182384",
condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_security_group_rules_create_with_invalid_id(self):
# Negative test: Creation of Security Group rule should FAIL
@ -186,8 +185,8 @@ class SecurityGroupRulesTestJSON(base.BaseComputeTest):
self.client.create_security_group_rule,
secgroup_id, ip_protocol, from_port, to_port)
@testtools.skipIf(config.TempestConfig().service_available.neutron,
"Skipped until the Bug #1182384 is resolved")
@skip_because(bug="1182384",
condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_security_group_rules_delete_with_invalid_id(self):
# Negative test: Deletion of Security Group rule should be FAIL

17
tempest/api/compute/security_groups/test_security_groups.py
View File

@ -22,6 +22,7 @@ from tempest.common.utils.data_utils import rand_name
from tempest import config
from tempest import exceptions
from tempest.test import attr
from tempest.test import skip_because
class SecurityGroupsTestJSON(base.BaseComputeTest):
@ -107,8 +108,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest):
"The fetched Security Group is different "
"from the created Group")
@testtools.skipIf(config.TempestConfig().service_available.neutron,
"Skipped until the Bug #1182384 is resolved")
@skip_because(bug="1182384",
condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_security_group_get_nonexistant_group(self):
# Negative test:Should not be able to GET the details
@ -125,8 +126,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest):
self.assertRaises(exceptions.NotFound, self.client.get_security_group,
non_exist_id)
@testtools.skipIf(config.TempestConfig().service_available.neutron,
"Skipped until the Bug #1161411 is resolved")
@skip_because(bug="1161411",
condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_security_group_create_with_invalid_group_name(self):
# Negative test: Security Group should not be created with group name
@ -145,8 +146,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest):
self.client.create_security_group, s_name,
s_description)
@testtools.skipIf(config.TempestConfig().service_available.neutron,
"Skipped until the Bug #1161411 is resolved")
@skip_because(bug="1161411",
condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_security_group_create_with_invalid_group_description(self):
# Negative test:Security Group should not be created with description
@ -197,8 +198,8 @@ class SecurityGroupsTestJSON(base.BaseComputeTest):
self.client.delete_security_group,
default_security_group_id)
@testtools.skipIf(config.TempestConfig().service_available.neutron,
"Skipped until the Bug #1182384 is resolved")
@skip_because(bug="1182384",
condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_delete_nonexistant_security_group(self):
# Negative test:Deletion of a non-existent Security Group should Fail

9
tempest/api/compute/servers/test_list_server_filters.py
View File

@ -15,14 +15,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import testtools
from tempest.api.compute import base
from tempest.api import utils
from tempest.common.utils.data_utils import rand_name
from tempest import config
from tempest import exceptions
from tempest.test import attr
from tempest.test import skip_because
class ListServerFiltersTestJSON(base.BaseComputeTest):
@ -205,7 +204,7 @@ class ListServerFiltersTestJSON(base.BaseComputeTest):
self.assertNotIn(self.s2_name, map(lambda x: x['name'], servers))
self.assertNotIn(self.s3_name, map(lambda x: x['name'], servers))
@testtools.skip('Skipped until the Bug #1170718 is resolved.')
@skip_because(bug="1170718")
@attr(type='gate')
def test_list_servers_filtered_by_ip(self):
# Filter servers by ip
@ -219,8 +218,8 @@ class ListServerFiltersTestJSON(base.BaseComputeTest):
self.assertNotIn(self.s2_name, map(lambda x: x['name'], servers))
self.assertNotIn(self.s3_name, map(lambda x: x['name'], servers))
@testtools.skipIf(config.TempestConfig().service_available.neutron,
"Skipped until the Bug #1182883 is resolved")
@skip_because(bug="1182883",
condition=config.TempestConfig().service_available.neutron)
@attr(type='gate')
def test_list_servers_filtered_by_ip_regex(self):
# Filter servers by regex ip

5
tempest/api/compute/servers/test_server_actions.py
View File

@ -27,6 +27,7 @@ from tempest.common.utils.linux.remote_client import RemoteClient
import tempest.config
from tempest import exceptions
from tempest.test import attr
from tempest.test import skip_because
class ServerActionsTestJSON(base.BaseComputeTest):
@ -86,7 +87,7 @@ class ServerActionsTestJSON(base.BaseComputeTest):
new_boot_time = linux_client.get_boot_time()
self.assertGreater(new_boot_time, boot_time)
@testtools.skip('Skipped until the Bug #1014647 is resolved.')
@skip_because(bug="1014647")
@attr(type='smoke')
def test_reboot_server_soft(self):
# The server should be signaled to reboot gracefully
@ -250,7 +251,7 @@ class ServerActionsTestJSON(base.BaseComputeTest):
self.servers_client.get_console_output,
'!@#$%^&*()', 10)
@testtools.skip('Skipped until the Bug #1014683 is resolved.')
@skip_because(bug="1014683")
@attr(type='gate')
def test_get_console_output_server_id_in_reboot_status(self):
# Positive test:Should be able to GET the console output

6
tempest/api/compute/servers/test_virtual_interfaces.py
View File

@ -16,13 +16,13 @@
# under the License.
import netaddr
import testtools
from tempest.api.compute import base
from tempest.common.utils.data_utils import rand_name
from tempest import config
from tempest import exceptions
from tempest.test import attr
from tempest.test import skip_because
class VirtualInterfacesTestJSON(base.BaseComputeTest):
@ -37,8 +37,8 @@ class VirtualInterfacesTestJSON(base.BaseComputeTest):
resp, server = cls.create_server(wait_until='ACTIVE')
cls.server_id = server['id']
@testtools.skipIf(CONF.service_available.neutron, "Not implemented by " +
"Neutron. Skipped until the Bug #1183436 is resolved.")
@skip_because(bug="1183436",
condition=CONF.service_available.neutron)
@attr(type='gate')
def test_list_virtual_interfaces(self):
# Positive test:Should be able to GET the virtual interfaces list

5
tempest/api/object_storage/test_container_sync.py
View File

@ -17,11 +17,10 @@
import time
import testtools
from tempest.api.object_storage import base
from tempest.common.utils.data_utils import rand_name
from tempest.test import attr
from tempest.test import skip_because
class ContainerSyncTest(base.BaseObjectTest):
@ -52,7 +51,7 @@ class ContainerSyncTest(base.BaseObjectTest):
cls.delete_containers(cls.containers, client[0], client[1])
super(ContainerSyncTest, cls).tearDownClass()
@testtools.skip('Skipped until the Bug #1093743 is resolved.')
@skip_because(bug="1093743")
@attr(type='gate')
def test_container_synchronization(self):
# container to container synchronization

5
tempest/api/object_storage/test_object_expiry.py
View File

@ -17,13 +17,12 @@
import time
import testtools
from tempest.api.object_storage import base
from tempest.common.utils.data_utils import arbitrary_string
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.test import attr
from tempest.test import skip_because
class ObjectExpiryTest(base.BaseObjectTest):
@ -43,7 +42,7 @@ class ObjectExpiryTest(base.BaseObjectTest):
cls.delete_containers([cls.container_name])
super(ObjectExpiryTest, cls).tearDownClass()
@testtools.skip('Skipped until the Bug #1069849 is resolved.')
@skip_because(bug="1069849")
@attr(type='gate')
def test_get_object_after_expiry_time(self):
# TODO(harika-vakadi): similar test case has to be created for

18
tempest/hacking/checks.py
View File

@ -19,28 +19,11 @@ import re
PYTHON_CLIENTS = ['cinder', 'glance', 'keystone', 'nova', 'swift', 'neutron']
SKIP_DECORATOR_RE = re.compile(r'\s*@testtools.skip\((.*)\)')
SKIP_STR_RE = re.compile(r'.*Bug #\d+.*')
PYTHON_CLIENT_RE = re.compile('import (%s)client' % '|'.join(PYTHON_CLIENTS))
TEST_DEFINITION = re.compile(r'^\s*def test.*')
SCENARIO_DECORATOR = re.compile(r'\s*@.*services\(')
def skip_bugs(physical_line):
"""Check skip lines for proper bug entries
T101: skips must contain "Bug #<bug_number>"
"""
res = SKIP_DECORATOR_RE.match(physical_line)
if res:
content = res.group(1)
res = SKIP_STR_RE.match(content)
if not res:
return (physical_line.find(content),
'T101: skips must contain "Bug #<bug_number>"')
def import_no_clients_in_api(physical_line, filename):
"""Check for client imports from tempest/api tests
@ -70,6 +53,5 @@ def scenario_tests_need_service_tags(physical_line, filename,
def factory(register):
register(skip_bugs)
register(import_no_clients_in_api)
register(scenario_tests_need_service_tags)

15
tempest/test.py
View File

@ -103,6 +103,21 @@ def stresstest(*args, **kwargs):
return decorator
def skip_because(*args, **kwargs):
"""A decorator useful to skip tests hitting known bugs
@param bug: bug number causing the test to skip
@param condition: optional condition to be True for the skip to have place
"""
def decorator(f):
if "bug" in kwargs:
if "condition" not in kwargs or kwargs["condition"] is True:
msg = "Skipped until Bug: %s is resolved." % kwargs["bug"]
raise testtools.TestCase.skipException(msg)
return f
return decorator
# there is a mis-match between nose and testtools for older pythons.
# testtools will set skipException to be either
# unittest.case.SkipTest, unittest2.case.SkipTest or an internal skip

6
tempest/thirdparty/boto/test_ec2_instance_run.py vendored
View File

@ -16,7 +16,6 @@
# under the License.
from boto import exception
import testtools
from tempest import clients
from tempest.common.utils.data_utils import rand_name
@ -24,6 +23,7 @@ from tempest.common.utils.linux.remote_client import RemoteClient
from tempest import exceptions
from tempest.openstack.common import log as logging
from tempest.test import attr
from tempest.test import skip_because
from tempest.thirdparty.boto.test import BotoTestCase
from tempest.thirdparty.boto.utils.s3 import s3_upload_dir
from tempest.thirdparty.boto.utils.wait import re_search_wait
@ -206,8 +206,8 @@ class InstanceRunTest(BotoTestCase):
instance.terminate()
self.cancelResourceCleanUp(rcuk)
@skip_because(bug="1098891")
@attr(type='smoke')
@testtools.skip("Skipped until the Bug #1098891 is resolved")
def test_run_terminate_instance(self):
# EC2 run, terminate immediately
image_ami = self.ec2_client.get_image(self.images["ami"]
@ -233,7 +233,7 @@ class InstanceRunTest(BotoTestCase):
# NOTE(afazekas): doctored test case,
# with normal validation it would fail
@testtools.skip("Skipped until the Bug #1182679 is resolved.")
@skip_because(bug="1182679")
@attr(type='smoke')
def test_integration_1(self):
# EC2 1. integration test (not strict)

5
tempest/thirdparty/boto/test_ec2_keys.py vendored
View File

@ -15,11 +15,10 @@
# License for the specific language governing permissions and limitations
# under the License.
import testtools
from tempest import clients
from tempest.common.utils.data_utils import rand_name
from tempest.test import attr
from tempest.test import skip_because
from tempest.thirdparty.boto.test import BotoTestCase
@ -47,8 +46,8 @@ class EC2KeysTest(BotoTestCase):
self.assertTrue(compare_key_pairs(keypair,
self.client.get_key_pair(key_name)))
@skip_because(bug="1072318")
@attr(type='smoke')
@testtools.skip("Skipped until the Bug #1072318 is resolved")
def test_delete_ec2_keypair(self):
# EC2 delete KeyPair
key_name = rand_name("keypair-")

7
tempest/thirdparty/boto/test_ec2_network.py vendored
View File

@ -15,10 +15,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import testtools
from tempest import clients
from tempest.test import attr
from tempest.test import skip_because
from tempest.thirdparty.boto.test import BotoTestCase
@ -30,8 +29,8 @@ class EC2NetworkTest(BotoTestCase):
cls.os = clients.Manager()
cls.client = cls.os.ec2api_client
# Note(afazekas): these tests for things duable without an instance
@testtools.skip("Skipped until the Bug #1080406 is resolved")
# Note(afazekas): these tests for things duable without an instance
@skip_because(bug="1080406")
@attr(type='smoke')
def test_disassociate_not_associated_floating_ip(self):
# EC2 disassociate not associated floating ip

5
tempest/thirdparty/boto/test_s3_buckets.py vendored
View File

@ -15,11 +15,10 @@
# License for the specific language governing permissions and limitations
# under the License.
import testtools
from tempest import clients
from tempest.common.utils.data_utils import rand_name
from tempest.test import attr
from tempest.test import skip_because
from tempest.thirdparty.boto.test import BotoTestCase
@ -31,7 +30,7 @@ class S3BucketsTest(BotoTestCase):
cls.os = clients.Manager()
cls.client = cls.os.s3_client
@testtools.skip("Skipped until the Bug #1076965 is resolved")
@skip_because(bug="1076965")
@attr(type='smoke')
def test_create_and_get_delete_bucket(self):
# S3 Create, get and delete bucket

4
tools/skip_tracker.py
View File

@ -61,8 +61,8 @@ def find_skips_in_file(path):
"""
Return the skip tuples in a test file
"""
BUG_RE = re.compile(r'.*skip.*bug:*\s*\#*(\d+)', re.IGNORECASE)
DEF_RE = re.compile(r'.*def (\w+)\(')
BUG_RE = re.compile(r'\s*@.*skip_because\(bug=[\'"](\d+)[\'"]')
DEF_RE = re.compile(r'\s*def (\w+)\(')
bug_found = False
results = []
lines = open(path, 'rb').readlines()