openstack/tempest
Code Issues Proposed changes

Add support for negative tests with admin client

Browse Source 
This adds the missing support for admin clients and implements an
example based on flavor creation. Instead of listing all result code
checks it is now possible to define on default.

Change-Id: I9512c1b91eb227e35faf24e3e88ed73a6ed3b734
Partially-implements: bp autogen-negative-tests
This commit is contained in:
Marc Koderer 2014-03-05 15:58:00 +01:00
parent 6ee82dccd1
commit f857fdaf19
5 changed files with 319 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
etc/schemas/compute/admin/flavor_create.json Normal file
View File

@ -0,0 +1,20 @@
{
"name": "flavor-create",
"http-method": "POST",
"admin_client": true,
"url": "flavors",
"default_result_code": 400,
"json-schema": {
"type": "object",
"properties": {
"name": { "type": "string"},
"ram": { "type": "integer", "minimum": 1},
"vcpus": { "type": "integer", "minimum": 1},
"disk": { "type": "integer"},
"id": { "type": "integer"},
"swap": { "type": "integer"},
"rxtx_factor": { "type": "integer"},
"OS-FLV-EXT-DATA:ephemeral": { "type": "integer"}
}
}
}

250
tempest/api/compute/admin/test_flavors_negative.py
View File

@ -13,6 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import testscenarios
import uuid
from tempest.api.compute import base
@ -20,6 +21,8 @@ from tempest.common.utils import data_utils
from tempest import exceptions
from tempest import test
load_tests = testscenarios.load_tests_apply_scenarios
class FlavorsAdminNegativeTestJSON(base.BaseV2ComputeAdminTest):
@ -44,11 +47,6 @@ class FlavorsAdminNegativeTestJSON(base.BaseV2ComputeAdminTest):
cls.swap = 1024
cls.rxtx = 2
def flavor_clean_up(self, flavor_id):
resp, body = self.client.delete_flavor(flavor_id)
self.assertEqual(resp.status, 202)
self.client.wait_for_resource_deletion(flavor_id)
@test.attr(type=['negative', 'gate'])
def test_get_flavor_details_for_deleted_flavor(self):
# Delete a flavor and ensure it is not listed
@ -84,13 +82,6 @@ class FlavorsAdminNegativeTestJSON(base.BaseV2ComputeAdminTest):
flag = False
self.assertTrue(flag)
@test.attr(type=['negative', 'gate'])
def test_invalid_is_public_string(self):
# the 'is_public' parameter can be 'none/true/false' if it exists
self.assertRaises(exceptions.BadRequest,
self.client.list_flavors_with_detail,
{'is_public': 'invalid'})
@test.attr(type=['negative', 'gate'])
def test_create_flavor_as_user(self):
# only admin user can create a flavor
@ -110,231 +101,16 @@ class FlavorsAdminNegativeTestJSON(base.BaseV2ComputeAdminTest):
self.user_client.delete_flavor,
self.flavor_ref_alt)
@test.attr(type=['negative', 'gate'])
def test_create_flavor_using_invalid_ram(self):
# the 'ram' attribute must be positive integer
flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
flavor_name, -1, self.vcpus,
self.disk, new_flavor_id)
class FlavorCreateNegativeTestJSON(base.BaseV2ComputeAdminTest,
test.NegativeAutoTest):
_interface = 'json'
_service = 'compute'
_schema_file = 'compute/admin/flavor_create.json'
scenarios = test.NegativeAutoTest.generate_scenario(_schema_file)
@test.attr(type=['negative', 'gate'])
def test_create_flavor_using_invalid_vcpus(self):
# the 'vcpu' attribute must be positive integer
flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
flavor_name, self.ram, -1,
self.disk, new_flavor_id)
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_name_length_less_than_1(self):
# ensure name length >= 1
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
'',
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_name_length_exceeds_255(self):
# ensure name do not exceed 255 characters
new_flavor_name = 'a' * 256
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_name(self):
# the regex of flavor_name is '^[\w\.\- ]*$'
invalid_flavor_name = data_utils.rand_name('invalid-!@#$%-')
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
invalid_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_flavor_id(self):
# the regex of flavor_id is '^[\w\.\- ]*$', and it cannot contain
# leading and/or trailing whitespace
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
invalid_flavor_id = '!@#$%'
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
invalid_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_id_length_exceeds_255(self):
# the length of flavor_id should not exceed 255 characters
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
invalid_flavor_id = 'a' * 256
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
invalid_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_root_gb(self):
# root_gb attribute should be non-negative ( >= 0) integer
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
-1,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_ephemeral_gb(self):
# ephemeral_gb attribute should be non-negative ( >= 0) integer
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=-1,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_swap(self):
# swap attribute should be non-negative ( >= 0) integer
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=-1,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_rxtx_factor(self):
# rxtx_factor attribute should be a positive float
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=-1.5,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_is_public(self):
# is_public attribute should be boolean
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='Invalid')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_already_exists(self):
flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
resp, flavor = self.client.create_flavor(flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx)
self.assertEqual(200, resp.status)
self.addCleanup(self.flavor_clean_up, flavor['id'])
self.assertRaises(exceptions.Conflict,
self.client.create_flavor,
flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx)
@test.attr(type=['negative', 'gate'])
def test_delete_nonexistent_flavor(self):
nonexistent_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.NotFound,
self.client.delete_flavor,
nonexistent_flavor_id)
class FlavorsAdminNegativeTestXML(FlavorsAdminNegativeTestJSON):
_interface = 'xml'
def test_create_flavor(self):
# flavor details are not returned for non-existent flavors
self.execute(self._schema_file)

268
tempest/api/compute/admin/test_flavors_negative_xml.py Normal file
View File

@ -0,0 +1,268 @@
# Copyright 2012 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import uuid
from tempest.api.compute.admin import test_flavors_negative
from tempest.common.utils import data_utils
from tempest import exceptions
from tempest import test
class FlavorsAdminNegativeTestXML(test_flavors_negative.
FlavorsAdminNegativeTestJSON):
"""
Tests Flavors API Create and Delete that require admin privileges
"""
_interface = 'xml'
def flavor_clean_up(self, flavor_id):
resp, body = self.client.delete_flavor(flavor_id)
self.assertEqual(resp.status, 202)
self.client.wait_for_resource_deletion(flavor_id)
@test.attr(type=['negative', 'gate'])
def test_invalid_is_public_string(self):
# the 'is_public' parameter can be 'none/true/false' if it exists
self.assertRaises(exceptions.BadRequest,
self.client.list_flavors_with_detail,
{'is_public': 'invalid'})
@test.attr(type=['negative', 'gate'])
def test_create_flavor_using_invalid_ram(self):
# the 'ram' attribute must be positive integer
flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
flavor_name, -1, self.vcpus,
self.disk, new_flavor_id)
@test.attr(type=['negative', 'gate'])
def test_create_flavor_using_invalid_vcpus(self):
# the 'vcpu' attribute must be positive integer
flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
flavor_name, self.ram, -1,
self.disk, new_flavor_id)
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_name_length_less_than_1(self):
# ensure name length >= 1
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
'',
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_name_length_exceeds_255(self):
# ensure name do not exceed 255 characters
new_flavor_name = 'a' * 256
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_name(self):
# the regex of flavor_name is '^[\w\.\- ]*$'
invalid_flavor_name = data_utils.rand_name('invalid-!@#$%-')
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
invalid_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_flavor_id(self):
# the regex of flavor_id is '^[\w\.\- ]*$', and it cannot contain
# leading and/or trailing whitespace
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
invalid_flavor_id = '!@#$%'
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
invalid_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_id_length_exceeds_255(self):
# the length of flavor_id should not exceed 255 characters
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
invalid_flavor_id = 'a' * 256
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
invalid_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_root_gb(self):
# root_gb attribute should be non-negative ( >= 0) integer
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
-1,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_ephemeral_gb(self):
# ephemeral_gb attribute should be non-negative ( >= 0) integer
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=-1,
swap=self.swap,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_swap(self):
# swap attribute should be non-negative ( >= 0) integer
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=-1,
rxtx=self.rxtx,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_rxtx_factor(self):
# rxtx_factor attribute should be a positive float
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=-1.5,
is_public='False')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_with_invalid_is_public(self):
# is_public attribute should be boolean
new_flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.BadRequest,
self.client.create_flavor,
new_flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx,
is_public='Invalid')
@test.attr(type=['negative', 'gate'])
def test_create_flavor_already_exists(self):
flavor_name = data_utils.rand_name(self.flavor_name_prefix)
new_flavor_id = str(uuid.uuid4())
resp, flavor = self.client.create_flavor(flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx)
self.assertEqual(200, resp.status)
self.addCleanup(self.flavor_clean_up, flavor['id'])
self.assertRaises(exceptions.Conflict,
self.client.create_flavor,
flavor_name,
self.ram, self.vcpus,
self.disk,
new_flavor_id,
ephemeral=self.ephemeral,
swap=self.swap,
rxtx=self.rxtx)
@test.attr(type=['negative', 'gate'])
def test_delete_nonexistent_flavor(self):
nonexistent_flavor_id = str(uuid.uuid4())
self.assertRaises(exceptions.NotFound,
self.client.delete_flavor,
nonexistent_flavor_id)

2
tempest/common/generator/base_generator.py
View File

@ -59,7 +59,9 @@ class BasicGeneratorSet(object):
"enum": ["GET", "PUT", "HEAD",
"POST", "PATCH", "DELETE", 'COPY']
},
"admin_client": {"type": "boolean"},
"url": {"type": "string"},
"default_result_code": {"type": "integer"},
"json-schema": jsonschema._utils.load_schema("draft4"),
"resources": {
"type": "array",

20
tempest/test.py
View File

@ -363,6 +363,9 @@ class NegativeAutoTest(BaseTestCase):
super(NegativeAutoTest, cls).setUpClass()
os = cls.get_client_manager()
cls.client = os.negative_client
os_admin = clients.AdminManager(interface=cls._interface,
service=cls._service)
cls.admin_client = os_admin.negative_client
@staticmethod
def load_schema(file):
@ -418,10 +421,13 @@ class NegativeAutoTest(BaseTestCase):
"expected_result": expected_result
}))
if schema is not None:
for invalid in generator.generate(schema):
scenario_list.append((invalid[0],
{"schema": invalid[1],
"expected_result": invalid[2]}))
for name, schema, expected_result in generator.generate(schema):
if (expected_result is None and
"default_result_code" in description):
expected_result = description["default_result_code"]
scenario_list.append((name,
{"schema": schema,
"expected_result": expected_result}))
LOG.debug(scenario_list)
return scenario_list
@ -470,7 +476,11 @@ class NegativeAutoTest(BaseTestCase):
elif hasattr(self, "schema"):
new_url, body = self._http_arguments(self.schema, url, method)
resp, resp_body = self.client.send_request(method, new_url,
if "admin_client" in description and description["admin_client"]:
client = self.admin_client
else:
client = self.client
resp, resp_body = client.send_request(method, new_url,
resources, body=body)
self._check_negative_response(resp.status, resp_body)