From f857fdaf193d6ab8871728e0720d18a06ed8db9d Mon Sep 17 00:00:00 2001 From: Marc Koderer Date: Wed, 5 Mar 2014 15:58:00 +0100 Subject: [PATCH] Add support for negative tests with admin client This adds the missing support for admin clients and implements an example based on flavor creation. Instead of listing all result code checks it is now possible to define on default. Change-Id: I9512c1b91eb227e35faf24e3e88ed73a6ed3b734 Partially-implements: bp autogen-negative-tests --- etc/schemas/compute/admin/flavor_create.json | 20 ++ .../compute/admin/test_flavors_negative.py | 250 +--------------- .../admin/test_flavors_negative_xml.py | 268 ++++++++++++++++++ tempest/common/generator/base_generator.py | 2 + tempest/test.py | 22 +- 5 files changed, 319 insertions(+), 243 deletions(-) create mode 100644 etc/schemas/compute/admin/flavor_create.json create mode 100644 tempest/api/compute/admin/test_flavors_negative_xml.py diff --git a/etc/schemas/compute/admin/flavor_create.json b/etc/schemas/compute/admin/flavor_create.json new file mode 100644 index 0000000000..0a3e7b3217 --- /dev/null +++ b/etc/schemas/compute/admin/flavor_create.json @@ -0,0 +1,20 @@ +{ + "name": "flavor-create", + "http-method": "POST", + "admin_client": true, + "url": "flavors", + "default_result_code": 400, + "json-schema": { + "type": "object", + "properties": { + "name": { "type": "string"}, + "ram": { "type": "integer", "minimum": 1}, + "vcpus": { "type": "integer", "minimum": 1}, + "disk": { "type": "integer"}, + "id": { "type": "integer"}, + "swap": { "type": "integer"}, + "rxtx_factor": { "type": "integer"}, + "OS-FLV-EXT-DATA:ephemeral": { "type": "integer"} + } + } +} diff --git a/tempest/api/compute/admin/test_flavors_negative.py b/tempest/api/compute/admin/test_flavors_negative.py index 49d49ef18b..b882ff40db 100644 --- a/tempest/api/compute/admin/test_flavors_negative.py +++ b/tempest/api/compute/admin/test_flavors_negative.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import testscenarios import uuid from tempest.api.compute import base @@ -20,6 +21,8 @@ from tempest.common.utils import data_utils from tempest import exceptions from tempest import test +load_tests = testscenarios.load_tests_apply_scenarios + class FlavorsAdminNegativeTestJSON(base.BaseV2ComputeAdminTest): @@ -44,11 +47,6 @@ class FlavorsAdminNegativeTestJSON(base.BaseV2ComputeAdminTest): cls.swap = 1024 cls.rxtx = 2 - def flavor_clean_up(self, flavor_id): - resp, body = self.client.delete_flavor(flavor_id) - self.assertEqual(resp.status, 202) - self.client.wait_for_resource_deletion(flavor_id) - @test.attr(type=['negative', 'gate']) def test_get_flavor_details_for_deleted_flavor(self): # Delete a flavor and ensure it is not listed @@ -84,13 +82,6 @@ class FlavorsAdminNegativeTestJSON(base.BaseV2ComputeAdminTest): flag = False self.assertTrue(flag) - @test.attr(type=['negative', 'gate']) - def test_invalid_is_public_string(self): - # the 'is_public' parameter can be 'none/true/false' if it exists - self.assertRaises(exceptions.BadRequest, - self.client.list_flavors_with_detail, - {'is_public': 'invalid'}) - @test.attr(type=['negative', 'gate']) def test_create_flavor_as_user(self): # only admin user can create a flavor @@ -110,231 +101,16 @@ class FlavorsAdminNegativeTestJSON(base.BaseV2ComputeAdminTest): self.user_client.delete_flavor, self.flavor_ref_alt) - @test.attr(type=['negative', 'gate']) - def test_create_flavor_using_invalid_ram(self): - # the 'ram' attribute must be positive integer - flavor_name = data_utils.rand_name(self.flavor_name_prefix) - new_flavor_id = str(uuid.uuid4()) - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - flavor_name, -1, self.vcpus, - self.disk, new_flavor_id) +class FlavorCreateNegativeTestJSON(base.BaseV2ComputeAdminTest, + test.NegativeAutoTest): + _interface = 'json' + _service = 'compute' + _schema_file = 'compute/admin/flavor_create.json' + + scenarios = test.NegativeAutoTest.generate_scenario(_schema_file) @test.attr(type=['negative', 'gate']) - def test_create_flavor_using_invalid_vcpus(self): - # the 'vcpu' attribute must be positive integer - flavor_name = data_utils.rand_name(self.flavor_name_prefix) - new_flavor_id = str(uuid.uuid4()) - - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - flavor_name, self.ram, -1, - self.disk, new_flavor_id) - - @test.attr(type=['negative', 'gate']) - def test_create_flavor_with_name_length_less_than_1(self): - # ensure name length >= 1 - new_flavor_id = str(uuid.uuid4()) - - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - '', - self.ram, self.vcpus, - self.disk, - new_flavor_id, - ephemeral=self.ephemeral, - swap=self.swap, - rxtx=self.rxtx, - is_public='False') - - @test.attr(type=['negative', 'gate']) - def test_create_flavor_with_name_length_exceeds_255(self): - # ensure name do not exceed 255 characters - new_flavor_name = 'a' * 256 - new_flavor_id = str(uuid.uuid4()) - - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - new_flavor_name, - self.ram, self.vcpus, - self.disk, - new_flavor_id, - ephemeral=self.ephemeral, - swap=self.swap, - rxtx=self.rxtx, - is_public='False') - - @test.attr(type=['negative', 'gate']) - def test_create_flavor_with_invalid_name(self): - # the regex of flavor_name is '^[\w\.\- ]*$' - invalid_flavor_name = data_utils.rand_name('invalid-!@#$%-') - new_flavor_id = str(uuid.uuid4()) - - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - invalid_flavor_name, - self.ram, self.vcpus, - self.disk, - new_flavor_id, - ephemeral=self.ephemeral, - swap=self.swap, - rxtx=self.rxtx, - is_public='False') - - @test.attr(type=['negative', 'gate']) - def test_create_flavor_with_invalid_flavor_id(self): - # the regex of flavor_id is '^[\w\.\- ]*$', and it cannot contain - # leading and/or trailing whitespace - new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) - invalid_flavor_id = '!@#$%' - - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - new_flavor_name, - self.ram, self.vcpus, - self.disk, - invalid_flavor_id, - ephemeral=self.ephemeral, - swap=self.swap, - rxtx=self.rxtx, - is_public='False') - - @test.attr(type=['negative', 'gate']) - def test_create_flavor_with_id_length_exceeds_255(self): - # the length of flavor_id should not exceed 255 characters - new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) - invalid_flavor_id = 'a' * 256 - - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - new_flavor_name, - self.ram, self.vcpus, - self.disk, - invalid_flavor_id, - ephemeral=self.ephemeral, - swap=self.swap, - rxtx=self.rxtx, - is_public='False') - - @test.attr(type=['negative', 'gate']) - def test_create_flavor_with_invalid_root_gb(self): - # root_gb attribute should be non-negative ( >= 0) integer - new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) - new_flavor_id = str(uuid.uuid4()) - - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - new_flavor_name, - self.ram, self.vcpus, - -1, - new_flavor_id, - ephemeral=self.ephemeral, - swap=self.swap, - rxtx=self.rxtx, - is_public='False') - - @test.attr(type=['negative', 'gate']) - def test_create_flavor_with_invalid_ephemeral_gb(self): - # ephemeral_gb attribute should be non-negative ( >= 0) integer - new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) - new_flavor_id = str(uuid.uuid4()) - - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - new_flavor_name, - self.ram, self.vcpus, - self.disk, - new_flavor_id, - ephemeral=-1, - swap=self.swap, - rxtx=self.rxtx, - is_public='False') - - @test.attr(type=['negative', 'gate']) - def test_create_flavor_with_invalid_swap(self): - # swap attribute should be non-negative ( >= 0) integer - new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) - new_flavor_id = str(uuid.uuid4()) - - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - new_flavor_name, - self.ram, self.vcpus, - self.disk, - new_flavor_id, - ephemeral=self.ephemeral, - swap=-1, - rxtx=self.rxtx, - is_public='False') - - @test.attr(type=['negative', 'gate']) - def test_create_flavor_with_invalid_rxtx_factor(self): - # rxtx_factor attribute should be a positive float - new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) - new_flavor_id = str(uuid.uuid4()) - - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - new_flavor_name, - self.ram, self.vcpus, - self.disk, - new_flavor_id, - ephemeral=self.ephemeral, - swap=self.swap, - rxtx=-1.5, - is_public='False') - - @test.attr(type=['negative', 'gate']) - def test_create_flavor_with_invalid_is_public(self): - # is_public attribute should be boolean - new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) - new_flavor_id = str(uuid.uuid4()) - - self.assertRaises(exceptions.BadRequest, - self.client.create_flavor, - new_flavor_name, - self.ram, self.vcpus, - self.disk, - new_flavor_id, - ephemeral=self.ephemeral, - swap=self.swap, - rxtx=self.rxtx, - is_public='Invalid') - - @test.attr(type=['negative', 'gate']) - def test_create_flavor_already_exists(self): - flavor_name = data_utils.rand_name(self.flavor_name_prefix) - new_flavor_id = str(uuid.uuid4()) - - resp, flavor = self.client.create_flavor(flavor_name, - self.ram, self.vcpus, - self.disk, - new_flavor_id, - ephemeral=self.ephemeral, - swap=self.swap, - rxtx=self.rxtx) - self.assertEqual(200, resp.status) - self.addCleanup(self.flavor_clean_up, flavor['id']) - - self.assertRaises(exceptions.Conflict, - self.client.create_flavor, - flavor_name, - self.ram, self.vcpus, - self.disk, - new_flavor_id, - ephemeral=self.ephemeral, - swap=self.swap, - rxtx=self.rxtx) - - @test.attr(type=['negative', 'gate']) - def test_delete_nonexistent_flavor(self): - nonexistent_flavor_id = str(uuid.uuid4()) - - self.assertRaises(exceptions.NotFound, - self.client.delete_flavor, - nonexistent_flavor_id) - - -class FlavorsAdminNegativeTestXML(FlavorsAdminNegativeTestJSON): - _interface = 'xml' + def test_create_flavor(self): + # flavor details are not returned for non-existent flavors + self.execute(self._schema_file) diff --git a/tempest/api/compute/admin/test_flavors_negative_xml.py b/tempest/api/compute/admin/test_flavors_negative_xml.py new file mode 100644 index 0000000000..a06b0e6754 --- /dev/null +++ b/tempest/api/compute/admin/test_flavors_negative_xml.py @@ -0,0 +1,268 @@ +# Copyright 2012 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid + +from tempest.api.compute.admin import test_flavors_negative +from tempest.common.utils import data_utils +from tempest import exceptions +from tempest import test + + +class FlavorsAdminNegativeTestXML(test_flavors_negative. + FlavorsAdminNegativeTestJSON): + + """ + Tests Flavors API Create and Delete that require admin privileges + """ + + _interface = 'xml' + + def flavor_clean_up(self, flavor_id): + resp, body = self.client.delete_flavor(flavor_id) + self.assertEqual(resp.status, 202) + self.client.wait_for_resource_deletion(flavor_id) + + @test.attr(type=['negative', 'gate']) + def test_invalid_is_public_string(self): + # the 'is_public' parameter can be 'none/true/false' if it exists + self.assertRaises(exceptions.BadRequest, + self.client.list_flavors_with_detail, + {'is_public': 'invalid'}) + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_using_invalid_ram(self): + # the 'ram' attribute must be positive integer + flavor_name = data_utils.rand_name(self.flavor_name_prefix) + new_flavor_id = str(uuid.uuid4()) + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + flavor_name, -1, self.vcpus, + self.disk, new_flavor_id) + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_using_invalid_vcpus(self): + # the 'vcpu' attribute must be positive integer + flavor_name = data_utils.rand_name(self.flavor_name_prefix) + new_flavor_id = str(uuid.uuid4()) + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + flavor_name, self.ram, -1, + self.disk, new_flavor_id) + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_with_name_length_less_than_1(self): + # ensure name length >= 1 + new_flavor_id = str(uuid.uuid4()) + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + '', + self.ram, self.vcpus, + self.disk, + new_flavor_id, + ephemeral=self.ephemeral, + swap=self.swap, + rxtx=self.rxtx, + is_public='False') + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_with_name_length_exceeds_255(self): + # ensure name do not exceed 255 characters + new_flavor_name = 'a' * 256 + new_flavor_id = str(uuid.uuid4()) + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + new_flavor_name, + self.ram, self.vcpus, + self.disk, + new_flavor_id, + ephemeral=self.ephemeral, + swap=self.swap, + rxtx=self.rxtx, + is_public='False') + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_with_invalid_name(self): + # the regex of flavor_name is '^[\w\.\- ]*$' + invalid_flavor_name = data_utils.rand_name('invalid-!@#$%-') + new_flavor_id = str(uuid.uuid4()) + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + invalid_flavor_name, + self.ram, self.vcpus, + self.disk, + new_flavor_id, + ephemeral=self.ephemeral, + swap=self.swap, + rxtx=self.rxtx, + is_public='False') + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_with_invalid_flavor_id(self): + # the regex of flavor_id is '^[\w\.\- ]*$', and it cannot contain + # leading and/or trailing whitespace + new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) + invalid_flavor_id = '!@#$%' + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + new_flavor_name, + self.ram, self.vcpus, + self.disk, + invalid_flavor_id, + ephemeral=self.ephemeral, + swap=self.swap, + rxtx=self.rxtx, + is_public='False') + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_with_id_length_exceeds_255(self): + # the length of flavor_id should not exceed 255 characters + new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) + invalid_flavor_id = 'a' * 256 + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + new_flavor_name, + self.ram, self.vcpus, + self.disk, + invalid_flavor_id, + ephemeral=self.ephemeral, + swap=self.swap, + rxtx=self.rxtx, + is_public='False') + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_with_invalid_root_gb(self): + # root_gb attribute should be non-negative ( >= 0) integer + new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) + new_flavor_id = str(uuid.uuid4()) + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + new_flavor_name, + self.ram, self.vcpus, + -1, + new_flavor_id, + ephemeral=self.ephemeral, + swap=self.swap, + rxtx=self.rxtx, + is_public='False') + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_with_invalid_ephemeral_gb(self): + # ephemeral_gb attribute should be non-negative ( >= 0) integer + new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) + new_flavor_id = str(uuid.uuid4()) + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + new_flavor_name, + self.ram, self.vcpus, + self.disk, + new_flavor_id, + ephemeral=-1, + swap=self.swap, + rxtx=self.rxtx, + is_public='False') + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_with_invalid_swap(self): + # swap attribute should be non-negative ( >= 0) integer + new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) + new_flavor_id = str(uuid.uuid4()) + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + new_flavor_name, + self.ram, self.vcpus, + self.disk, + new_flavor_id, + ephemeral=self.ephemeral, + swap=-1, + rxtx=self.rxtx, + is_public='False') + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_with_invalid_rxtx_factor(self): + # rxtx_factor attribute should be a positive float + new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) + new_flavor_id = str(uuid.uuid4()) + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + new_flavor_name, + self.ram, self.vcpus, + self.disk, + new_flavor_id, + ephemeral=self.ephemeral, + swap=self.swap, + rxtx=-1.5, + is_public='False') + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_with_invalid_is_public(self): + # is_public attribute should be boolean + new_flavor_name = data_utils.rand_name(self.flavor_name_prefix) + new_flavor_id = str(uuid.uuid4()) + + self.assertRaises(exceptions.BadRequest, + self.client.create_flavor, + new_flavor_name, + self.ram, self.vcpus, + self.disk, + new_flavor_id, + ephemeral=self.ephemeral, + swap=self.swap, + rxtx=self.rxtx, + is_public='Invalid') + + @test.attr(type=['negative', 'gate']) + def test_create_flavor_already_exists(self): + flavor_name = data_utils.rand_name(self.flavor_name_prefix) + new_flavor_id = str(uuid.uuid4()) + + resp, flavor = self.client.create_flavor(flavor_name, + self.ram, self.vcpus, + self.disk, + new_flavor_id, + ephemeral=self.ephemeral, + swap=self.swap, + rxtx=self.rxtx) + self.assertEqual(200, resp.status) + self.addCleanup(self.flavor_clean_up, flavor['id']) + + self.assertRaises(exceptions.Conflict, + self.client.create_flavor, + flavor_name, + self.ram, self.vcpus, + self.disk, + new_flavor_id, + ephemeral=self.ephemeral, + swap=self.swap, + rxtx=self.rxtx) + + @test.attr(type=['negative', 'gate']) + def test_delete_nonexistent_flavor(self): + nonexistent_flavor_id = str(uuid.uuid4()) + + self.assertRaises(exceptions.NotFound, + self.client.delete_flavor, + nonexistent_flavor_id) diff --git a/tempest/common/generator/base_generator.py b/tempest/common/generator/base_generator.py index 35f81589f3..7e7a2d6c3f 100644 --- a/tempest/common/generator/base_generator.py +++ b/tempest/common/generator/base_generator.py @@ -59,7 +59,9 @@ class BasicGeneratorSet(object): "enum": ["GET", "PUT", "HEAD", "POST", "PATCH", "DELETE", 'COPY'] }, + "admin_client": {"type": "boolean"}, "url": {"type": "string"}, + "default_result_code": {"type": "integer"}, "json-schema": jsonschema._utils.load_schema("draft4"), "resources": { "type": "array", diff --git a/tempest/test.py b/tempest/test.py index 212504742a..20581ab094 100644 --- a/tempest/test.py +++ b/tempest/test.py @@ -363,6 +363,9 @@ class NegativeAutoTest(BaseTestCase): super(NegativeAutoTest, cls).setUpClass() os = cls.get_client_manager() cls.client = os.negative_client + os_admin = clients.AdminManager(interface=cls._interface, + service=cls._service) + cls.admin_client = os_admin.negative_client @staticmethod def load_schema(file): @@ -418,10 +421,13 @@ class NegativeAutoTest(BaseTestCase): "expected_result": expected_result })) if schema is not None: - for invalid in generator.generate(schema): - scenario_list.append((invalid[0], - {"schema": invalid[1], - "expected_result": invalid[2]})) + for name, schema, expected_result in generator.generate(schema): + if (expected_result is None and + "default_result_code" in description): + expected_result = description["default_result_code"] + scenario_list.append((name, + {"schema": schema, + "expected_result": expected_result})) LOG.debug(scenario_list) return scenario_list @@ -470,8 +476,12 @@ class NegativeAutoTest(BaseTestCase): elif hasattr(self, "schema"): new_url, body = self._http_arguments(self.schema, url, method) - resp, resp_body = self.client.send_request(method, new_url, - resources, body=body) + if "admin_client" in description and description["admin_client"]: + client = self.admin_client + else: + client = self.client + resp, resp_body = client.send_request(method, new_url, + resources, body=body) self._check_negative_response(resp.status, resp_body) def _http_arguments(self, json_dict, url, method):