From 482e3ce6ab6d21a6349367d8d34d28439adc29c5 Mon Sep 17 00:00:00 2001 From: Jordan Pittier Date: Mon, 3 Oct 2016 10:45:21 +0200 Subject: [PATCH] Remove the NegativeAutoTest Framework Since it's not really used and a bit complex. It was only used for negative compute flavor tests, and I think we can live easily without these tests. Change-Id: Iab676ae9bf95ee858c5e748c9579f7778e87bd77 --- .coveragerc | 2 +- .../compute/flavors/test_flavors_negative.py | 43 ---- tempest/api_schema/__init__.py | 0 tempest/api_schema/request/__init__.py | 0 .../api_schema/request/compute/__init__.py | 0 tempest/api_schema/request/compute/flavors.py | 58 ----- .../api_schema/request/compute/v2/__init__.py | 0 .../api_schema/request/compute/v2/flavors.py | 39 --- tempest/test.py | 225 ------------------ .../tests/negative/test_negative_auto_test.py | 67 ------ tempest/tests/test_decorators.py | 17 -- 11 files changed, 1 insertion(+), 450 deletions(-) delete mode 100644 tempest/api/compute/flavors/test_flavors_negative.py delete mode 100644 tempest/api_schema/__init__.py delete mode 100644 tempest/api_schema/request/__init__.py delete mode 100644 tempest/api_schema/request/compute/__init__.py delete mode 100644 tempest/api_schema/request/compute/flavors.py delete mode 100644 tempest/api_schema/request/compute/v2/__init__.py delete mode 100644 tempest/api_schema/request/compute/v2/flavors.py delete mode 100644 tempest/tests/negative/test_negative_auto_test.py diff --git a/.coveragerc b/.coveragerc index 51482d3545..449e62c2d3 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,4 @@ [run] branch = True source = tempest -omit = tempest/tests/*,tempest/scenario/test_*.py,tempest/api_schema/*,tempest/api/* +omit = tempest/tests/*,tempest/scenario/test_*.py,tempest/api/* diff --git a/tempest/api/compute/flavors/test_flavors_negative.py b/tempest/api/compute/flavors/test_flavors_negative.py deleted file mode 100644 index 83f8e19e5c..0000000000 --- a/tempest/api/compute/flavors/test_flavors_negative.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from tempest.api.compute import base -from tempest.api_schema.request.compute.v2 import flavors -from tempest import config -from tempest import test - - -CONF = config.CONF - -load_tests = test.NegativeAutoTest.load_tests - - -@test.SimpleNegativeAutoTest -class FlavorsListWithDetailsNegativeTestJSON(base.BaseV2ComputeTest, - test.NegativeAutoTest): - _service = CONF.compute.catalog_type - _schema = flavors.flavor_list - - -@test.SimpleNegativeAutoTest -class FlavorDetailsNegativeTestJSON(base.BaseV2ComputeTest, - test.NegativeAutoTest): - _service = CONF.compute.catalog_type - _schema = flavors.flavors_details - - @classmethod - def resource_setup(cls): - super(FlavorDetailsNegativeTestJSON, cls).resource_setup() - cls.set_resource("flavor", cls.flavor_ref) diff --git a/tempest/api_schema/__init__.py b/tempest/api_schema/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tempest/api_schema/request/__init__.py b/tempest/api_schema/request/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tempest/api_schema/request/compute/__init__.py b/tempest/api_schema/request/compute/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tempest/api_schema/request/compute/flavors.py b/tempest/api_schema/request/compute/flavors.py deleted file mode 100644 index adaaf270c0..0000000000 --- a/tempest/api_schema/request/compute/flavors.py +++ /dev/null @@ -1,58 +0,0 @@ -# (c) 2014 Deutsche Telekom AG -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -common_flavor_details = { - "name": "get-flavor-details", - "http-method": "GET", - "url": "flavors/%s", - "resources": [ - {"name": "flavor", "expected_result": 404} - ] -} - -common_flavor_list = { - "name": "list-flavors-with-detail", - "http-method": "GET", - "url": "flavors/detail", - "json-schema": { - "type": "object", - "properties": { - } - } -} - -common_admin_flavor_create = { - "name": "flavor-create", - "http-method": "POST", - "admin_client": True, - "url": "flavors", - "default_result_code": 400, - "json-schema": { - "type": "object", - "properties": { - "flavor": { - "type": "object", - "properties": { - "name": {"type": "string", - "exclude_tests": ["gen_str_min_length"]}, - "ram": {"type": "integer", "minimum": 1}, - "vcpus": {"type": "integer", "minimum": 1}, - "disk": {"type": "integer"}, - "id": {"type": "integer", - "exclude_tests": ["gen_none", "gen_string"] - }, - } - } - } - } -} diff --git a/tempest/api_schema/request/compute/v2/__init__.py b/tempest/api_schema/request/compute/v2/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tempest/api_schema/request/compute/v2/flavors.py b/tempest/api_schema/request/compute/v2/flavors.py deleted file mode 100644 index bc459ad477..0000000000 --- a/tempest/api_schema/request/compute/v2/flavors.py +++ /dev/null @@ -1,39 +0,0 @@ -# (c) 2014 Deutsche Telekom AG -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy - -from tempest.api_schema.request.compute import flavors - -flavors_details = copy.deepcopy(flavors.common_flavor_details) - -flavor_list = copy.deepcopy(flavors.common_flavor_list) - -flavor_create = copy.deepcopy(flavors.common_admin_flavor_create) - -flavor_list["json-schema"]["properties"] = { - "minRam": { - "type": "integer", - "results": { - "gen_none": 400, - "gen_string": 400 - } - }, - "minDisk": { - "type": "integer", - "results": { - "gen_none": 400, - "gen_string": 400 - } - } -} diff --git a/tempest/test.py b/tempest/test.py index 609f1f69e7..6dc065ca4e 100644 --- a/tempest/test.py +++ b/tempest/test.py @@ -16,28 +16,21 @@ import atexit import functools import os -import re import sys import debtcollector.moves import fixtures from oslo_log import log as logging -from oslo_serialization import jsonutils as json -from oslo_utils import importutils import six -from six.moves import urllib -import testscenarios import testtools from tempest import clients from tempest.common import cred_client from tempest.common import credentials_factory as credentials from tempest.common import fixed_network -import tempest.common.generator.valid_generator as valid import tempest.common.validation_resources as vresources from tempest import config from tempest import exceptions -from tempest.lib.common.utils import data_utils from tempest.lib.common.utils import test_utils from tempest.lib import decorators from tempest.lib import exceptions as lib_exc @@ -649,224 +642,6 @@ class BaseTestCase(testtools.testcase.WithAttributes, self.assertTrue(len(list) > 0, msg) -class NegativeAutoTest(BaseTestCase): - - _resources = {} - - @classmethod - def setUpClass(cls): - super(NegativeAutoTest, cls).setUpClass() - os = cls.get_client_manager(credential_type='primary') - cls.client = os.negative_client - - @staticmethod - def load_tests(*args): - """Wrapper for testscenarios - - To set the mandatory scenarios variable only in case a real test - loader is in place. Will be automatically called in case the variable - "load_tests" is set. - """ - if getattr(args[0], 'suiteClass', None) is not None: - loader, standard_tests, pattern = args - else: - standard_tests, module, loader = args - for test in testtools.iterate_tests(standard_tests): - schema = getattr(test, '_schema', None) - if schema is not None: - setattr(test, 'scenarios', - NegativeAutoTest.generate_scenario(schema)) - return testscenarios.load_tests_apply_scenarios(*args) - - @staticmethod - def generate_scenario(description): - """Generates the test scenario list for a given description. - - :param description: A file or dictionary with the following entries: - name (required) name for the api - http-method (required) one of HEAD,GET,PUT,POST,PATCH,DELETE - url (required) the url to be appended to the catalog url with '%s' - for each resource mentioned - resources: (optional) A list of resource names such as "server", - "flavor", etc. with an element for each '%s' in the url. This - method will call self.get_resource for each element when - constructing the positive test case template so negative - subclasses are expected to return valid resource ids when - appropriate. - json-schema (optional) A valid json schema that will be used to - create invalid data for the api calls. For "GET" and "HEAD", - the data is used to generate query strings appended to the url, - otherwise for the body of the http call. - """ - LOG.debug(description) - generator = importutils.import_class( - CONF.negative.test_generator)() - generator.validate_schema(description) - schema = description.get("json-schema", None) - resources = description.get("resources", []) - scenario_list = [] - expected_result = None - for resource in resources: - if isinstance(resource, dict): - expected_result = resource['expected_result'] - resource = resource['name'] - LOG.debug("Add resource to test %s" % resource) - scn_name = "inv_res_%s" % (resource) - scenario_list.append((scn_name, { - "resource": (resource, data_utils.rand_uuid()), - "expected_result": expected_result - })) - if schema is not None: - for scenario in generator.generate_scenarios(schema): - scenario_list.append((scenario['_negtest_name'], - scenario)) - LOG.debug(scenario_list) - return scenario_list - - def execute(self, description): - """Execute a http call - - Execute a http call on an api that are expected to - result in client errors. First it uses invalid resources that are part - of the url, and then invalid data for queries and http request bodies. - - :param description: A json file or dictionary with the following - entries: - name (required) name for the api - http-method (required) one of HEAD,GET,PUT,POST,PATCH,DELETE - url (required) the url to be appended to the catalog url with '%s' - for each resource mentioned - resources: (optional) A list of resource names such as "server", - "flavor", etc. with an element for each '%s' in the url. This - method will call self.get_resource for each element when - constructing the positive test case template so negative - subclasses are expected to return valid resource ids when - appropriate. - json-schema (optional) A valid json schema that will be used to - create invalid data for the api calls. For "GET" and "HEAD", - the data is used to generate query strings appended to the url, - otherwise for the body of the http call. - - """ - LOG.info("Executing %s" % description["name"]) - LOG.debug(description) - generator = importutils.import_class( - CONF.negative.test_generator)() - schema = description.get("json-schema", None) - method = description["http-method"] - url = description["url"] - expected_result = None - if "default_result_code" in description: - expected_result = description["default_result_code"] - - resources = [self.get_resource(r) for - r in description.get("resources", [])] - - if hasattr(self, "resource"): - # Note(mkoderer): The resources list already contains an invalid - # entry (see get_resource). - # We just send a valid json-schema with it - valid_schema = None - if schema: - valid_schema = \ - valid.ValidTestGenerator().generate_valid(schema) - new_url, body = self._http_arguments(valid_schema, url, method) - elif hasattr(self, "_negtest_name"): - schema_under_test = \ - valid.ValidTestGenerator().generate_valid(schema) - local_expected_result = \ - generator.generate_payload(self, schema_under_test) - if local_expected_result is not None: - expected_result = local_expected_result - new_url, body = \ - self._http_arguments(schema_under_test, url, method) - else: - raise Exception("testscenarios are not active. Please make sure " - "that your test runner supports the load_tests " - "mechanism") - - if "admin_client" in description and description["admin_client"]: - if not credentials.is_admin_available( - identity_version=self.get_identity_version()): - msg = ("Missing Identity Admin API credentials in" - "configuration.") - raise self.skipException(msg) - creds = self.credentials_provider.get_admin_creds() - os_adm = clients.Manager(credentials=creds) - client = os_adm.negative_client - else: - client = self.client - resp, resp_body = client.send_request(method, new_url, - resources, body=body) - self._check_negative_response(expected_result, resp.status, resp_body) - - def _http_arguments(self, json_dict, url, method): - LOG.debug("dict: %s url: %s method: %s" % (json_dict, url, method)) - if not json_dict: - return url, None - elif method in ["GET", "HEAD", "PUT", "DELETE"]: - return "%s?%s" % (url, urllib.parse.urlencode(json_dict)), None - else: - return url, json.dumps(json_dict) - - def _check_negative_response(self, expected_result, result, body): - self.assertTrue(result >= 400 and result < 500 and result != 413, - "Expected client error, got %s:%s" % - (result, body)) - self.assertTrue(expected_result is None or expected_result == result, - "Expected %s, got %s:%s" % - (expected_result, result, body)) - - @classmethod - def set_resource(cls, name, resource): - """Register a resource for a test - - This function can be used in setUpClass context to register a resource - for a test. - - :param name: The name of the kind of resource such as "flavor", "role", - etc. - :resource: The id of the resource - """ - cls._resources[name] = resource - - def get_resource(self, name): - """Return a valid uuid for a type of resource. - - If a real resource is needed as part of a url then this method should - return one. Otherwise it can return None. - - :param name: The name of the kind of resource such as "flavor", "role", - etc. - """ - if isinstance(name, dict): - name = name['name'] - if hasattr(self, "resource") and self.resource[0] == name: - LOG.debug("Return invalid resource (%s) value: %s" % - (self.resource[0], self.resource[1])) - return self.resource[1] - if name in self._resources: - return self._resources[name] - return None - - -def SimpleNegativeAutoTest(klass): - """This decorator registers a test function on basis of the class name.""" - @attr(type=['negative']) - def generic_test(self): - if hasattr(self, '_schema'): - self.execute(self._schema) - - cn = klass.__name__ - cn = cn.replace('JSON', '') - cn = cn.replace('Test', '') - # NOTE(mkoderer): replaces uppercase chars inside the class name with '_' - lower_cn = re.sub('(?