From 6ee82dccd1dbcb95fc63eb3c07b78db0432723ea Mon Sep 17 00:00:00 2001 From: Marc Koderer Date: Mon, 17 Feb 2014 10:26:29 +0100 Subject: [PATCH] Add multiple negative test generator support In order to support different generator styles and sets of tests a new configuration parameter is introduced to define the negative test generator class. This can be used to define generators that create only random values (random fuzzy test) or pattern based values. With this functionality it is also possible to reduce the amount of negative tests that are automatically produced. Change-Id: Icfad55d1eea92dc2a42642b37d34c253c26c0846 Partially-implements: bp fuzzy-test --- etc/tempest.conf.sample | 10 + tempest/common/generate_json.py | 265 ------------------ tempest/common/generator/__init__.py | 0 tempest/common/generator/base_generator.py | 141 ++++++++++ .../common/generator/negative_generator.py | 111 ++++++++ tempest/common/generator/valid_generator.py | 58 ++++ tempest/config.py | 11 + tempest/test.py | 15 +- tempest/tests/fake_config.py | 6 + tempest/tests/negative/test_generate_json.py | 14 +- .../tests/negative/test_negative_auto_test.py | 8 +- 11 files changed, 362 insertions(+), 277 deletions(-) delete mode 100644 tempest/common/generate_json.py create mode 100644 tempest/common/generator/__init__.py create mode 100644 tempest/common/generator/base_generator.py create mode 100644 tempest/common/generator/negative_generator.py create mode 100644 tempest/common/generator/valid_generator.py diff --git a/etc/tempest.conf.sample b/etc/tempest.conf.sample index ee2da40856..e46b813b90 100644 --- a/etc/tempest.conf.sample +++ b/etc/tempest.conf.sample @@ -548,6 +548,16 @@ #ssh_user_regex=[["^.*[Cc]irros.*$", "root"]] +[negative] + +# +# Options defined in tempest.config +# + +# Test generator class for all negative tests (string value) +#test_generator=tempest.common.generator.negative_generator.NegativeTestGenerator + + [network] # diff --git a/tempest/common/generate_json.py b/tempest/common/generate_json.py deleted file mode 100644 index c8e86dca55..0000000000 --- a/tempest/common/generate_json.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright 2014 Red Hat, Inc. & Deutsche Telekom AG -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import jsonschema - -from tempest.openstack.common import log as logging - -LOG = logging.getLogger(__name__) - - -def generate_valid(schema): - """ - Create a valid dictionary based on the types in a json schema. - """ - LOG.debug("generate_valid: %s" % schema) - schema_type = schema["type"] - if isinstance(schema_type, list): - # Just choose the first one since all are valid. - schema_type = schema_type[0] - return type_map_valid[schema_type](schema) - - -def generate_valid_string(schema): - size = schema.get("minLength", 0) - # TODO(dkr mko): handle format and pattern - return "x" * size - - -def generate_valid_integer(schema): - # TODO(dkr mko): handle multipleOf - if "minimum" in schema: - minimum = schema["minimum"] - if "exclusiveMinimum" not in schema: - return minimum - else: - return minimum + 1 - if "maximum" in schema: - maximum = schema["maximum"] - if "exclusiveMaximum" not in schema: - return maximum - else: - return maximum - 1 - return 0 - - -def generate_valid_object(schema): - obj = {} - for k, v in schema["properties"].iteritems(): - obj[k] = generate_valid(v) - return obj - - -def generate_invalid(schema): - """ - Generate an invalid json dictionary based on a schema. - Only one value is mis-generated for each dictionary created. - - Any generator must return a list of tuples or a single tuple. - The values of this tuple are: - result[0]: Name of the test - result[1]: json schema for the test - result[2]: expected result of the test (can be None) - """ - LOG.debug("generate_invalid: %s" % schema) - schema_type = schema["type"] - if isinstance(schema_type, list): - if "integer" in schema_type: - schema_type = "integer" - else: - raise Exception("non-integer list types not supported") - result = [] - for generator in type_map_invalid[schema_type]: - ret = generator(schema) - if ret is not None: - if isinstance(ret, list): - result.extend(ret) - elif isinstance(ret, tuple): - result.append(ret) - else: - raise Exception("generator (%s) returns invalid result" - % generator) - LOG.debug("result: %s" % result) - return result - - -def _check_for_expected_result(name, schema): - expected_result = None - if "results" in schema: - if name in schema["results"]: - expected_result = schema["results"][name] - return expected_result - - -def generator(fn): - """ - Decorator for simple generators that simply return one value - """ - def wrapped(schema): - result = fn(schema) - if result is not None: - expected_result = _check_for_expected_result(fn.__name__, schema) - return (fn.__name__, result, expected_result) - return - return wrapped - - -@generator -def gen_int(_): - return 4 - - -@generator -def gen_string(_): - return "XXXXXX" - - -def gen_none(schema): - # Note(mkoderer): it's not using the decorator otherwise it'd be filtered - expected_result = _check_for_expected_result('gen_none', schema) - return ('gen_none', None, expected_result) - - -@generator -def gen_str_min_length(schema): - min_length = schema.get("minLength", 0) - if min_length > 0: - return "x" * (min_length - 1) - - -@generator -def gen_str_max_length(schema): - max_length = schema.get("maxLength", -1) - if max_length > -1: - return "x" * (max_length + 1) - - -@generator -def gen_int_min(schema): - if "minimum" in schema: - minimum = schema["minimum"] - if "exclusiveMinimum" not in schema: - minimum -= 1 - return minimum - - -@generator -def gen_int_max(schema): - if "maximum" in schema: - maximum = schema["maximum"] - if "exclusiveMaximum" not in schema: - maximum += 1 - return maximum - - -def gen_obj_remove_attr(schema): - invalids = [] - valid = generate_valid(schema) - required = schema.get("required", []) - for r in required: - new_valid = copy.deepcopy(valid) - del new_valid[r] - invalids.append(("gen_obj_remove_attr", new_valid, None)) - return invalids - - -@generator -def gen_obj_add_attr(schema): - valid = generate_valid(schema) - if not schema.get("additionalProperties", True): - new_valid = copy.deepcopy(valid) - new_valid["$$$$$$$$$$"] = "xxx" - return new_valid - - -def gen_inv_prop_obj(schema): - LOG.debug("generate_invalid_object: %s" % schema) - valid = generate_valid(schema) - invalids = [] - properties = schema["properties"] - - for k, v in properties.iteritems(): - for invalid in generate_invalid(v): - LOG.debug(v) - new_valid = copy.deepcopy(valid) - new_valid[k] = invalid[1] - name = "prop_%s_%s" % (k, invalid[0]) - invalids.append((name, new_valid, invalid[2])) - - LOG.debug("generate_invalid_object return: %s" % invalids) - return invalids - - -type_map_valid = { - "string": generate_valid_string, - "integer": generate_valid_integer, - "object": generate_valid_object -} - -type_map_invalid = { - "string": [ - gen_int, - gen_none, - gen_str_min_length, - gen_str_max_length], - "integer": [ - gen_string, - gen_none, - gen_int_min, - gen_int_max], - "object": [ - gen_obj_remove_attr, - gen_obj_add_attr, - gen_inv_prop_obj] -} - -schema = { - "type": "object", - "properties": { - "name": {"type": "string"}, - "http-method": { - "enum": ["GET", "PUT", "HEAD", - "POST", "PATCH", "DELETE", 'COPY'] - }, - "url": {"type": "string"}, - "json-schema": jsonschema._utils.load_schema("draft4"), - "resources": { - "type": "array", - "items": { - "oneOf": [ - {"type": "string"}, - { - "type": "object", - "properties": { - "name": {"type": "string"}, - "expected_result": {"type": "integer"} - } - } - ] - } - }, - "results": { - "type": "object", - "properties": {} - } - }, - "required": ["name", "http-method", "url"], - "additionalProperties": False, -} - - -def validate_negative_test_schema(nts): - jsonschema.validate(nts, schema) diff --git a/tempest/common/generator/__init__.py b/tempest/common/generator/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tempest/common/generator/base_generator.py b/tempest/common/generator/base_generator.py new file mode 100644 index 0000000000..35f81589f3 --- /dev/null +++ b/tempest/common/generator/base_generator.py @@ -0,0 +1,141 @@ +# Copyright 2014 Deutsche Telekom AG +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import jsonschema + +from tempest.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + + +def _check_for_expected_result(name, schema): + expected_result = None + if "results" in schema: + if name in schema["results"]: + expected_result = schema["results"][name] + return expected_result + + +def generator_type(*args): + def wrapper(func): + func.types = args + return func + return wrapper + + +def simple_generator(fn): + """ + Decorator for simple generators that return one value + """ + def wrapped(self, schema): + result = fn(self, schema) + if result is not None: + expected_result = _check_for_expected_result(fn.__name__, schema) + return (fn.__name__, result, expected_result) + return + return wrapped + + +class BasicGeneratorSet(object): + _instance = None + + schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "http-method": { + "enum": ["GET", "PUT", "HEAD", + "POST", "PATCH", "DELETE", 'COPY'] + }, + "url": {"type": "string"}, + "json-schema": jsonschema._utils.load_schema("draft4"), + "resources": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "name": {"type": "string"}, + "expected_result": {"type": "integer"} + } + } + ] + } + }, + "results": { + "type": "object", + "properties": {} + } + }, + "required": ["name", "http-method", "url"], + "additionalProperties": False, + } + + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(BasicGeneratorSet, cls).__new__(cls, *args, + **kwargs) + return cls._instance + + def __init__(self): + self.types_dict = {} + for m in dir(self): + if callable(getattr(self, m)) and not'__' in m: + method = getattr(self, m) + if hasattr(method, "types"): + for type in method.types: + if type not in self.types_dict: + self.types_dict[type] = [] + self.types_dict[type].append(method) + + def validate_schema(self, schema): + jsonschema.validate(schema, self.schema) + + def generate(self, schema): + """ + Generate an json dictionary based on a schema. + Only one value is mis-generated for each dictionary created. + + Any generator must return a list of tuples or a single tuple. + The values of this tuple are: + result[0]: Name of the test + result[1]: json schema for the test + result[2]: expected result of the test (can be None) + """ + LOG.debug("generate_invalid: %s" % schema) + schema_type = schema["type"] + if isinstance(schema_type, list): + if "integer" in schema_type: + schema_type = "integer" + else: + raise Exception("non-integer list types not supported") + result = [] + if schema_type not in self.types_dict: + raise Exception("generator (%s) doesn't support type: %s" + % (self.__class__.__name__, schema_type)) + for generator in self.types_dict[schema_type]: + ret = generator(schema) + if ret is not None: + if isinstance(ret, list): + result.extend(ret) + elif isinstance(ret, tuple): + result.append(ret) + else: + raise Exception("generator (%s) returns invalid result: %s" + % (generator, ret)) + LOG.debug("result: %s" % result) + return result diff --git a/tempest/common/generator/negative_generator.py b/tempest/common/generator/negative_generator.py new file mode 100644 index 0000000000..4f3d2cd597 --- /dev/null +++ b/tempest/common/generator/negative_generator.py @@ -0,0 +1,111 @@ +# Copyright 2014 Deutsche Telekom AG +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +import tempest.common.generator.base_generator as base +import tempest.common.generator.valid_generator as valid +from tempest.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + + +class NegativeTestGenerator(base.BasicGeneratorSet): + @base.generator_type("string") + @base.simple_generator + def gen_int(self, _): + return 4 + + @base.generator_type("integer") + @base.simple_generator + def gen_string(self, _): + return "XXXXXX" + + @base.generator_type("integer", "string") + def gen_none(self, schema): + # Note(mkoderer): it's not using the decorator otherwise it'd be + # filtered + expected_result = base._check_for_expected_result('gen_none', schema) + return ('gen_none', None, expected_result) + + @base.generator_type("string") + @base.simple_generator + def gen_str_min_length(self, schema): + min_length = schema.get("minLength", 0) + if min_length > 0: + return "x" * (min_length - 1) + + @base.generator_type("string") + @base.simple_generator + def gen_str_max_length(self, schema): + max_length = schema.get("maxLength", -1) + if max_length > -1: + return "x" * (max_length + 1) + + @base.generator_type("integer") + @base.simple_generator + def gen_int_min(self, schema): + if "minimum" in schema: + minimum = schema["minimum"] + if "exclusiveMinimum" not in schema: + minimum -= 1 + return minimum + + @base.generator_type("integer") + @base.simple_generator + def gen_int_max(self, schema): + if "maximum" in schema: + maximum = schema["maximum"] + if "exclusiveMaximum" not in schema: + maximum += 1 + return maximum + + @base.generator_type("object") + def gen_obj_remove_attr(self, schema): + invalids = [] + valid_schema = valid.ValidTestGenerator().generate_valid(schema) + required = schema.get("required", []) + for r in required: + new_valid = copy.deepcopy(valid_schema) + del new_valid[r] + invalids.append(("gen_obj_remove_attr", new_valid, None)) + return invalids + + @base.generator_type("object") + @base.simple_generator + def gen_obj_add_attr(self, schema): + valid_schema = valid.ValidTestGenerator().generate_valid(schema) + if not schema.get("additionalProperties", True): + new_valid = copy.deepcopy(valid_schema) + new_valid["$$$$$$$$$$"] = "xxx" + return new_valid + + @base.generator_type("object") + def gen_inv_prop_obj(self, schema): + LOG.debug("generate_invalid_object: %s" % schema) + valid_schema = valid.ValidTestGenerator().generate_valid(schema) + invalids = [] + properties = schema["properties"] + + for k, v in properties.iteritems(): + for invalid in self.generate(v): + LOG.debug(v) + new_valid = copy.deepcopy(valid_schema) + new_valid[k] = invalid[1] + name = "prop_%s_%s" % (k, invalid[0]) + invalids.append((name, new_valid, invalid[2])) + + LOG.debug("generate_invalid_object return: %s" % invalids) + return invalids diff --git a/tempest/common/generator/valid_generator.py b/tempest/common/generator/valid_generator.py new file mode 100644 index 0000000000..a99bbc019b --- /dev/null +++ b/tempest/common/generator/valid_generator.py @@ -0,0 +1,58 @@ +# Copyright 2014 Deutsche Telekom AG +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tempest.common.generator.base_generator as base +from tempest.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +class ValidTestGenerator(base.BasicGeneratorSet): + @base.generator_type("string") + @base.simple_generator + def generate_valid_string(self, schema): + size = schema.get("minLength", 0) + # TODO(dkr mko): handle format and pattern + return "x" * size + + @base.generator_type("integer") + @base.simple_generator + def generate_valid_integer(self, schema): + # TODO(dkr mko): handle multipleOf + if "minimum" in schema: + minimum = schema["minimum"] + if "exclusiveMinimum" not in schema: + return minimum + else: + return minimum + 1 + if "maximum" in schema: + maximum = schema["maximum"] + if "exclusiveMaximum" not in schema: + return maximum + else: + return maximum - 1 + return 0 + + @base.generator_type("object") + @base.simple_generator + def generate_valid_object(self, schema): + obj = {} + for k, v in schema["properties"].iteritems(): + obj[k] = self.generate_valid(v) + return obj + + def generate_valid(self, schema): + return self.generate(schema)[0][1] diff --git a/tempest/config.py b/tempest/config.py index 0f5e23c6d0..26b3bf1808 100644 --- a/tempest/config.py +++ b/tempest/config.py @@ -806,6 +806,15 @@ CLIGroup = [ help="Number of seconds to wait on a CLI timeout"), ] +negative_group = cfg.OptGroup(name='negative', title="Negative Test Options") + +NegativeGroup = [ + cfg.StrOpt('test_generator', + default='tempest.common.' + + 'generator.negative_generator.NegativeTestGenerator', + help="Test generator class for all negative tests"), +] + def register_opts(): register_opt_group(cfg.CONF, compute_group, ComputeGroup) @@ -840,6 +849,7 @@ def register_opts(): register_opt_group(cfg.CONF, baremetal_group, BaremetalGroup) register_opt_group(cfg.CONF, input_scenario_group, InputScenarioGroup) register_opt_group(cfg.CONF, cli_group, CLIGroup) + register_opt_group(cfg.CONF, negative_group, NegativeGroup) # this should never be called outside of this class @@ -879,6 +889,7 @@ class TempestConfigPrivate(object): self.baremetal = cfg.CONF.baremetal self.input_scenario = cfg.CONF['input-scenario'] self.cli = cfg.CONF.cli + self.negative = cfg.CONF.negative if not self.compute_admin.username: self.compute_admin.username = self.identity.admin_username self.compute_admin.password = self.identity.admin_password diff --git a/tempest/test.py b/tempest/test.py index c6e3d6e77b..212504742a 100644 --- a/tempest/test.py +++ b/tempest/test.py @@ -27,10 +27,11 @@ import testresources import testtools from tempest import clients -from tempest.common import generate_json +import tempest.common.generator.valid_generator as valid from tempest.common import isolated_creds from tempest import config from tempest import exceptions +from tempest.openstack.common import importutils from tempest.openstack.common import log as logging LOG = logging.getLogger(__name__) @@ -400,7 +401,8 @@ class NegativeAutoTest(BaseTestCase): """ description = NegativeAutoTest.load_schema(description_file) LOG.debug(description) - generate_json.validate_negative_test_schema(description) + generator = importutils.import_class(CONF.negative.test_generator)() + generator.validate_schema(description) schema = description.get("json-schema", None) resources = description.get("resources", []) scenario_list = [] @@ -416,7 +418,7 @@ class NegativeAutoTest(BaseTestCase): "expected_result": expected_result })) if schema is not None: - for invalid in generate_json.generate_invalid(schema): + for invalid in generator.generate(schema): scenario_list.append((invalid[0], {"schema": invalid[1], "expected_result": invalid[2]})) @@ -459,11 +461,12 @@ class NegativeAutoTest(BaseTestCase): # Note(mkoderer): The resources list already contains an invalid # entry (see get_resource). # We just send a valid json-schema with it - valid = None + valid_schema = None schema = description.get("json-schema", None) if schema: - valid = generate_json.generate_valid(schema) - new_url, body = self._http_arguments(valid, url, method) + valid_schema = \ + valid.ValidTestGenerator().generate_valid(schema) + new_url, body = self._http_arguments(valid_schema, url, method) elif hasattr(self, "schema"): new_url, body = self._http_arguments(self.schema, url, method) diff --git a/tempest/tests/fake_config.py b/tempest/tests/fake_config.py index 41b0558eed..e941606be2 100644 --- a/tempest/tests/fake_config.py +++ b/tempest/tests/fake_config.py @@ -43,6 +43,10 @@ class FakeConfig(object): swift = True horizon = True + class fake_negative(object): + test_generator = 'tempest.common.' \ + 'generator.negative_generator.NegativeTestGenerator' + compute_feature_enabled = fake_compute_feature_enabled() volume_feature_enabled = fake_default_feature_enabled() network_feature_enabled = fake_default_feature_enabled() @@ -52,3 +56,5 @@ class FakeConfig(object): compute = fake_compute() identity = fake_identity() + + negative = fake_negative() diff --git a/tempest/tests/negative/test_generate_json.py b/tempest/tests/negative/test_generate_json.py index a0aa088d10..e09fcdf480 100644 --- a/tempest/tests/negative/test_generate_json.py +++ b/tempest/tests/negative/test_generate_json.py @@ -13,11 +13,11 @@ # License for the specific language governing permissions and limitations # under the License. -from tempest.common import generate_json as gen +from tempest.common.generator import negative_generator import tempest.test -class TestGenerateJson(tempest.test.BaseTestCase): +class TestNegativeGenerator(tempest.test.BaseTestCase): fake_input_str = {"type": "string", "minLength": 2, @@ -35,19 +35,23 @@ class TestGenerateJson(tempest.test.BaseTestCase): } } + def setUp(self): + super(TestNegativeGenerator, self).setUp() + self.negative = negative_generator.NegativeTestGenerator() + def _validate_result(self, data): self.assertTrue(isinstance(data, list)) for t in data: self.assertTrue(isinstance(t, tuple)) def test_generate_invalid_string(self): - result = gen.generate_invalid(self.fake_input_str) + result = self.negative.generate(self.fake_input_str) self._validate_result(result) def test_generate_invalid_integer(self): - result = gen.generate_invalid(self.fake_input_int) + result = self.negative.generate(self.fake_input_int) self._validate_result(result) def test_generate_invalid_obj(self): - result = gen.generate_invalid(self.fake_input_obj) + result = self.negative.generate(self.fake_input_obj) self._validate_result(result) diff --git a/tempest/tests/negative/test_negative_auto_test.py b/tempest/tests/negative/test_negative_auto_test.py index 4c593838a2..27ddc953d6 100644 --- a/tempest/tests/negative/test_negative_auto_test.py +++ b/tempest/tests/negative/test_negative_auto_test.py @@ -16,9 +16,11 @@ import mock import tempest.test as test +from tempest.tests import base +from tempest.tests import fake_config -class TestNegativeAutoTest(test.BaseTestCase): +class TestNegativeAutoTest(base.TestCase): # Fake entries _interface = 'json' _service = 'compute' @@ -34,6 +36,10 @@ class TestNegativeAutoTest(test.BaseTestCase): "resources": ["flavor", "volume", "image"] } + def setUp(self): + super(TestNegativeAutoTest, self).setUp() + self.stubs.Set(test, 'CONF', fake_config.FakeConfig) + def _check_prop_entries(self, result, entry): entries = [a for a in result if entry in a[0]] self.assertIsNotNone(entries)