openstack/tempest
Code Issues Proposed changes

Add multiple negative test generator support

Browse Source 
In order to support different generator styles and sets of tests
a new configuration parameter is introduced to define the negative
test generator class. This can be used to define generators that create
only random values (random fuzzy test) or pattern based values.
With this functionality it is also possible to reduce the amount
of negative tests that are automatically produced.

Change-Id: Icfad55d1eea92dc2a42642b37d34c253c26c0846
Partially-implements: bp fuzzy-test
This commit is contained in:
Marc Koderer 2014-02-17 10:26:29 +01:00
parent ff956a6669
commit 6ee82dccd1
11 changed files with 362 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
etc/tempest.conf.sample
View File

@ -548,6 +548,16 @@
#ssh_user_regex=[["^.*[Cc]irros.*$", "root"]]
[negative]
#
# Options defined in tempest.config
#
# Test generator class for all negative tests (string value)
#test_generator=tempest.common.generator.negative_generator.NegativeTestGenerator
[network]
#

265
tempest/common/generate_json.py
View File

@ -1,265 +0,0 @@
# Copyright 2014 Red Hat, Inc. & Deutsche Telekom AG
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import jsonschema
from tempest.openstack.common import log as logging
LOG = logging.getLogger(__name__)
def generate_valid(schema):
"""
Create a valid dictionary based on the types in a json schema.
"""
LOG.debug("generate_valid: %s" % schema)
schema_type = schema["type"]
if isinstance(schema_type, list):
# Just choose the first one since all are valid.
schema_type = schema_type[0]
return type_map_valid[schema_type](schema)
def generate_valid_string(schema):
size = schema.get("minLength", 0)
# TODO(dkr mko): handle format and pattern
return "x" * size
def generate_valid_integer(schema):
# TODO(dkr mko): handle multipleOf
if "minimum" in schema:
minimum = schema["minimum"]
if "exclusiveMinimum" not in schema:
return minimum
else:
return minimum + 1
if "maximum" in schema:
maximum = schema["maximum"]
if "exclusiveMaximum" not in schema:
return maximum
else:
return maximum - 1
return 0
def generate_valid_object(schema):
obj = {}
for k, v in schema["properties"].iteritems():
obj[k] = generate_valid(v)
return obj
def generate_invalid(schema):
"""
Generate an invalid json dictionary based on a schema.
Only one value is mis-generated for each dictionary created.
Any generator must return a list of tuples or a single tuple.
The values of this tuple are:
result[0]: Name of the test
result[1]: json schema for the test
result[2]: expected result of the test (can be None)
"""
LOG.debug("generate_invalid: %s" % schema)
schema_type = schema["type"]
if isinstance(schema_type, list):
if "integer" in schema_type:
schema_type = "integer"
else:
raise Exception("non-integer list types not supported")
result = []
for generator in type_map_invalid[schema_type]:
ret = generator(schema)
if ret is not None:
if isinstance(ret, list):
result.extend(ret)
elif isinstance(ret, tuple):
result.append(ret)
else:
raise Exception("generator (%s) returns invalid result"
% generator)
LOG.debug("result: %s" % result)
return result
def _check_for_expected_result(name, schema):
expected_result = None
if "results" in schema:
if name in schema["results"]:
expected_result = schema["results"][name]
return expected_result
def generator(fn):
"""
Decorator for simple generators that simply return one value
"""
def wrapped(schema):
result = fn(schema)
if result is not None:
expected_result = _check_for_expected_result(fn.__name__, schema)
return (fn.__name__, result, expected_result)
return
return wrapped
@generator
def gen_int(_):
return 4
@generator
def gen_string(_):
return "XXXXXX"
def gen_none(schema):
# Note(mkoderer): it's not using the decorator otherwise it'd be filtered
expected_result = _check_for_expected_result('gen_none', schema)
return ('gen_none', None, expected_result)
@generator
def gen_str_min_length(schema):
min_length = schema.get("minLength", 0)
if min_length > 0:
return "x" * (min_length - 1)
@generator
def gen_str_max_length(schema):
max_length = schema.get("maxLength", -1)
if max_length > -1:
return "x" * (max_length + 1)
@generator
def gen_int_min(schema):
if "minimum" in schema:
minimum = schema["minimum"]
if "exclusiveMinimum" not in schema:
minimum -= 1
return minimum
@generator
def gen_int_max(schema):
if "maximum" in schema:
maximum = schema["maximum"]
if "exclusiveMaximum" not in schema:
maximum += 1
return maximum
def gen_obj_remove_attr(schema):
invalids = []
valid = generate_valid(schema)
required = schema.get("required", [])
for r in required:
new_valid = copy.deepcopy(valid)
del new_valid[r]
invalids.append(("gen_obj_remove_attr", new_valid, None))
return invalids
@generator
def gen_obj_add_attr(schema):
valid = generate_valid(schema)
if not schema.get("additionalProperties", True):
new_valid = copy.deepcopy(valid)
new_valid["$$$$$$$$$$"] = "xxx"
return new_valid
def gen_inv_prop_obj(schema):
LOG.debug("generate_invalid_object: %s" % schema)
valid = generate_valid(schema)
invalids = []
properties = schema["properties"]
for k, v in properties.iteritems():
for invalid in generate_invalid(v):
LOG.debug(v)
new_valid = copy.deepcopy(valid)
new_valid[k] = invalid[1]
name = "prop_%s_%s" % (k, invalid[0])
invalids.append((name, new_valid, invalid[2]))
LOG.debug("generate_invalid_object return: %s" % invalids)
return invalids
type_map_valid = {
"string": generate_valid_string,
"integer": generate_valid_integer,
"object": generate_valid_object
}
type_map_invalid = {
"string": [
gen_int,
gen_none,
gen_str_min_length,
gen_str_max_length],
"integer": [
gen_string,
gen_none,
gen_int_min,
gen_int_max],
"object": [
gen_obj_remove_attr,
gen_obj_add_attr,
gen_inv_prop_obj]
}
schema = {
"type": "object",
"properties": {
"name": {"type": "string"},
"http-method": {
"enum": ["GET", "PUT", "HEAD",
"POST", "PATCH", "DELETE", 'COPY']
},
"url": {"type": "string"},
"json-schema": jsonschema._utils.load_schema("draft4"),
"resources": {
"type": "array",
"items": {
"oneOf": [
{"type": "string"},
{
"type": "object",
"properties": {
"name": {"type": "string"},
"expected_result": {"type": "integer"}
}
}
]
}
},
"results": {
"type": "object",
"properties": {}
}
},
"required": ["name", "http-method", "url"],
"additionalProperties": False,
}
def validate_negative_test_schema(nts):
jsonschema.validate(nts, schema)

0
tempest/common/generator/__init__.py Normal file
View File

141
tempest/common/generator/base_generator.py Normal file
View File

@ -0,0 +1,141 @@
# Copyright 2014 Deutsche Telekom AG
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import jsonschema
from tempest.openstack.common import log as logging
LOG = logging.getLogger(__name__)
def _check_for_expected_result(name, schema):
expected_result = None
if "results" in schema:
if name in schema["results"]:
expected_result = schema["results"][name]
return expected_result
def generator_type(*args):
def wrapper(func):
func.types = args
return func
return wrapper
def simple_generator(fn):
"""
Decorator for simple generators that return one value
"""
def wrapped(self, schema):
result = fn(self, schema)
if result is not None:
expected_result = _check_for_expected_result(fn.__name__, schema)
return (fn.__name__, result, expected_result)
return
return wrapped
class BasicGeneratorSet(object):
_instance = None
schema = {
"type": "object",
"properties": {
"name": {"type": "string"},
"http-method": {
"enum": ["GET", "PUT", "HEAD",
"POST", "PATCH", "DELETE", 'COPY']
},
"url": {"type": "string"},
"json-schema": jsonschema._utils.load_schema("draft4"),
"resources": {
"type": "array",
"items": {
"oneOf": [
{"type": "string"},
{
"type": "object",
"properties": {
"name": {"type": "string"},
"expected_result": {"type": "integer"}
}
}
]
}
},
"results": {
"type": "object",
"properties": {}
}
},
"required": ["name", "http-method", "url"],
"additionalProperties": False,
}
def __new__(cls, *args, **kwargs):
if not cls._instance:
cls._instance = super(BasicGeneratorSet, cls).__new__(cls, *args,
**kwargs)
return cls._instance
def __init__(self):
self.types_dict = {}
for m in dir(self):
if callable(getattr(self, m)) and not'__' in m:
method = getattr(self, m)
if hasattr(method, "types"):
for type in method.types:
if type not in self.types_dict:
self.types_dict[type] = []
self.types_dict[type].append(method)
def validate_schema(self, schema):
jsonschema.validate(schema, self.schema)
def generate(self, schema):
"""
Generate an json dictionary based on a schema.
Only one value is mis-generated for each dictionary created.
Any generator must return a list of tuples or a single tuple.
The values of this tuple are:
result[0]: Name of the test
result[1]: json schema for the test
result[2]: expected result of the test (can be None)
"""
LOG.debug("generate_invalid: %s" % schema)
schema_type = schema["type"]
if isinstance(schema_type, list):
if "integer" in schema_type:
schema_type = "integer"
else:
raise Exception("non-integer list types not supported")
result = []
if schema_type not in self.types_dict:
raise Exception("generator (%s) doesn't support type: %s"
% (self.__class__.__name__, schema_type))
for generator in self.types_dict[schema_type]:
ret = generator(schema)
if ret is not None:
if isinstance(ret, list):
result.extend(ret)
elif isinstance(ret, tuple):
result.append(ret)
else:
raise Exception("generator (%s) returns invalid result: %s"
% (generator, ret))
LOG.debug("result: %s" % result)
return result

111
tempest/common/generator/negative_generator.py Normal file
View File

@ -0,0 +1,111 @@
# Copyright 2014 Deutsche Telekom AG
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import tempest.common.generator.base_generator as base
import tempest.common.generator.valid_generator as valid
from tempest.openstack.common import log as logging
LOG = logging.getLogger(__name__)
class NegativeTestGenerator(base.BasicGeneratorSet):
@base.generator_type("string")
@base.simple_generator
def gen_int(self, _):
return 4
@base.generator_type("integer")
@base.simple_generator
def gen_string(self, _):
return "XXXXXX"
@base.generator_type("integer", "string")
def gen_none(self, schema):
# Note(mkoderer): it's not using the decorator otherwise it'd be
# filtered
expected_result = base._check_for_expected_result('gen_none', schema)
return ('gen_none', None, expected_result)
@base.generator_type("string")
@base.simple_generator
def gen_str_min_length(self, schema):
min_length = schema.get("minLength", 0)
if min_length > 0:
return "x" * (min_length - 1)
@base.generator_type("string")
@base.simple_generator
def gen_str_max_length(self, schema):
max_length = schema.get("maxLength", -1)
if max_length > -1:
return "x" * (max_length + 1)
@base.generator_type("integer")
@base.simple_generator
def gen_int_min(self, schema):
if "minimum" in schema:
minimum = schema["minimum"]
if "exclusiveMinimum" not in schema:
minimum -= 1
return minimum
@base.generator_type("integer")
@base.simple_generator
def gen_int_max(self, schema):
if "maximum" in schema:
maximum = schema["maximum"]
if "exclusiveMaximum" not in schema:
maximum += 1
return maximum
@base.generator_type("object")
def gen_obj_remove_attr(self, schema):
invalids = []
valid_schema = valid.ValidTestGenerator().generate_valid(schema)
required = schema.get("required", [])
for r in required:
new_valid = copy.deepcopy(valid_schema)
del new_valid[r]
invalids.append(("gen_obj_remove_attr", new_valid, None))
return invalids
@base.generator_type("object")
@base.simple_generator
def gen_obj_add_attr(self, schema):
valid_schema = valid.ValidTestGenerator().generate_valid(schema)
if not schema.get("additionalProperties", True):
new_valid = copy.deepcopy(valid_schema)
new_valid["$$$$$$$$$$"] = "xxx"
return new_valid
@base.generator_type("object")
def gen_inv_prop_obj(self, schema):
LOG.debug("generate_invalid_object: %s" % schema)
valid_schema = valid.ValidTestGenerator().generate_valid(schema)
invalids = []
properties = schema["properties"]
for k, v in properties.iteritems():
for invalid in self.generate(v):
LOG.debug(v)
new_valid = copy.deepcopy(valid_schema)
new_valid[k] = invalid[1]
name = "prop_%s_%s" % (k, invalid[0])
invalids.append((name, new_valid, invalid[2]))
LOG.debug("generate_invalid_object return: %s" % invalids)
return invalids

58
tempest/common/generator/valid_generator.py Normal file
View File

@ -0,0 +1,58 @@
# Copyright 2014 Deutsche Telekom AG
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import tempest.common.generator.base_generator as base
from tempest.openstack.common import log as logging
LOG = logging.getLogger(__name__)
class ValidTestGenerator(base.BasicGeneratorSet):
@base.generator_type("string")
@base.simple_generator
def generate_valid_string(self, schema):
size = schema.get("minLength", 0)
# TODO(dkr mko): handle format and pattern
return "x" * size
@base.generator_type("integer")
@base.simple_generator
def generate_valid_integer(self, schema):
# TODO(dkr mko): handle multipleOf
if "minimum" in schema:
minimum = schema["minimum"]
if "exclusiveMinimum" not in schema:
return minimum
else:
return minimum + 1
if "maximum" in schema:
maximum = schema["maximum"]
if "exclusiveMaximum" not in schema:
return maximum
else:
return maximum - 1
return 0
@base.generator_type("object")
@base.simple_generator
def generate_valid_object(self, schema):
obj = {}
for k, v in schema["properties"].iteritems():
obj[k] = self.generate_valid(v)
return obj
def generate_valid(self, schema):
return self.generate(schema)[0][1]

11
tempest/config.py
View File

@ -806,6 +806,15 @@ CLIGroup = [
help="Number of seconds to wait on a CLI timeout"),
]
negative_group = cfg.OptGroup(name='negative', title="Negative Test Options")
NegativeGroup = [
cfg.StrOpt('test_generator',
default='tempest.common.' +
'generator.negative_generator.NegativeTestGenerator',
help="Test generator class for all negative tests"),
]
def register_opts():
register_opt_group(cfg.CONF, compute_group, ComputeGroup)
@ -840,6 +849,7 @@ def register_opts():
register_opt_group(cfg.CONF, baremetal_group, BaremetalGroup)
register_opt_group(cfg.CONF, input_scenario_group, InputScenarioGroup)
register_opt_group(cfg.CONF, cli_group, CLIGroup)
register_opt_group(cfg.CONF, negative_group, NegativeGroup)
# this should never be called outside of this class
@ -879,6 +889,7 @@ class TempestConfigPrivate(object):
self.baremetal = cfg.CONF.baremetal
self.input_scenario = cfg.CONF['input-scenario']
self.cli = cfg.CONF.cli
self.negative = cfg.CONF.negative
if not self.compute_admin.username:
self.compute_admin.username = self.identity.admin_username
self.compute_admin.password = self.identity.admin_password

15
tempest/test.py
View File

@ -27,10 +27,11 @@ import testresources
import testtools
from tempest import clients
from tempest.common import generate_json
import tempest.common.generator.valid_generator as valid
from tempest.common import isolated_creds
from tempest import config
from tempest import exceptions
from tempest.openstack.common import importutils
from tempest.openstack.common import log as logging
LOG = logging.getLogger(__name__)
@ -400,7 +401,8 @@ class NegativeAutoTest(BaseTestCase):
"""
description = NegativeAutoTest.load_schema(description_file)
LOG.debug(description)
generate_json.validate_negative_test_schema(description)
generator = importutils.import_class(CONF.negative.test_generator)()
generator.validate_schema(description)
schema = description.get("json-schema", None)
resources = description.get("resources", [])
scenario_list = []
@ -416,7 +418,7 @@ class NegativeAutoTest(BaseTestCase):
"expected_result": expected_result
}))
if schema is not None:
for invalid in generate_json.generate_invalid(schema):
for invalid in generator.generate(schema):
scenario_list.append((invalid[0],
{"schema": invalid[1],
"expected_result": invalid[2]}))
@ -459,11 +461,12 @@ class NegativeAutoTest(BaseTestCase):
# Note(mkoderer): The resources list already contains an invalid
# entry (see get_resource).
# We just send a valid json-schema with it
valid = None
valid_schema = None
schema = description.get("json-schema", None)
if schema:
valid = generate_json.generate_valid(schema)
new_url, body = self._http_arguments(valid, url, method)
valid_schema = \
valid.ValidTestGenerator().generate_valid(schema)
new_url, body = self._http_arguments(valid_schema, url, method)
elif hasattr(self, "schema"):
new_url, body = self._http_arguments(self.schema, url, method)

6
tempest/tests/fake_config.py
View File

@ -43,6 +43,10 @@ class FakeConfig(object):
swift = True
horizon = True
class fake_negative(object):
test_generator = 'tempest.common.' \
'generator.negative_generator.NegativeTestGenerator'
compute_feature_enabled = fake_compute_feature_enabled()
volume_feature_enabled = fake_default_feature_enabled()
network_feature_enabled = fake_default_feature_enabled()
@ -52,3 +56,5 @@ class FakeConfig(object):
compute = fake_compute()
identity = fake_identity()
negative = fake_negative()

14
tempest/tests/negative/test_generate_json.py
View File

@ -13,11 +13,11 @@
# License for the specific language governing permissions and limitations
# under the License.
from tempest.common import generate_json as gen
from tempest.common.generator import negative_generator
import tempest.test
class TestGenerateJson(tempest.test.BaseTestCase):
class TestNegativeGenerator(tempest.test.BaseTestCase):
fake_input_str = {"type": "string",
"minLength": 2,
@ -35,19 +35,23 @@ class TestGenerateJson(tempest.test.BaseTestCase):
}
}
def setUp(self):
super(TestNegativeGenerator, self).setUp()
self.negative = negative_generator.NegativeTestGenerator()
def _validate_result(self, data):
self.assertTrue(isinstance(data, list))
for t in data:
self.assertTrue(isinstance(t, tuple))
def test_generate_invalid_string(self):
result = gen.generate_invalid(self.fake_input_str)
result = self.negative.generate(self.fake_input_str)
self._validate_result(result)
def test_generate_invalid_integer(self):
result = gen.generate_invalid(self.fake_input_int)
result = self.negative.generate(self.fake_input_int)
self._validate_result(result)
def test_generate_invalid_obj(self):
result = gen.generate_invalid(self.fake_input_obj)
result = self.negative.generate(self.fake_input_obj)
self._validate_result(result)

8
tempest/tests/negative/test_negative_auto_test.py
View File

@ -16,9 +16,11 @@
import mock
import tempest.test as test
from tempest.tests import base
from tempest.tests import fake_config
class TestNegativeAutoTest(test.BaseTestCase):
class TestNegativeAutoTest(base.TestCase):
# Fake entries
_interface = 'json'
_service = 'compute'
@ -34,6 +36,10 @@ class TestNegativeAutoTest(test.BaseTestCase):
"resources": ["flavor", "volume", "image"]
}
def setUp(self):
super(TestNegativeAutoTest, self).setUp()
self.stubs.Set(test, 'CONF', fake_config.FakeConfig)
def _check_prop_entries(self, result, entry):
entries = [a for a in result if entry in a[0]]
self.assertIsNotNone(entries)