Add multiple negative test generator support
In order to support different generator styles and sets of tests a new configuration parameter is introduced to define the negative test generator class. This can be used to define generators that create only random values (random fuzzy test) or pattern based values. With this functionality it is also possible to reduce the amount of negative tests that are automatically produced. Change-Id: Icfad55d1eea92dc2a42642b37d34c253c26c0846 Partially-implements: bp fuzzy-test
This commit is contained in:
Marc Koderer
parent
ff956a6669
commit
6ee82dccd1
|
|
@ -548,6 +548,16 @@
|
|||
#ssh_user_regex=[["^.*[Cc]irros.*$", "root"]]
|
||||
|
||||
|
||||
[negative]
|
||||
|
||||
#
|
||||
# Options defined in tempest.config
|
||||
#
|
||||
|
||||
# Test generator class for all negative tests (string value)
|
||||
#test_generator=tempest.common.generator.negative_generator.NegativeTestGenerator
|
||||
|
||||
|
||||
[network]
|
||||
|
||||
#
|
||||
|
|
|
|||
|
|
@ -1,265 +0,0 @@
|
|||
# Copyright 2014 Red Hat, Inc. & Deutsche Telekom AG
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import copy
|
||||
import jsonschema
|
||||
|
||||
from tempest.openstack.common import log as logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def generate_valid(schema):
|
||||
"""
|
||||
Create a valid dictionary based on the types in a json schema.
|
||||
"""
|
||||
LOG.debug("generate_valid: %s" % schema)
|
||||
schema_type = schema["type"]
|
||||
if isinstance(schema_type, list):
|
||||
# Just choose the first one since all are valid.
|
||||
schema_type = schema_type[0]
|
||||
return type_map_valid[schema_type](schema)
|
||||
|
||||
|
||||
def generate_valid_string(schema):
|
||||
size = schema.get("minLength", 0)
|
||||
# TODO(dkr mko): handle format and pattern
|
||||
return "x" * size
|
||||
|
||||
|
||||
def generate_valid_integer(schema):
|
||||
# TODO(dkr mko): handle multipleOf
|
||||
if "minimum" in schema:
|
||||
minimum = schema["minimum"]
|
||||
if "exclusiveMinimum" not in schema:
|
||||
return minimum
|
||||
else:
|
||||
return minimum + 1
|
||||
if "maximum" in schema:
|
||||
maximum = schema["maximum"]
|
||||
if "exclusiveMaximum" not in schema:
|
||||
return maximum
|
||||
else:
|
||||
return maximum - 1
|
||||
return 0
|
||||
|
||||
|
||||
def generate_valid_object(schema):
|
||||
obj = {}
|
||||
for k, v in schema["properties"].iteritems():
|
||||
obj[k] = generate_valid(v)
|
||||
return obj
|
||||
|
||||
|
||||
def generate_invalid(schema):
|
||||
"""
|
||||
Generate an invalid json dictionary based on a schema.
|
||||
Only one value is mis-generated for each dictionary created.
|
||||
|
||||
Any generator must return a list of tuples or a single tuple.
|
||||
The values of this tuple are:
|
||||
result[0]: Name of the test
|
||||
result[1]: json schema for the test
|
||||
result[2]: expected result of the test (can be None)
|
||||
"""
|
||||
LOG.debug("generate_invalid: %s" % schema)
|
||||
schema_type = schema["type"]
|
||||
if isinstance(schema_type, list):
|
||||
if "integer" in schema_type:
|
||||
schema_type = "integer"
|
||||
else:
|
||||
raise Exception("non-integer list types not supported")
|
||||
result = []
|
||||
for generator in type_map_invalid[schema_type]:
|
||||
ret = generator(schema)
|
||||
if ret is not None:
|
||||
if isinstance(ret, list):
|
||||
result.extend(ret)
|
||||
elif isinstance(ret, tuple):
|
||||
result.append(ret)
|
||||
else:
|
||||
raise Exception("generator (%s) returns invalid result"
|
||||
% generator)
|
||||
LOG.debug("result: %s" % result)
|
||||
return result
|
||||
|
||||
|
||||
def _check_for_expected_result(name, schema):
|
||||
expected_result = None
|
||||
if "results" in schema:
|
||||
if name in schema["results"]:
|
||||
expected_result = schema["results"][name]
|
||||
return expected_result
|
||||
|
||||
|
||||
def generator(fn):
|
||||
"""
|
||||
Decorator for simple generators that simply return one value
|
||||
"""
|
||||
def wrapped(schema):
|
||||
result = fn(schema)
|
||||
if result is not None:
|
||||
expected_result = _check_for_expected_result(fn.__name__, schema)
|
||||
return (fn.__name__, result, expected_result)
|
||||
return
|
||||
return wrapped
|
||||
|
||||
|
||||
@generator
|
||||
def gen_int(_):
|
||||
return 4
|
||||
|
||||
|
||||
@generator
|
||||
def gen_string(_):
|
||||
return "XXXXXX"
|
||||
|
||||
|
||||
def gen_none(schema):
|
||||
# Note(mkoderer): it's not using the decorator otherwise it'd be filtered
|
||||
expected_result = _check_for_expected_result('gen_none', schema)
|
||||
return ('gen_none', None, expected_result)
|
||||
|
||||
|
||||
@generator
|
||||
def gen_str_min_length(schema):
|
||||
min_length = schema.get("minLength", 0)
|
||||
if min_length > 0:
|
||||
return "x" * (min_length - 1)
|
||||
|
||||
|
||||
@generator
|
||||
def gen_str_max_length(schema):
|
||||
max_length = schema.get("maxLength", -1)
|
||||
if max_length > -1:
|
||||
return "x" * (max_length + 1)
|
||||
|
||||
|
||||
@generator
|
||||
def gen_int_min(schema):
|
||||
if "minimum" in schema:
|
||||
minimum = schema["minimum"]
|
||||
if "exclusiveMinimum" not in schema:
|
||||
minimum -= 1
|
||||
return minimum
|
||||
|
||||
|
||||
@generator
|
||||
def gen_int_max(schema):
|
||||
if "maximum" in schema:
|
||||
maximum = schema["maximum"]
|
||||
if "exclusiveMaximum" not in schema:
|
||||
maximum += 1
|
||||
return maximum
|
||||
|
||||
|
||||
def gen_obj_remove_attr(schema):
|
||||
invalids = []
|
||||
valid = generate_valid(schema)
|
||||
required = schema.get("required", [])
|
||||
for r in required:
|
||||
new_valid = copy.deepcopy(valid)
|
||||
del new_valid[r]
|
||||
invalids.append(("gen_obj_remove_attr", new_valid, None))
|
||||
return invalids
|
||||
|
||||
|
||||
@generator
|
||||
def gen_obj_add_attr(schema):
|
||||
valid = generate_valid(schema)
|
||||
if not schema.get("additionalProperties", True):
|
||||
new_valid = copy.deepcopy(valid)
|
||||
new_valid["$$$$$$$$$$"] = "xxx"
|
||||
return new_valid
|
||||
|
||||
|
||||
def gen_inv_prop_obj(schema):
|
||||
LOG.debug("generate_invalid_object: %s" % schema)
|
||||
valid = generate_valid(schema)
|
||||
invalids = []
|
||||
properties = schema["properties"]
|
||||
|
||||
for k, v in properties.iteritems():
|
||||
for invalid in generate_invalid(v):
|
||||
LOG.debug(v)
|
||||
new_valid = copy.deepcopy(valid)
|
||||
new_valid[k] = invalid[1]
|
||||
name = "prop_%s_%s" % (k, invalid[0])
|
||||
invalids.append((name, new_valid, invalid[2]))
|
||||
|
||||
LOG.debug("generate_invalid_object return: %s" % invalids)
|
||||
return invalids
|
||||
|
||||
|
||||
type_map_valid = {
|
||||
"string": generate_valid_string,
|
||||
"integer": generate_valid_integer,
|
||||
"object": generate_valid_object
|
||||
}
|
||||
|
||||
type_map_invalid = {
|
||||
"string": [
|
||||
gen_int,
|
||||
gen_none,
|
||||
gen_str_min_length,
|
||||
gen_str_max_length],
|
||||
"integer": [
|
||||
gen_string,
|
||||
gen_none,
|
||||
gen_int_min,
|
||||
gen_int_max],
|
||||
"object": [
|
||||
gen_obj_remove_attr,
|
||||
gen_obj_add_attr,
|
||||
gen_inv_prop_obj]
|
||||
}
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"http-method": {
|
||||
"enum": ["GET", "PUT", "HEAD",
|
||||
"POST", "PATCH", "DELETE", 'COPY']
|
||||
},
|
||||
"url": {"type": "string"},
|
||||
"json-schema": jsonschema._utils.load_schema("draft4"),
|
||||
"resources": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"oneOf": [
|
||||
{"type": "string"},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"expected_result": {"type": "integer"}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"results": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
}
|
||||
},
|
||||
"required": ["name", "http-method", "url"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
|
||||
def validate_negative_test_schema(nts):
|
||||
jsonschema.validate(nts, schema)
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
# Copyright 2014 Deutsche Telekom AG
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import jsonschema
|
||||
|
||||
from tempest.openstack.common import log as logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _check_for_expected_result(name, schema):
|
||||
expected_result = None
|
||||
if "results" in schema:
|
||||
if name in schema["results"]:
|
||||
expected_result = schema["results"][name]
|
||||
return expected_result
|
||||
|
||||
|
||||
def generator_type(*args):
|
||||
def wrapper(func):
|
||||
func.types = args
|
||||
return func
|
||||
return wrapper
|
||||
|
||||
|
||||
def simple_generator(fn):
|
||||
"""
|
||||
Decorator for simple generators that return one value
|
||||
"""
|
||||
def wrapped(self, schema):
|
||||
result = fn(self, schema)
|
||||
if result is not None:
|
||||
expected_result = _check_for_expected_result(fn.__name__, schema)
|
||||
return (fn.__name__, result, expected_result)
|
||||
return
|
||||
return wrapped
|
||||
|
||||
|
||||
class BasicGeneratorSet(object):
|
||||
_instance = None
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"http-method": {
|
||||
"enum": ["GET", "PUT", "HEAD",
|
||||
"POST", "PATCH", "DELETE", 'COPY']
|
||||
},
|
||||
"url": {"type": "string"},
|
||||
"json-schema": jsonschema._utils.load_schema("draft4"),
|
||||
"resources": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"oneOf": [
|
||||
{"type": "string"},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"expected_result": {"type": "integer"}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"results": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
}
|
||||
},
|
||||
"required": ["name", "http-method", "url"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if not cls._instance:
|
||||
cls._instance = super(BasicGeneratorSet, cls).__new__(cls, *args,
|
||||
**kwargs)
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
self.types_dict = {}
|
||||
for m in dir(self):
|
||||
if callable(getattr(self, m)) and not'__' in m:
|
||||
method = getattr(self, m)
|
||||
if hasattr(method, "types"):
|
||||
for type in method.types:
|
||||
if type not in self.types_dict:
|
||||
self.types_dict[type] = []
|
||||
self.types_dict[type].append(method)
|
||||
|
||||
def validate_schema(self, schema):
|
||||
jsonschema.validate(schema, self.schema)
|
||||
|
||||
def generate(self, schema):
|
||||
"""
|
||||
Generate an json dictionary based on a schema.
|
||||
Only one value is mis-generated for each dictionary created.
|
||||
|
||||
Any generator must return a list of tuples or a single tuple.
|
||||
The values of this tuple are:
|
||||
result[0]: Name of the test
|
||||
result[1]: json schema for the test
|
||||
result[2]: expected result of the test (can be None)
|
||||
"""
|
||||
LOG.debug("generate_invalid: %s" % schema)
|
||||
schema_type = schema["type"]
|
||||
if isinstance(schema_type, list):
|
||||
if "integer" in schema_type:
|
||||
schema_type = "integer"
|
||||
else:
|
||||
raise Exception("non-integer list types not supported")
|
||||
result = []
|
||||
if schema_type not in self.types_dict:
|
||||
raise Exception("generator (%s) doesn't support type: %s"
|
||||
% (self.__class__.__name__, schema_type))
|
||||
for generator in self.types_dict[schema_type]:
|
||||
ret = generator(schema)
|
||||
if ret is not None:
|
||||
if isinstance(ret, list):
|
||||
result.extend(ret)
|
||||
elif isinstance(ret, tuple):
|
||||
result.append(ret)
|
||||
else:
|
||||
raise Exception("generator (%s) returns invalid result: %s"
|
||||
% (generator, ret))
|
||||
LOG.debug("result: %s" % result)
|
||||
return result
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
# Copyright 2014 Deutsche Telekom AG
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import copy
|
||||
|
||||
import tempest.common.generator.base_generator as base
|
||||
import tempest.common.generator.valid_generator as valid
|
||||
from tempest.openstack.common import log as logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NegativeTestGenerator(base.BasicGeneratorSet):
|
||||
@base.generator_type("string")
|
||||
@base.simple_generator
|
||||
def gen_int(self, _):
|
||||
return 4
|
||||
|
||||
@base.generator_type("integer")
|
||||
@base.simple_generator
|
||||
def gen_string(self, _):
|
||||
return "XXXXXX"
|
||||
|
||||
@base.generator_type("integer", "string")
|
||||
def gen_none(self, schema):
|
||||
# Note(mkoderer): it's not using the decorator otherwise it'd be
|
||||
# filtered
|
||||
expected_result = base._check_for_expected_result('gen_none', schema)
|
||||
return ('gen_none', None, expected_result)
|
||||
|
||||
@base.generator_type("string")
|
||||
@base.simple_generator
|
||||
def gen_str_min_length(self, schema):
|
||||
min_length = schema.get("minLength", 0)
|
||||
if min_length > 0:
|
||||
return "x" * (min_length - 1)
|
||||
|
||||
@base.generator_type("string")
|
||||
@base.simple_generator
|
||||
def gen_str_max_length(self, schema):
|
||||
max_length = schema.get("maxLength", -1)
|
||||
if max_length > -1:
|
||||
return "x" * (max_length + 1)
|
||||
|
||||
@base.generator_type("integer")
|
||||
@base.simple_generator
|
||||
def gen_int_min(self, schema):
|
||||
if "minimum" in schema:
|
||||
minimum = schema["minimum"]
|
||||
if "exclusiveMinimum" not in schema:
|
||||
minimum -= 1
|
||||
return minimum
|
||||
|
||||
@base.generator_type("integer")
|
||||
@base.simple_generator
|
||||
def gen_int_max(self, schema):
|
||||
if "maximum" in schema:
|
||||
maximum = schema["maximum"]
|
||||
if "exclusiveMaximum" not in schema:
|
||||
maximum += 1
|
||||
return maximum
|
||||
|
||||
@base.generator_type("object")
|
||||
def gen_obj_remove_attr(self, schema):
|
||||
invalids = []
|
||||
valid_schema = valid.ValidTestGenerator().generate_valid(schema)
|
||||
required = schema.get("required", [])
|
||||
for r in required:
|
||||
new_valid = copy.deepcopy(valid_schema)
|
||||
del new_valid[r]
|
||||
invalids.append(("gen_obj_remove_attr", new_valid, None))
|
||||
return invalids
|
||||
|
||||
@base.generator_type("object")
|
||||
@base.simple_generator
|
||||
def gen_obj_add_attr(self, schema):
|
||||
valid_schema = valid.ValidTestGenerator().generate_valid(schema)
|
||||
if not schema.get("additionalProperties", True):
|
||||
new_valid = copy.deepcopy(valid_schema)
|
||||
new_valid["$$$$$$$$$$"] = "xxx"
|
||||
return new_valid
|
||||
|
||||
@base.generator_type("object")
|
||||
def gen_inv_prop_obj(self, schema):
|
||||
LOG.debug("generate_invalid_object: %s" % schema)
|
||||
valid_schema = valid.ValidTestGenerator().generate_valid(schema)
|
||||
invalids = []
|
||||
properties = schema["properties"]
|
||||
|
||||
for k, v in properties.iteritems():
|
||||
for invalid in self.generate(v):
|
||||
LOG.debug(v)
|
||||
new_valid = copy.deepcopy(valid_schema)
|
||||
new_valid[k] = invalid[1]
|
||||
name = "prop_%s_%s" % (k, invalid[0])
|
||||
invalids.append((name, new_valid, invalid[2]))
|
||||
|
||||
LOG.debug("generate_invalid_object return: %s" % invalids)
|
||||
return invalids
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
# Copyright 2014 Deutsche Telekom AG
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import tempest.common.generator.base_generator as base
|
||||
from tempest.openstack.common import log as logging
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ValidTestGenerator(base.BasicGeneratorSet):
|
||||
@base.generator_type("string")
|
||||
@base.simple_generator
|
||||
def generate_valid_string(self, schema):
|
||||
size = schema.get("minLength", 0)
|
||||
# TODO(dkr mko): handle format and pattern
|
||||
return "x" * size
|
||||
|
||||
@base.generator_type("integer")
|
||||
@base.simple_generator
|
||||
def generate_valid_integer(self, schema):
|
||||
# TODO(dkr mko): handle multipleOf
|
||||
if "minimum" in schema:
|
||||
minimum = schema["minimum"]
|
||||
if "exclusiveMinimum" not in schema:
|
||||
return minimum
|
||||
else:
|
||||
return minimum + 1
|
||||
if "maximum" in schema:
|
||||
maximum = schema["maximum"]
|
||||
if "exclusiveMaximum" not in schema:
|
||||
return maximum
|
||||
else:
|
||||
return maximum - 1
|
||||
return 0
|
||||
|
||||
@base.generator_type("object")
|
||||
@base.simple_generator
|
||||
def generate_valid_object(self, schema):
|
||||
obj = {}
|
||||
for k, v in schema["properties"].iteritems():
|
||||
obj[k] = self.generate_valid(v)
|
||||
return obj
|
||||
|
||||
def generate_valid(self, schema):
|
||||
return self.generate(schema)[0][1]
|
||||
|
|
@ -806,6 +806,15 @@ CLIGroup = [
|
|||
help="Number of seconds to wait on a CLI timeout"),
|
||||
]
|
||||
|
||||
negative_group = cfg.OptGroup(name='negative', title="Negative Test Options")
|
||||
|
||||
NegativeGroup = [
|
||||
cfg.StrOpt('test_generator',
|
||||
default='tempest.common.' +
|
||||
'generator.negative_generator.NegativeTestGenerator',
|
||||
help="Test generator class for all negative tests"),
|
||||
]
|
||||
|
||||
|
||||
def register_opts():
|
||||
register_opt_group(cfg.CONF, compute_group, ComputeGroup)
|
||||
|
|
@ -840,6 +849,7 @@ def register_opts():
|
|||
register_opt_group(cfg.CONF, baremetal_group, BaremetalGroup)
|
||||
register_opt_group(cfg.CONF, input_scenario_group, InputScenarioGroup)
|
||||
register_opt_group(cfg.CONF, cli_group, CLIGroup)
|
||||
register_opt_group(cfg.CONF, negative_group, NegativeGroup)
|
||||
|
||||
|
||||
# this should never be called outside of this class
|
||||
|
|
@ -879,6 +889,7 @@ class TempestConfigPrivate(object):
|
|||
self.baremetal = cfg.CONF.baremetal
|
||||
self.input_scenario = cfg.CONF['input-scenario']
|
||||
self.cli = cfg.CONF.cli
|
||||
self.negative = cfg.CONF.negative
|
||||
if not self.compute_admin.username:
|
||||
self.compute_admin.username = self.identity.admin_username
|
||||
self.compute_admin.password = self.identity.admin_password
|
||||
|
|
|
|||
|
|
@ -27,10 +27,11 @@ import testresources
|
|||
import testtools
|
||||
|
||||
from tempest import clients
|
||||
from tempest.common import generate_json
|
||||
import tempest.common.generator.valid_generator as valid
|
||||
from tempest.common import isolated_creds
|
||||
from tempest import config
|
||||
from tempest import exceptions
|
||||
from tempest.openstack.common import importutils
|
||||
from tempest.openstack.common import log as logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
|
@ -400,7 +401,8 @@ class NegativeAutoTest(BaseTestCase):
|
|||
"""
|
||||
description = NegativeAutoTest.load_schema(description_file)
|
||||
LOG.debug(description)
|
||||
generate_json.validate_negative_test_schema(description)
|
||||
generator = importutils.import_class(CONF.negative.test_generator)()
|
||||
generator.validate_schema(description)
|
||||
schema = description.get("json-schema", None)
|
||||
resources = description.get("resources", [])
|
||||
scenario_list = []
|
||||
|
|
@ -416,7 +418,7 @@ class NegativeAutoTest(BaseTestCase):
|
|||
"expected_result": expected_result
|
||||
}))
|
||||
if schema is not None:
|
||||
for invalid in generate_json.generate_invalid(schema):
|
||||
for invalid in generator.generate(schema):
|
||||
scenario_list.append((invalid[0],
|
||||
{"schema": invalid[1],
|
||||
"expected_result": invalid[2]}))
|
||||
|
|
@ -459,11 +461,12 @@ class NegativeAutoTest(BaseTestCase):
|
|||
# Note(mkoderer): The resources list already contains an invalid
|
||||
# entry (see get_resource).
|
||||
# We just send a valid json-schema with it
|
||||
valid = None
|
||||
valid_schema = None
|
||||
schema = description.get("json-schema", None)
|
||||
if schema:
|
||||
valid = generate_json.generate_valid(schema)
|
||||
new_url, body = self._http_arguments(valid, url, method)
|
||||
valid_schema = \
|
||||
valid.ValidTestGenerator().generate_valid(schema)
|
||||
new_url, body = self._http_arguments(valid_schema, url, method)
|
||||
elif hasattr(self, "schema"):
|
||||
new_url, body = self._http_arguments(self.schema, url, method)
|
||||
|
||||
|
|
|
|||
|
|
@ -43,6 +43,10 @@ class FakeConfig(object):
|
|||
swift = True
|
||||
horizon = True
|
||||
|
||||
class fake_negative(object):
|
||||
test_generator = 'tempest.common.' \
|
||||
'generator.negative_generator.NegativeTestGenerator'
|
||||
|
||||
compute_feature_enabled = fake_compute_feature_enabled()
|
||||
volume_feature_enabled = fake_default_feature_enabled()
|
||||
network_feature_enabled = fake_default_feature_enabled()
|
||||
|
|
@ -52,3 +56,5 @@ class FakeConfig(object):
|
|||
|
||||
compute = fake_compute()
|
||||
identity = fake_identity()
|
||||
|
||||
negative = fake_negative()
|
||||
|
|
|
|||
|
|
@ -13,11 +13,11 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from tempest.common import generate_json as gen
|
||||
from tempest.common.generator import negative_generator
|
||||
import tempest.test
|
||||
|
||||
|
||||
class TestGenerateJson(tempest.test.BaseTestCase):
|
||||
class TestNegativeGenerator(tempest.test.BaseTestCase):
|
||||
|
||||
fake_input_str = {"type": "string",
|
||||
"minLength": 2,
|
||||
|
|
@ -35,19 +35,23 @@ class TestGenerateJson(tempest.test.BaseTestCase):
|
|||
}
|
||||
}
|
||||
|
||||
def setUp(self):
|
||||
super(TestNegativeGenerator, self).setUp()
|
||||
self.negative = negative_generator.NegativeTestGenerator()
|
||||
|
||||
def _validate_result(self, data):
|
||||
self.assertTrue(isinstance(data, list))
|
||||
for t in data:
|
||||
self.assertTrue(isinstance(t, tuple))
|
||||
|
||||
def test_generate_invalid_string(self):
|
||||
result = gen.generate_invalid(self.fake_input_str)
|
||||
result = self.negative.generate(self.fake_input_str)
|
||||
self._validate_result(result)
|
||||
|
||||
def test_generate_invalid_integer(self):
|
||||
result = gen.generate_invalid(self.fake_input_int)
|
||||
result = self.negative.generate(self.fake_input_int)
|
||||
self._validate_result(result)
|
||||
|
||||
def test_generate_invalid_obj(self):
|
||||
result = gen.generate_invalid(self.fake_input_obj)
|
||||
result = self.negative.generate(self.fake_input_obj)
|
||||
self._validate_result(result)
|
||||
|
|
|
|||
|
|
@ -16,9 +16,11 @@
|
|||
import mock
|
||||
|
||||
import tempest.test as test
|
||||
from tempest.tests import base
|
||||
from tempest.tests import fake_config
|
||||
|
||||
|
||||
class TestNegativeAutoTest(test.BaseTestCase):
|
||||
class TestNegativeAutoTest(base.TestCase):
|
||||
# Fake entries
|
||||
_interface = 'json'
|
||||
_service = 'compute'
|
||||
|
|
@ -34,6 +36,10 @@ class TestNegativeAutoTest(test.BaseTestCase):
|
|||
"resources": ["flavor", "volume", "image"]
|
||||
}
|
||||
|
||||
def setUp(self):
|
||||
super(TestNegativeAutoTest, self).setUp()
|
||||
self.stubs.Set(test, 'CONF', fake_config.FakeConfig)
|
||||
|
||||
def _check_prop_entries(self, result, entry):
|
||||
entries = [a for a in result if entry in a[0]]
|
||||
self.assertIsNotNone(entries)
|
||||
|
|
|
|||
Loading…
Reference in New Issue