openstack/rally
Code Issues Proposed changes

Merge "[samples] Speed up checking samples"

Browse Source
This commit is contained in:
Jenkins 2017-09-05 15:25:09 +00:00 committed by Gerrit Code Review
commit bda193785f
11 changed files with 126 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
samples/tasks/scenarios/glance/create-and-deactivate-image.json
View File

@ -16,6 +16,11 @@
"tenants": 1, "tenants": 1,
"users_per_tenant": 1 "users_per_tenant": 1
} }
},
"sla": {
"failure_rate": {
"max": 0
}
} }
} }
] ]

3
samples/tasks/scenarios/glance/create-and-deactivate-image.yaml
View File

@ -13,3 +13,6 @@
users: users:
tenants: 1 tenants: 1
users_per_tenant: 1 users_per_tenant: 1
sla:
failure_rate:
max: 0

5
samples/tasks/scenarios/sahara/jobs/dfsio-job-sequence-scaling.json
View File

@ -55,6 +55,11 @@
"auto_security_group": true "auto_security_group": true
}, },
"network": {} "network": {}
},
"sla": {
"failure_rate": {
"max": 0
}
} }
} }
] ]

3
samples/tasks/scenarios/sahara/jobs/dfsio-job-sequence-scaling.yaml
View File

@ -54,3 +54,6 @@
hadoop_version: "2.6.0" hadoop_version: "2.6.0"
auto_security_group: True auto_security_group: True
network: {} network: {}
sla:
failure_rate:
max: 0

5
samples/tasks/scenarios/sahara/jobs/dfsio-job-sequence.json
View File

@ -55,6 +55,11 @@
"auto_security_group": true "auto_security_group": true
}, },
"network": {} "network": {}
},
"sla": {
"failure_rate": {
"max": 0
}
} }
} }
] ]

3
samples/tasks/scenarios/sahara/jobs/dfsio-job-sequence.yaml
View File

@ -51,3 +51,6 @@
hadoop_version: "2.6.0" hadoop_version: "2.6.0"
auto_security_group: True auto_security_group: True
network: {} network: {}
sla:
failure_rate:
max: 0

5
samples/tasks/scenarios/sahara/jobs/java-action-job.json
View File

@ -43,6 +43,11 @@
"auto_security_group": true "auto_security_group": true
}, },
"network": {} "network": {}
},
"sla": {
"failure_rate": {
"max": 0
}
} }
} }
] ]

3
samples/tasks/scenarios/sahara/jobs/java-action-job.yaml
View File

@ -35,3 +35,6 @@
hadoop_version: "2.6.0" hadoop_version: "2.6.0"
auto_security_group: True auto_security_group: True
network: {} network: {}
sla:
failure_rate:
max: 0

5
samples/tasks/scenarios/sahara/jobs/pig-script-job.json
View File

@ -52,6 +52,11 @@
"auto_security_group": true "auto_security_group": true
}, },
"network": {} "network": {}
},
"sla": {
"failure_rate": {
"max": 0
}
} }
} }
] ]

3
samples/tasks/scenarios/sahara/jobs/pig-script-job.yaml
View File

@ -40,3 +40,6 @@
hadoop_version: "2.6.0" hadoop_version: "2.6.0"
auto_security_group: True auto_security_group: True
network: {} network: {}
sla:
failure_rate:
max: 0

185
tests/unit/doc/test_task_samples.py
View File

@ -14,9 +14,8 @@
# under the License. # under the License.
import inspect import inspect
import json import itertools
import os import os
import re
import traceback import traceback
import mock import mock
@ -29,7 +28,7 @@ from rally.task import engine
from rally.task import scenario from rally.task import scenario
from tests.unit import test from tests.unit import test
RALLY_PATH = os.path.join(os.path.dirname(rally.__file__), os.pardir) RALLY_PATH = os.path.dirname(os.path.dirname(rally.__file__))
class TaskSampleTestCase(test.TestCase): class TaskSampleTestCase(test.TestCase):
@ -42,53 +41,58 @@ class TaskSampleTestCase(test.TestCase):
with mock.patch("rally.api.API.check_db_revision"): with mock.patch("rally.api.API.check_db_revision"):
self.rapi = api.API() self.rapi = api.API()
def _check_missing_sla_section(self, raw, filename): def iterate_samples(self, merge_pairs=True):
re_sla_section = re.compile(r"\"?failure_rate\"?:") """Iterates all task samples
matches = re_sla_section.findall(raw)
self.assertTrue(
len(matches),
"sla section is required in %s." % filename)
def test_check_missing_sla_section(self): :param merge_pairs: Whether or not to return both json and yaml samples
of one sample.
"""
for dirname, dirnames, filenames in os.walk(self.samples_path): for dirname, dirnames, filenames in os.walk(self.samples_path):
for filename in filenames: for filename in filenames:
full_path = os.path.join(dirname, filename) # NOTE(hughsaunders): Skip non config files
# (bug https://bugs.launchpad.net/rally/+bug/1314369)
if filename.endswith("json") or (
not merge_pairs and filename.endswith("yaml")):
yield os.path.join(dirname, filename)
if not full_path.startswith("./samples/tasks/scenarios/"): def test_check_missing_sla_section(self):
continue failures = []
for path in self.iterate_samples():
if not re.search("\.(ya?ml|json)$", filename, flags=re.I): if "tasks/scenarios" not in path:
continue continue
with open(path) as task_file:
with open(full_path) as task_file: task_config = yaml.safe_load(
data = task_file.read() self.rapi.task.render_template(
self._check_missing_sla_section(data, task_file) task_template=task_file.read()))
for workload in itertools.chain(*task_config.values()):
if not workload.get("sla", {}):
failures.append(path)
if failures:
self.fail("One or several workloads from the list of samples below"
" doesn't have SLA section: \n %s" %
"\n ".join(failures))
def test_schema_is_valid(self): def test_schema_is_valid(self):
scenarios = set() scenarios = set()
for dirname, dirnames, filenames in os.walk(self.samples_path): for path in self.iterate_samples():
for filename in filenames: with open(path) as task_file:
full_path = os.path.join(dirname, filename) try:
# NOTE(hughsaunders): Skip non config files
# (bug https://bugs.launchpad.net/rally/+bug/1314369)
if not re.search("\.(ya?ml|json)$", filename, flags=re.I):
continue
with open(full_path) as task_file:
try: try:
task_config = yaml.safe_load( task_config = yaml.safe_load(
self.rapi.task.render_template( self.rapi.task.render_template(
task_template=task_file.read())) task_template=task_file.read()))
eng = engine.TaskEngine(task_config,
mock.MagicMock(), mock.Mock())
eng.validate(only_syntax=True)
except Exception: except Exception:
print(traceback.format_exc()) print(traceback.format_exc())
self.fail("Invalid task file: %s" % full_path) self.fail("Invalid JSON file: %s" % path)
else: eng = engine.TaskEngine(task_config,
scenarios.update(task_config.keys()) mock.MagicMock(), mock.Mock())
eng.validate(only_syntax=True)
except Exception:
print(traceback.format_exc())
self.fail("Invalid task file: %s" % path)
else:
scenarios.update(task_config.keys())
missing = set(s.get_name() for s in scenario.Scenario.get_all()) missing = set(s.get_name() for s in scenario.Scenario.get_all())
missing -= scenarios missing -= scenarios
@ -98,80 +102,63 @@ class TaskSampleTestCase(test.TestCase):
self.assertEqual(missing, [], self.assertEqual(missing, [],
"These scenarios don't have samples: %s" % missing) "These scenarios don't have samples: %s" % missing)
def test_json_correct_syntax(self): def test_task_config_pairs(self):
for dirname, dirnames, filenames in os.walk(self.samples_path):
for filename in filenames:
if not filename.endswith(".json"):
continue
full_path = os.path.join(dirname, filename)
with open(full_path) as task_file:
try:
json.loads(self.rapi.task.render_template(
task_template=task_file.read()))
except Exception:
print(traceback.format_exc())
self.fail("Invalid JSON file: %s" % full_path)
def test_task_config_pair_existance(self): not_equal = []
inexistent_paths = [] missed = []
checked = []
for dirname, dirnames, filenames in os.walk(self.samples_path): for path in self.iterate_samples(merge_pairs=False):
# iterate over unique config names if path.endswith(".json"):
for sample_name in set( json_path = path
f[:-5] for f in filenames yaml_path = json_path.replace(".json", ".yaml")
if f.endswith(".json") or f.endswith(".yaml")): else:
yaml_path = path
json_path = yaml_path.replace(".yaml", ".json")
partial_path = os.path.join(dirname, sample_name) if json_path in checked:
yaml_path = partial_path + ".yaml" continue
json_path = partial_path + ".json" else:
checked.append(json_path)
if not os.path.exists(yaml_path): if not os.path.exists(yaml_path):
inexistent_paths.append(yaml_path) missed.append(yaml_path)
elif not os.path.exists(json_path): elif not os.path.exists(json_path):
inexistent_paths.append(json_path) missed.append(json_path)
else:
with open(json_path) as json_file:
json_config = yaml.safe_load(
self.rapi.task.render_template(
task_template=json_file.read()))
with open(yaml_path) as yaml_file:
yaml_config = yaml.safe_load(
self.rapi.task.render_template(
task_template=yaml_file.read()))
if json_config != yaml_config:
not_equal.append("'%s' and '%s'" % (yaml_path, json_path))
if inexistent_paths: error = ""
self.fail("Sample task configs are missing:\n%r" if not_equal:
% inexistent_paths) error += ("Sample task configs are not equal:\n\t%s\n"
% "\n\t".join(not_equal))
if missed:
self.fail("Sample task configs are missing:\n\t%s\n"
% "\n\t".join(missed))
def test_task_config_pairs_equality(self): if error:
for dirname, dirnames, filenames in os.walk(self.samples_path): self.fail(error)
# iterate over unique config names
for sample_name in set(
f[:-5] for f in filenames
if f.endswith(".json") or f.endswith(".yaml")):
partial_path = os.path.join(dirname, sample_name)
yaml_path = partial_path + ".yaml"
json_path = partial_path + ".json"
if os.path.exists(yaml_path) and os.path.exists(json_path):
with open(json_path) as json_file:
json_config = yaml.safe_load(
self.rapi.task.render_template(
task_template=json_file.read()))
with open(yaml_path) as yaml_file:
yaml_config = yaml.safe_load(
self.rapi.task.render_template(
task_template=yaml_file.read()))
self.assertEqual(json_config, yaml_config,
"Sample task configs are not equal:"
"\n%s\n%s" % (yaml_path, json_path))
def test_no_underscores_in_filename(self): def test_no_underscores_in_filename(self):
bad_filenames = [] bad_filenames = []
for dirname, dirnames, filenames in os.walk(self.samples_path): for path in self.iterate_samples(merge_pairs=False):
for filename in filenames: if "_" in path:
if "_" in filename and (filename.endswith(".yaml") or bad_filenames.append(path)
filename.endswith(".json")):
full_path = os.path.join(dirname, filename)
bad_filenames.append(full_path)
self.assertEqual([], bad_filenames, if bad_filenames:
"Following sample task filenames contain " self.fail("Following sample task filename(s) contain underscores "
"underscores (_) but must use dashes (-) instead: " "(_) but must use dashes (-) instead: %s" %
"{}".format(bad_filenames)) bad_filenames)
def test_context_samples_found(self): def test_context_samples_found(self):
all_plugins = context.Context.get_all() all_plugins = context.Context.get_all()