diff --git a/rally/api.py b/rally/api.py index 6f561a80c6..4202f75b36 100644 --- a/rally/api.py +++ b/rally/api.py @@ -452,7 +452,7 @@ class _Verifier(object): LOG.info("Creating verifier '%s'.", name) try: - verifier = cls.get(name) + verifier = cls._get(name) except exceptions.ResourceNotFound: verifier = objects.Verifier.create( name=name, source=source, system_wide=system_wide, @@ -489,21 +489,29 @@ class _Verifier(object): return verifier.uuid @staticmethod - def get(verifier_id): + def _get(verifier_id): """Get a verifier. :param verifier_id: Verifier name or UUID """ return objects.Verifier.get(verifier_id) + @classmethod + def get(cls, verifier_id): + return cls._get(verifier_id).to_dict() + @staticmethod - def list(status=None): + def _list(status=None): """List all verifiers. :param status: Status to filter verifiers by """ return objects.Verifier.list(status) + @classmethod + def list(cls, status=None): + return [item.to_dict() for item in cls._list(status)] + @classmethod def delete(cls, verifier_id, deployment_id=None, force=False): """Delete a verifier. @@ -516,7 +524,7 @@ class _Verifier(object): If deployment_id specified, only verifications of this deployment will be deleted """ - verifier = cls.get(verifier_id) + verifier = cls._get(verifier_id) verifications = _Verification.list(verifier_id, deployment_id) if verifications: d_msg = ((" for deployment '%s'" % deployment_id) @@ -525,7 +533,7 @@ class _Verifier(object): LOG.info("Deleting all verifications created by verifier " "%s%s.", verifier, d_msg) for verification in verifications: - _Verification.delete(verification.uuid) + _Verification.delete(verification["uuid"]) else: raise exceptions.RallyException( "Failed to delete verifier {0} because there are stored " @@ -562,7 +570,7 @@ class _Verifier(object): "At least one of the following parameters should be " "specified: 'system_wide', 'version', 'update_venv'.") - verifier = cls.get(verifier_id) + verifier = cls._get(verifier_id) LOG.info("Updating verifier %s.", verifier) if verifier.status != consts.VerifierStatus.INSTALLED: @@ -646,7 +654,7 @@ class _Verifier(object): :param reconfigure: Reconfigure verifier """ if not isinstance(verifier, objects.Verifier): - verifier = cls.get(verifier) + verifier = cls._get(verifier) verifier.set_deployment(deployment_id) LOG.info( "Configuring verifier %s for deployment '%s' (UUID=%s).", @@ -698,7 +706,7 @@ class _Verifier(object): :param deployment_id: Deployment name or UUID :param new_configuration: New configuration for verifier """ - verifier = cls.get(verifier_id) + verifier = cls._get(verifier_id) if verifier.status != consts.VerifierStatus.INSTALLED: raise exceptions.RallyException( "Failed to override verifier configuration for deployment " @@ -724,7 +732,7 @@ class _Verifier(object): :param verifier_id: Verifier name or UUID :param pattern: Pattern which will be used for matching """ - verifier = cls.get(verifier_id) + verifier = cls._get(verifier_id) if verifier.status != consts.VerifierStatus.INSTALLED: raise exceptions.RallyException( "Failed to list verifier tests because verifier %s is in '%s' " @@ -749,7 +757,7 @@ class _Verifier(object): :param extra_settings: Extra installation settings for verifier extension """ - verifier = cls.get(verifier_id) + verifier = cls._get(verifier_id) if verifier.status != consts.VerifierStatus.INSTALLED: raise exceptions.RallyException( "Failed to add verifier extension because verifier %s " @@ -777,7 +785,7 @@ class _Verifier(object): :param verifier_id: Verifier name or UUID """ - verifier = cls.get(verifier_id) + verifier = cls._get(verifier_id) if verifier.status != consts.VerifierStatus.INSTALLED: raise exceptions.RallyException( "Failed to list verifier extensions because verifier %s " @@ -794,7 +802,7 @@ class _Verifier(object): :param verifier_id: Verifier name or UUID :param name: Verifier extension name """ - verifier = cls.get(verifier_id) + verifier = cls._get(verifier_id) if verifier.status != consts.VerifierStatus.INSTALLED: raise exceptions.RallyException( "Failed to delete verifier extension because verifier %s " @@ -832,7 +840,7 @@ class _Verification(object): uuid=deployment["uuid"], status=deployment["status"]) - verifier = _Verifier.get(verifier_id) + verifier = _Verifier._get(verifier_id) if verifier.status != consts.VerifierStatus.INSTALLED: raise exceptions.RallyException( "Failed to start verification because verifier %s is in '%s' " @@ -876,7 +884,9 @@ class _Verification(object): "deployment '%s' (UUID=%s)!", verification.uuid, verifier.deployment["name"], verifier.deployment["uuid"]) - return verification, results + return {"verification": verification.to_dict(), + "totals": results.totals, + "tests": results.tests} @classmethod def rerun(cls, verification_uuid, deployment_id=None, failed=False, @@ -896,7 +906,7 @@ class _Verification(object): if concurrency: run_args["concurrency"] = concurrency - verification = cls.get(verification_uuid) + verification = cls._get(verification_uuid) tests = verification.tests if failed: @@ -913,18 +923,22 @@ class _Verification(object): LOG.info("Re-running %stests from verification (UUID=%s) for " "deployment '%s' (UUID=%s).", "failed " if failed else "", verification.uuid, deployment["name"], deployment["uuid"]) - return cls.start( - verification.verifier_uuid, deployment["uuid"], load_list=tests, - tags=tags, **run_args) + return cls.start(verification.verifier_uuid, + deployment["uuid"], load_list=tests, + tags=tags, **run_args) @staticmethod - def get(verification_uuid): + def _get(verification_uuid): """Get a verification. :param verification_uuid: Verification UUID """ return objects.Verification.get(verification_uuid) + @classmethod + def get(cls, verification_uuid): + return cls._get(verification_uuid).to_dict() + @staticmethod def list(verifier_id=None, deployment_id=None, tags=None, status=None): """List all verifications. @@ -934,9 +948,9 @@ class _Verification(object): :param tags: Tags to filter verifications by :param status: Status to filter verifications by """ - return objects.Verification.list(verifier_id, - deployment_id=deployment_id, - tags=tags, status=status) + return [item.to_dict() for item in objects.Verification.list( + verifier_id, deployment_id=deployment_id, + tags=tags, status=status)] @classmethod def delete(cls, verification_uuid): @@ -944,7 +958,7 @@ class _Verification(object): :param verification_uuid: Verification UUID """ - verification = cls.get(verification_uuid) + verification = cls._get(verification_uuid) LOG.info("Deleting verification (UUID=%s).", verification.uuid) verification.delete() LOG.info("Verification has been successfully deleted!") @@ -957,7 +971,7 @@ class _Verification(object): :param output_type: Plugin name of verification reporter :param output_dest: Destination for verification report """ - verifications = [cls.get(uuid) for uuid in uuids] + verifications = [cls._get(uuid) for uuid in uuids] reporter_cls = vreporter.VerificationReporter.get(output_type) reporter_cls.validate(output_dest) @@ -983,7 +997,7 @@ class _Verification(object): # required in the blueprint [1]. # [1] https://blueprints.launchpad.net/rally/+spec/verification-import - verifier = _Verifier.get(verifier_id) + verifier = _Verifier._get(verifier_id) verifier.set_deployment(deployment_id) LOG.info("Importing test results into a new verification for " "deployment '%s' (UUID=%s), using verifier %s.", @@ -1006,7 +1020,9 @@ class _Verification(object): LOG.info("Test results have been successfully imported.") - return verification, results + return {"verification": verification.to_dict(), + "totals": results.totals, + "tests": results.tests} class _DeprecatedAPIClass(object): diff --git a/rally/cli/cliutils.py b/rally/cli/cliutils.py index 8b29c660ce..2a0a9fa047 100644 --- a/rally/cli/cliutils.py +++ b/rally/cli/cliutils.py @@ -140,7 +140,6 @@ def print_list(objs, fields, formatters=None, sortby_index=0, data = o.get(field_name, "") else: data = getattr(o, field_name, "") - row.append(data) pt.add_row(row) diff --git a/rally/cli/commands/task.py b/rally/cli/commands/task.py index 6f2617c0b8..951d63b646 100644 --- a/rally/cli/commands/task.py +++ b/rally/cli/commands/task.py @@ -482,8 +482,7 @@ class TaskCommands(object): "load_duration": x["data"]["load_duration"], "full_duration": x["data"]["full_duration"], "created_at": x.get("created_at").strftime( - "%Y-%d-%mT%H:%M:%S")} - for x in task.get_results()] + "%Y-%d-%mT%H:%M:%S")} for x in task.get_results()] print(json.dumps(results, sort_keys=False, indent=4)) diff --git a/rally/cli/commands/verify.py b/rally/cli/commands/verify.py index dbeebf40b8..b7587458e1 100644 --- a/rally/cli/commands/verify.py +++ b/rally/cli/commands/verify.py @@ -17,6 +17,7 @@ from __future__ import print_function +import datetime as dt import json import os import webbrowser @@ -32,6 +33,7 @@ from rally.common import yamlutils as yaml from rally import exceptions from rally import plugins +TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" LIST_VERIFIERS_HINT = ("HINT: You can list all verifiers, executing " "command `rally verify list-verifiers`.") @@ -67,6 +69,25 @@ class VerifyCommands(object): failure = "\n%s%s\n" % (header, f["traceback"].strip()) print(failure) + def _print_details_after_run(self, results): + failures = [t for t in results["tests"].values() + if t["status"] == "fail"] + if failures: + h_text = "Failed %d %s - output below:" % ( + len(failures), "tests" if len(failures) > 1 else "test") + self._print_failures(h_text, failures, "=") + else: + print(_("\nCongratulations! Verification doesn't have failed " + "tests! :)")) + + @staticmethod + def _base_dir(uuid): + return os.path.expanduser( + "~/.rally/verification/verifier-%s" % uuid) + + def _get_location(self, uuid, loc): + return os.path.join(self._base_dir(uuid), loc) + @cliutils.args("--namespace", dest="namespace", type=str, metavar="", required=False, help="Namespace name (for example, openstack).") @@ -130,9 +151,10 @@ class VerifyCommands(object): def use_verifier(self, api, verifier_id): """Choose a verifier to use for the future operations.""" verifier = api.verifier.get(verifier_id) - fileutils.update_globals_file(envutils.ENV_VERIFIER, verifier.uuid) + fileutils.update_globals_file(envutils.ENV_VERIFIER, verifier["uuid"]) print(_("Using verifier '%s' (UUID=%s) as the default verifier " - "for the future operations.") % (verifier.name, verifier.uuid)) + "for the future operations.") % (verifier["name"], + verifier["uuid"])) @cliutils.help_group("verifier") @cliutils.args("--status", dest="status", type=str, required=False, @@ -147,9 +169,9 @@ class VerifyCommands(object): "Active"] cv = envutils.get_global(envutils.ENV_VERIFIER) formatters = { - "Created at": lambda v: v.created_at.replace(microsecond=0), - "Updated at": lambda v: v.updated_at.replace(microsecond=0), - "Active": lambda v: u"\u2714" if v.uuid == cv else "", + "Created at": lambda v: v["created_at"], + "Updated at": lambda v: v["updated_at"], + "Active": lambda v: u"\u2714" if v["uuid"] == cv else "", } cliutils.print_list(verifiers, fields, formatters=formatters, normalize_field_names=True, sortby_index=4) @@ -169,19 +191,22 @@ class VerifyCommands(object): verifier = api.verifier.get(verifier_id) fields = ["UUID", "Status", "Created at", "Updated at", "Active", "Name", "Description", "Type", "Namespace", "Source", - "Version", "System-wide", "Extra settings", "Location"] + "Version", "System-wide", "Extra settings", "Location", + "Venv location"] used_verifier = envutils.get_global(envutils.ENV_VERIFIER) formatters = { - "Created at": lambda v: v.created_at.replace(microsecond=0), - "Updated at": lambda v: v.updated_at.replace(microsecond=0), - "Active": lambda v: u"\u2714" if v.uuid == used_verifier else None, - "Extra settings": lambda v: (json.dumps(v.extra_settings, indent=4) - if v.extra_settings else None), - "Location": lambda v: v.manager.repo_dir + "Created at": lambda v: v["created_at"].replace("T", " "), + "Updated at": lambda v: v["updated_at"].replace("T", " "), + "Active": lambda v: u"\u2714" + if v["uuid"] == used_verifier else None, + "Extra settings": lambda v: (json.dumps(v["extra_settings"], + indent=4) + if v["extra_settings"] else None), + "Location": lambda v: self._get_location((v["uuid"]), "repo") } - if not verifier.system_wide: - fields.append("Venv location") - formatters["Venv location"] = lambda v: v.manager.venv_dir + if not verifier["system_wide"]: + formatters["Venv location"] = lambda v: self._get_location( + v["uuid"], ".venv") cliutils.print_dict(verifier, fields=fields, formatters=formatters, normalize_field_names=True, print_header=False, table_label="Verifier") @@ -379,16 +404,6 @@ class VerifyCommands(object): "verifier extension, using command `rally verify " "add-verifier-ext`.")) - def _print_details_after_run(self, results): - failures = results.filter_tests("fail").values() - if failures: - h_text = "Failed %d %s - output below:" % ( - len(failures), "tests" if len(failures) > 1 else "test") - self._print_failures(h_text, failures, "=") - else: - print(_("\nCongratulations! Verification doesn't have failed " - "tests! :)")) - @cliutils.help_group("verifier-ext") @cliutils.args("--id", dest="verifier_id", type=str, help="Verifier name or UUID. " + LIST_VERIFIERS_HINT) @@ -480,8 +495,9 @@ class VerifyCommands(object): ("concurrency", concur)) if value} try: - verification, results = api.verification.start( + results = api.verification.start( verifier_id, deployment, tags=tags, **run_args) + verification_uuid = results["verification"]["uuid"] except exceptions.DeploymentNotFinishedStatus as e: print(_("Cannot start a verefication on " "unfinished deployment: %s") % e) @@ -490,12 +506,12 @@ class VerifyCommands(object): if detailed: self._print_details_after_run(results) - self._print_totals(results.totals) + self._print_totals(results["totals"]) if do_use: - self.use(api, verification.uuid) + self.use(api, verification_uuid) else: - print(_("Verification UUID: %s.") % verification.uuid) + print(_("Verification UUID: %s.") % verification_uuid) @cliutils.help_group("verification") @cliutils.args("--uuid", dest="verification_uuid", type=str, required=True, @@ -504,9 +520,9 @@ class VerifyCommands(object): """Choose a verification to use for the future operations.""" verification = api.verification.get(verification_uuid) fileutils.update_globals_file( - envutils.ENV_VERIFICATION, verification.uuid) + envutils.ENV_VERIFICATION, verification["uuid"]) print(_("Using verification (UUID=%s) as the default verification " - "for the future operations.") % verification.uuid) + "for the future operations.") % verification["uuid"]) @cliutils.help_group("verification") @cliutils.args("--uuid", dest="verification_uuid", type=str, @@ -536,20 +552,21 @@ class VerifyCommands(object): def rerun(self, api, verification_uuid=None, deployment=None, tags=None, concur=None, failed=False, detailed=False, do_use=True): """Rerun tests from a verification for a specific deployment.""" - verification, results = api.verification.rerun(verification_uuid, - deployment=deployment, - failed=failed, - tags=tags, - concur=concur) + results = api.verification.rerun(verification_uuid, + deployment=deployment, + failed=failed, + tags=tags, + concur=concur) if detailed: self._print_details_after_run(results) - self._print_totals(results.totals) + self._print_totals(results["totals"]) if do_use: - self.use(api, verification.uuid) + self.use(api, results["verification"]["uuid"]) else: - print(_("Verification UUID: %s.") % verification.uuid) + print(_("Verification UUID: %s.") + % results["verification"]["uuid"]) @cliutils.help_group("verification") @cliutils.args("--uuid", dest="verification_uuid", type=str, @@ -566,12 +583,12 @@ class VerifyCommands(object): detailed=False): """Show detailed information about a verification.""" verification = api.verification.get(verification_uuid) - verifier = api.verifier.get(verification.verifier_uuid) - deployment = api.deployment.get(verification.deployment_uuid) + verifier = api.verifier.get(verification["verifier_uuid"]) + deployment = api.deployment.get(verification["deployment_uuid"]) def run_args_formatter(v): run_args = [] - for k in sorted(v.run_args): + for k in sorted(v["run_args"]): if k in ("load_list", "skip_list", "xfail_list"): value = "(value is too long, %s)" if detailed: @@ -579,7 +596,7 @@ class VerifyCommands(object): else: value %= "use 'detailed' flag to display it" else: - value = v.run_args[k] + value = v["run_args"][k] run_args.append("%s: %s" % (k, value)) return "\n".join(run_args) @@ -590,21 +607,23 @@ class VerifyCommands(object): "Success", "Skipped", "Expected failures", "Unexpected success", "Failures"] formatters = { - "Started at": lambda v: v.created_at.replace(microsecond=0), - "Finished at": lambda v: v.updated_at.replace(microsecond=0), - "Duration": lambda v: (v.updated_at.replace(microsecond=0) - - v.created_at.replace(microsecond=0)), + "Started at": lambda v: v["created_at"].replace("T", " "), + "Finished at": lambda v: v["updated_at"].replace("T", " "), + "Duration": lambda v: (dt.datetime.strptime(v["updated_at"], + TIME_FORMAT) - + dt.datetime.strptime(v["created_at"], + TIME_FORMAT)), "Run arguments": run_args_formatter, - "Tags": lambda v: ", ".join(v.tags) or None, - "Verifier name": lambda v: "%s (UUID: %s)" % (verifier.name, - verifier.uuid), + "Tags": lambda v: ", ".join(v["tags"]) or None, + "Verifier name": lambda v: "%s (UUID: %s)" % (verifier["name"], + verifier["uuid"]), "Verifier type": ( - lambda v: "%s (namespace: %s)" % (verifier.type, - verifier.namespace)), + lambda v: "%s (namespace: %s)" % (verifier["type"], + verifier["namespace"])), "Deployment name": ( lambda v: "%s (UUID: %s)" % (deployment["name"], deployment["uuid"])), - "Tests duration, sec": lambda v: v.tests_duration + "Tests duration, sec": lambda v: v["tests_duration"] } cliutils.print_dict(verification, fields, formatters=formatters, normalize_field_names=True, print_header=False, @@ -613,10 +632,10 @@ class VerifyCommands(object): if detailed: h = _("Run arguments") print("\n%s" % cliutils.make_header(h, len(h)).strip()) - print("\n%s\n" % json.dumps(verification.run_args, indent=4)) + print("\n%s\n" % json.dumps(verification["run_args"], indent=4)) # Tests table - tests = verification.tests + tests = verification["tests"] values = [tests[test_id] for test_id in tests] fields = ["Name", "Duration, sec", "Status"] formatters = {"Duration, sec": lambda v: v["duration"]} @@ -646,21 +665,23 @@ class VerifyCommands(object): def list(self, api, verifier_id=None, deployment=None, tags=None, status=None): """List all verifications.""" - verifications = api.verification.list(verifier_id, deployment, tags, - status) + verifications = api.verification.list(verifier_id, deployment, + tags, status) if verifications: fields = ["UUID", "Tags", "Verifier name", "Deployment name", "Started at", "Finished at", "Duration", "Status"] formatters = { - "Tags": lambda v: ", ".join(v.tags) or "-", - "Verifier name": ( - lambda v: api.verifier.get(v.verifier_uuid).name), - "Deployment name": ( - lambda v: api.deployment.get(v.deployment_uuid)["name"]), - "Started at": lambda v: v.created_at.replace(microsecond=0), - "Finished at": lambda v: v.updated_at.replace(microsecond=0), - "Duration": lambda v: (v.updated_at.replace(microsecond=0) - - v.created_at.replace(microsecond=0)) + "Tags": lambda v: ", ".join(v["tags"]) or "-", + "Verifier name": (lambda v: api.verifier.get( + v["verifier_uuid"])["name"]), + "Deployment name": (lambda v: api.deployment.get( + v["deployment_uuid"])["name"]), + "Started at": lambda v: v["created_at"], + "Finished at": lambda v: v["updated_at"], + "Duration": lambda v: (dt.datetime.strptime(v["updated_at"], + TIME_FORMAT) - + dt.datetime.strptime(v["created_at"], + TIME_FORMAT)) } cliutils.print_list(verifications, fields, formatters=formatters, normalize_field_names=True, sortby_index=4) @@ -765,9 +786,10 @@ class VerifyCommands(object): run_args = yaml.safe_load(run_args) if run_args else {} verification, results = api.verification.import_results( verifier_id, deployment, data, **run_args) - self._print_totals(results.totals) + self._print_totals(results["totals"]) + verification_uuid = verification["uuid"] if do_use: - self.use(verification.uuid) + self.use(api, verification_uuid) else: - print(_("Verification UUID: %s.") % verification.uuid) + print(_("Verification UUID: %s.") % verification_uuid) diff --git a/rally/common/db/sqlalchemy/models.py b/rally/common/db/sqlalchemy/models.py index 868085b772..f8fead13dc 100644 --- a/rally/common/db/sqlalchemy/models.py +++ b/rally/common/db/sqlalchemy/models.py @@ -37,6 +37,7 @@ def UUID(): class RallyBase(models.ModelBase): + metadata = None created_at = sa.Column(sa.DateTime, default=lambda: timeutils.utcnow()) updated_at = sa.Column(sa.DateTime, default=lambda: timeutils.utcnow(), diff --git a/rally/common/objects/verification.py b/rally/common/objects/verification.py index 4e4afd73e2..eafa637b65 100644 --- a/rally/common/objects/verification.py +++ b/rally/common/objects/verification.py @@ -19,6 +19,7 @@ from rally import consts class Verification(object): """Represents a verification object.""" + TIME_FORMAT = consts.TimeFormat.TIME_FORMAT_ISO8601 def __init__(self, verification): """Init a verification object. @@ -34,6 +35,20 @@ class Verification(object): def __getitem__(self, item): return self._db_entry[item] + def to_dict(self, item=None): + data = {} + formatters = ["created_at", "updated_at"] + fields = ["deployment_uuid", "verifier_uuid", "uuid", "id", + "unexpected_success", "status", "tests", "skipped", + "tags", "tests_duration", "run_args", "success", + "expected_failures", "tests_count", "failures"] + for field in fields: + data[field] = self._db_entry.get(field, "") + for field in formatters: + data[field] = self._db_entry.get(field, "").strftime( + self.TIME_FORMAT) + return data + @classmethod def create(cls, verifier_id, deployment_id, tags=None, run_args=None): return cls(db.verification_create( diff --git a/rally/common/objects/verifier.py b/rally/common/objects/verifier.py index df18748ebb..f69fe3277c 100644 --- a/rally/common/objects/verifier.py +++ b/rally/common/objects/verifier.py @@ -14,12 +14,14 @@ # under the License. from rally.common import db +from rally import consts from rally import exceptions from rally.verification import manager class Verifier(object): """Represents a verifier object.""" + TIME_FORMAT = consts.TimeFormat.TIME_FORMAT_ISO8601 def __init__(self, verifier): """Init a verifier object. @@ -40,6 +42,19 @@ class Verifier(object): def __str__(self): return "'%s' (UUID=%s)" % (self.name, self.uuid) + def to_dict(self, item=None): + data = {} + formatters = ["created_at", "updated_at"] + fields = ["status", "system_wide", "uuid", "type", "namespace", + "name", "source", "version", "extra_settings", + "id", "description"] + for field in fields: + data[field] = self._db_entry.get(field, "") + for field in formatters: + data[field] = self._db_entry.get(field, "").strftime( + self.TIME_FORMAT) + return data + @classmethod def create(cls, name, vtype, namespace, source, version, system_wide, extra_settings=None): diff --git a/rally/consts.py b/rally/consts.py index 5d76ebca23..d5215542cb 100644 --- a/rally/consts.py +++ b/rally/consts.py @@ -214,6 +214,11 @@ class _VerificationStatus(utils.ImmutableMixin, utils.EnumMixin): CRASHED = "crashed" +class _TimeFormat(utils.ImmutableMixin, utils.EnumMixin): + """International time formats""" + TIME_FORMAT_ISO8601 = "%Y-%m-%dT%H:%M:%S%z" + + TaskStatus = _TaskStatus() SubtaskStatus = _SubtaskStatus() DeployStatus = _DeployStatus() @@ -225,3 +230,4 @@ HookStatus = _HookStatus() TagType = _TagType() VerifierStatus = _VerifierStatus() VerificationStatus = _VerificationStatus() +TimeFormat = _TimeFormat() diff --git a/rally/plugins/common/verification/reporters.py b/rally/plugins/common/verification/reporters.py index 96185388fe..d887452b70 100644 --- a/rally/plugins/common/verification/reporters.py +++ b/rally/plugins/common/verification/reporters.py @@ -19,13 +19,14 @@ import re import xml.etree.ElementTree as ET from rally.common import version +from rally import consts from rally.ui import utils from rally.verification import reporter SKIP_RE = re.compile("Skipped until Bug: ?(?P\d+) is resolved.") LP_BUG_LINK = "https://launchpad.net/bugs/%s" -TIME_FORMAT_ISO8601 = "%Y-%m-%dT%H:%M:%S%z" +TIME_FORMAT = consts.TimeFormat.TIME_FORMAT_ISO8601 @reporter.configure("json") @@ -176,7 +177,6 @@ class JSONReporter(reporter.VerificationReporter): } """ - TIME_FORMAT = TIME_FORMAT_ISO8601 @classmethod def validate(cls, output_destination): @@ -195,8 +195,8 @@ class JSONReporter(reporter.VerificationReporter): for v in self.verifications: verifications[v.uuid] = { - "started_at": v.created_at.strftime(self.TIME_FORMAT), - "finished_at": v.updated_at.strftime(self.TIME_FORMAT), + "started_at": v.created_at.strftime(TIME_FORMAT), + "finished_at": v.updated_at.strftime(TIME_FORMAT), "status": v.status, "run_args": v.run_args, "tests_count": v.tests_count, @@ -434,7 +434,7 @@ class JUnitXMLReporter(reporter.VerificationReporter): root.append(ET.Comment("Report is generated by Rally %s at %s" % ( version.version_string(), - dt.datetime.utcnow().strftime(TIME_FORMAT_ISO8601)))) + dt.datetime.utcnow().strftime(TIME_FORMAT)))) for v in self.verifications: verification = ET.SubElement(root, "testsuite", { @@ -444,7 +444,7 @@ class JUnitXMLReporter(reporter.VerificationReporter): "errors": "0", "skipped": str(v.skipped), "failures": str(v.failures + v.unexpected_success), - "timestamp": v.created_at.strftime(TIME_FORMAT_ISO8601) + "timestamp": v.created_at.strftime(TIME_FORMAT) }) tests = sorted(v.tests.values(), key=lambda t: (t.get("timestamp", ""), t["name"])) diff --git a/tests/unit/cli/commands/test_verify.py b/tests/unit/cli/commands/test_verify.py index 1f51b8041f..1ef9962046 100644 --- a/tests/unit/cli/commands/test_verify.py +++ b/tests/unit/cli/commands/test_verify.py @@ -13,7 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -import datetime as dt import tempfile import mock @@ -38,347 +37,39 @@ class VerifyCommandsTestCase(test.TestCase): self.verify = verify.VerifyCommands() self.fake_api = fakes.FakeAPI() - @mock.patch("rally.cli.commands.verify.cliutils.print_list") - @mock.patch("rally.cli.commands.verify.logging.is_debug", - return_value=True) - def test_list_plugins(self, mock_is_debug, mock_print_list): - self.verify.list_plugins(self.fake_api, namespace="some") - self.fake_api.verifier.list_plugins.assert_called_once_with("some") + self.deployment_name = "Some Deploy" + self.deployment_uuid = "some-deploy-uuid" + self.verifier_name = "My Verifier" + self.verifier_uuid = "my-verifier-uuid" + self.verifier_type = "OldSchoolTestTool" + self.verifier_namespace = "OpenStack" + self.verification_uuid = "uuuiiiiddd" - @mock.patch("rally.cli.commands.verify.fileutils.update_globals_file") - def test_create_verifier(self, mock_update_globals_file): - self.fake_api.verifier.create.return_value = "v_uuid" - self.fake_api.verifier.get.return_value = mock.Mock(uuid="v_uuid") - - self.verify.create_verifier(self.fake_api, "a", vtype="b", - namespace="c", source="d", version="e", - system_wide=True, extra={}) - self.fake_api.verifier.create.assert_called_once_with( - "a", vtype="b", namespace="c", source="d", version="e", - system_wide=True, extra_settings={}) - - self.fake_api.verifier.get.assert_called_once_with("v_uuid") - mock_update_globals_file.assert_called_once_with( - envutils.ENV_VERIFIER, "v_uuid") - - @mock.patch("rally.cli.commands.verify.fileutils.update_globals_file") - def test_use_verifier(self, mock_update_globals_file): - self.fake_api.verifier.get.return_value = mock.Mock(uuid="v_uuid") - self.verify.use_verifier(self.fake_api, "v_uuid") - self.fake_api.verifier.get.assert_called_once_with("v_uuid") - mock_update_globals_file.assert_called_once_with( - envutils.ENV_VERIFIER, "v_uuid") - - @mock.patch("rally.cli.commands.verify.cliutils.print_list") - @mock.patch("rally.cli.commands.verify.logging.is_debug", - return_value=True) - def test_list_verifiers(self, mock_is_debug, mock_print_list): - self.verify.list_verifiers(self.fake_api) - - self.fake_api.verifier.list.return_value = [] - self.verify.list_verifiers(self.fake_api, "foo") - self.verify.list_verifiers(self.fake_api) - - self.fake_api.verifier.list.assert_has_calls([mock.call(None), - mock.call("foo")]) - - @mock.patch("rally.cli.commands.verify.envutils.get_global") - def test_show_verifier(self, mock_get_global): - fake_verifier = self.fake_api.verifier.get.return_value - fake_verifier.uuid = "v_uuid" - fake_verifier.name = "Verifier!" - fake_verifier.type = "CoolTool" - fake_verifier.namespace = "ExampleNamespace" - fake_verifier.description = "The best tool in the world" - fake_verifier.created_at = dt.datetime(2016, 1, 1, 17, 0, 3, 66) - fake_verifier.updated_at = dt.datetime(2016, 1, 1, 17, 1, 5, 77) - fake_verifier.status = "installed" - fake_verifier.source = "https://example.com" - fake_verifier.version = "master" - fake_verifier.system_wide = False - fake_verifier.extra_settings = {} - fake_verifier.manager.repo_dir = "./verifiers/repo" - fake_verifier.manager.venv_dir = "./verifiers/.venv" - - # It is a hard task to mock default value of function argument, so we - # need to apply this workaround - original_print_dict = cliutils.print_dict - print_dict_calls = [] - - def print_dict(*args, **kwargs): - print_dict_calls.append(six.StringIO()) - kwargs["out"] = print_dict_calls[-1] - original_print_dict(*args, **kwargs) - - with mock.patch.object(verify.cliutils, "print_dict", - new=print_dict): - self.verify.show_verifier(self.fake_api, "v_uuid") - - self.assertEqual(1, len(print_dict_calls)) - - self.assertEqual( - "+---------------------------------------------+\n" - "| Verifier |\n" - "+----------------+----------------------------+\n" - "| UUID | v_uuid |\n" - "| Status | installed |\n" - "| Created at | 2016-01-01 17:00:03 |\n" - "| Updated at | 2016-01-01 17:01:05 |\n" - "| Active | - |\n" - "| Name | Verifier! |\n" - "| Description | The best tool in the world |\n" - "| Type | CoolTool |\n" - "| Namespace | ExampleNamespace |\n" - "| Source | https://example.com |\n" - "| Version | master |\n" - "| System-wide | False |\n" - "| Extra settings | - |\n" - "| Location | ./verifiers/repo |\n" - "| Venv location | ./verifiers/.venv |\n" - "+----------------+----------------------------+\n", - print_dict_calls[0].getvalue()) - - self.fake_api.verifier.get.assert_called_once_with("v_uuid") - - def test_delete_verifier(self): - self.verify.delete_verifier(self.fake_api, "v_id", "d_id", force=True) - self.fake_api.verifier.delete.assert_called_once_with( - "v_id", "d_id", True) - - def test_update_verifier(self): - self.verify.update_verifier(self.fake_api, "v_id") - self.assertFalse(self.fake_api.verifier.update.called) - - self.verify.update_verifier(self.fake_api, "v_id", update_venv=True, - system_wide=True) - self.assertFalse(self.fake_api.verifier.update.called) - - self.verify.update_verifier(self.fake_api, "v_id", system_wide=True, - no_system_wide=True) - self.assertFalse(self.fake_api.verifier.update.called) - - self.verify.update_verifier(self.fake_api, "v_id", version="a", - system_wide=True) - self.fake_api.verifier.update.assert_called_once_with( - "v_id", system_wide=True, version="a", update_venv=None) - - @mock.patch("rally.cli.commands.verify.open", create=True) - @mock.patch("rally.cli.commands.verify.os.path.exists") - def test_configure_verifier(self, mock_exists, mock_open): - self.verify.configure_verifier(self.fake_api, "v_id", "d_id", - new_configuration="/p/a/t/h", - reconfigure=True, - show=True) - self.assertFalse(self.fake_api.verifier.configure.called) - - mock_exists.return_value = False - self.verify.configure_verifier(self.fake_api, "v_id", "d_id", - new_configuration="/p/a/t/h", - show=True) - self.assertFalse(self.fake_api.verifier.override_configuration.called) - - mock_exists.return_value = True - mock_open.return_value = mock.mock_open(read_data="data").return_value - self.verify.configure_verifier(self.fake_api, "v_id", "d_id", - new_configuration="/p/a/t/h", - show=True) - mock_open.assert_called_once_with("/p/a/t/h") - self.fake_api.verifier.override_configuration("v_id", "d_id", "data") - - tf = tempfile.NamedTemporaryFile() - with open(tf.name, "w") as f: - f.write("[DEFAULT]\nopt = val\n[foo]\nopt = val") - self.verify.configure_verifier(self.fake_api, "v_id", "d_id", - extra_options=tf.name) - expected_options = {"foo": {"opt": "val"}, - "DEFAULT": {"opt": "val"}} - self.fake_api.verifier.configure.assert_called_once_with( - "v_id", "d_id", extra_options=expected_options, reconfigure=False) - - self.verify.configure_verifier(self.fake_api, "v_id", "d_id", - extra_options="{foo: {opt: val}, " - "DEFAULT: {opt: val}}") - self.fake_api.verifier.configure.assert_called_with( - "v_id", "d_id", extra_options=expected_options, reconfigure=False) - - def test_list_verifier_tests(self): - self.fake_api.verifier.list_tests.return_value = ["test_1", "test_2"] - self.verify.list_verifier_tests(self.fake_api, "v_id", "p") - - self.fake_api.verifier.list_tests.return_value = [] - self.verify.list_verifier_tests(self.fake_api, "v_id", "p") - - self.fake_api.verifier.list_tests.assert_has_calls( - [mock.call("v_id", "p"), mock.call("v_id", "p")]) - - def test_add_verifier_ext(self): - self.verify.add_verifier_ext(self.fake_api, "v_id", "a", "b", "c") - self.fake_api.verifier.add_extension.assert_called_once_with( - "v_id", source="a", version="b", extra_settings="c") - - @mock.patch("rally.cli.commands.verify.cliutils.print_list") - @mock.patch("rally.cli.commands.verify.logging.is_debug", - return_value=True) - def test_list_verifier_exts(self, mock_is_debug, mock_print_list): - self.verify.list_verifier_exts(self.fake_api, "v_id") - - self.fake_api.verifier.list_extensions.return_value = [] - self.verify.list_verifier_exts(self.fake_api, "v_id") - - self.fake_api.verifier.list_extensions.assert_has_calls( - [mock.call("v_id"), mock.call("v_id")]) - - def test_delete_verifier_ext(self): - self.verify.delete_verifier_ext(self.fake_api, "v_id", "ext_name") - self.fake_api.verifier.delete_extension.assert_called_once_with( - "v_id", "ext_name") - - @mock.patch("rally.cli.commands.verify.fileutils.update_globals_file") - @mock.patch("rally.cli.commands.verify.os.path.exists") - def test_start(self, mock_exists, mock_update_globals_file): - self.verify.start(self.fake_api, "v_id", "d_id", pattern="pattern", - load_list="load-list") - self.assertFalse(self.fake_api.verification.start.called) - - verification = mock.Mock(uuid="v_uuid") - failed_test = { - "test_2": { - "name": "test_2", - "status": "fail", - "duration": 2, - "traceback": "Some traceback" - } + self.verifier_data = { + "uuid": self.verifier_uuid, + "name": self.verifier_name, + "type": self.verifier_type, + "namespace": self.verifier_namespace, + "description": "The best tool in the world", + "created_at": "2016-01-01T17:00:03", + "updated_at": "2016-01-01T17:01:05", + "status": "installed", + "source": "https://example.com", + "version": "master", + "system_wide": False, + "extra_settings": {}, + "manager.repo_dir": "./verifiers/repo", + "manager.venv_dir": "./verifiers/.venv" } - test_results = { - "tests": { - "test_1": { - "name": "test_1", - "status": "success", - "duration": 2, - "tags": [] - } - }, - "totals": { - "tests_count": 2, - "tests_duration": 4, - "success": 2, - "skipped": 0, - "expected_failures": 0, - "unexpected_success": 0, - "failures": 0 - } - } - test_results["tests"].update(failed_test) - results = mock.Mock(**test_results) - results.filter_tests.return_value = failed_test - self.fake_api.verification.start.return_value = (verification, results) - self.fake_api.verification.get.return_value = verification - mock_exists.return_value = False - self.verify.start(self.fake_api, "v_id", "d_id", load_list="/p/a/t/h") - self.assertFalse(self.fake_api.verification.start.called) - - mock_exists.return_value = True - tf = tempfile.NamedTemporaryFile() - with open(tf.name, "w") as f: - f.write("test_1\ntest_2") - self.verify.start(self.fake_api, "v_id", "d_id", tags=["foo"], - load_list=tf.name) - self.fake_api.verification.start.assert_called_once_with( - "v_id", "d_id", tags=["foo"], load_list=["test_1", "test_2"]) - - mock_exists.return_value = False - self.fake_api.verification.start.reset_mock() - self.verify.start(self.fake_api, "v_id", "d_id", skip_list="/p/a/t/h") - self.assertFalse(self.fake_api.verification.start.called) - - tf = tempfile.NamedTemporaryFile() - with open(tf.name, "w") as f: - f.write("test_1:\ntest_2: Reason\n") - mock_exists.return_value = True - self.verify.start(self.fake_api, "v_id", "d_id", skip_list=tf.name) - self.fake_api.verification.start.assert_called_once_with( - "v_id", "d_id", tags=None, skip_list={"test_1": None, - "test_2": "Reason"}) - - mock_exists.return_value = False - self.fake_api.verification.start.reset_mock() - self.verify.start(self.fake_api, "v_id", "d_id", xfail_list="/p/a/t/h") - self.assertFalse(self.fake_api.verification.start.called) - - tf = tempfile.NamedTemporaryFile() - with open(tf.name, "w") as f: - f.write("test_1:\ntest_2: Reason\n") - mock_exists.return_value = True - self.verify.start(self.fake_api, "v_id", "d_id", xfail_list=tf.name) - self.fake_api.verification.start.assert_called_once_with( - "v_id", "d_id", tags=None, xfail_list={"test_1": None, - "test_2": "Reason"}) - - self.fake_api.verification.get.assert_called_with("v_uuid") - mock_update_globals_file.assert_called_with( - envutils.ENV_VERIFICATION, "v_uuid") - - self.fake_api.verification.get.reset_mock() - mock_update_globals_file.reset_mock() - self.verify.start(self.fake_api, "v_id", "d_id", detailed=True, - do_use=False) - self.assertFalse(self.fake_api.verification.get.called) - self.assertFalse(mock_update_globals_file.called) - - @mock.patch("rally.cli.commands.verify.os.path.exists") - @mock.patch("rally.cli.commands.verify.fileutils.update_globals_file") - def test_start_on_unfinished_deployment(self, mock_update_globals_file, - mock_exists): - deployment_id = "d_id" - deployment_name = "xxx_name" - exc = exceptions.DeploymentNotFinishedStatus( - name=deployment_name, - uuid=deployment_id, - status=consts.DeployStatus.DEPLOY_INIT) - self.fake_api.verification.start.side_effect = exc - self.assertEqual( - 1, self.verify.start(self.fake_api, "v_id", deployment_id)) - - @mock.patch("rally.cli.commands.verify.fileutils.update_globals_file") - def test_use(self, mock_update_globals_file): - self.fake_api.verification.get.return_value = mock.Mock(uuid="v_uuid") - self.verify.use(self.fake_api, "v_uuid") - self.fake_api.verification.get.assert_called_once_with("v_uuid") - mock_update_globals_file.assert_called_once_with( - envutils.ENV_VERIFICATION, "v_uuid") - - def test_rerun(self): - verification = mock.Mock(uuid="v_uuid") - results = mock.Mock(totals={"tests_count": 2, - "tests_duration": 4, - "success": 2, - "skipped": 0, - "expected_failures": 0, - "unexpected_success": 0, - "failures": 0}) - self.fake_api.verification.rerun.return_value = (verification, results) - - self.verify.rerun(self.fake_api, "v_uuid", "d_id", failed=True,) - self.fake_api.verification.rerun.assert_called_once_with( - "v_uuid", deployment="d_id", failed=True, tags=None, concur=None) - - def test_show(self): - deployment_name = "Some Deploy" - deployment_uuid = "some-deploy-uuid" - verifier_name = "My Verifier" - verifier_uuid = "my-verifier-uuid" - verifier_type = "OldSchoolTestTool" - verifier_namespace = "OpenStack" - verifier = mock.Mock(type=verifier_type, namespace=verifier_namespace) - verifier.name = verifier_name - verifier.uuid = verifier_uuid - verification = { - "uuid": "uuuiiiiddd", + self.verification_data = { + "uuid": self.verification_uuid, + "verifier_uuid": self.verifier_uuid, + "deployment_uuid": self.deployment_uuid, "tags": ["bar", "foo"], "status": "success", - "created_at": dt.datetime(2016, 1, 1, 17, 0, 3, 66), - "updated_at": dt.datetime(2016, 1, 1, 17, 1, 5, 77), + "created_at": "2016-01-01T17:00:03", + "updated_at": "2016-01-01T17:01:05", "tests_count": 2, "tests_duration": 4, "success": 1, @@ -389,27 +80,111 @@ class VerifyCommandsTestCase(test.TestCase): "run_args": { "load_list": ["test_1", "test_2"], "skip_list": ["test_3"], - "concurrency": "3" - }, + "concurrency": "3"}, "tests": { "test_1": { "name": "test_1", "status": "success", "duration": 2, - "tags": [] - }, + "tags": []}, "test_2": { "name": "test_2", "status": "fail", "duration": 2, - "traceback": "Some traceback" - } - } + "traceback": "Some traceback"} + }, + "test_2": { + "name": "test_2", + "status": "fail", + "duration": 2, + "traceback": "Some traceback"} } - self.fake_api.verifier.get.return_value = verifier - self.fake_api.verification.get.return_value = mock.Mock(**verification) - self.fake_api.deployment.get.return_value = {"name": deployment_name, - "uuid": deployment_uuid} + + self.results_data = { + "totals": {"tests_count": 2, + "tests_duration": 4, + "success": 1, + "skipped": 0, + "expected_failures": 0, + "unexpected_success": 0, + "failures": 1}, + "tests": { + "test_1": { + "name": "test_1", + "status": "success", + "duration": 2, + "tags": []} + }, + "test_2": { + "name": "test_2", + "status": "fail", + "duration": 4, + "tags": []} + } + + @mock.patch("rally.cli.commands.verify.cliutils.print_list") + @mock.patch("rally.cli.commands.verify.logging.is_debug", + return_value=True) + def test_list_plugins(self, mock_is_debug, mock_print_list): + self.verify.list_plugins(self.fake_api, namespace="some") + self.fake_api.verifier.list_plugins.assert_called_once_with("some") + + @mock.patch("rally.cli.commands.verify.fileutils.update_globals_file") + def test_create_verifier(self, mock_update_globals_file): + self.fake_api.verifier.create.return_value = self.verifier_uuid + self.fake_api.verifier.get.return_value = self.verifier_data + + self.verify.create_verifier(self.fake_api, "a", vtype="b", + namespace="c", source="d", version="e", + system_wide=True, extra={}) + self.fake_api.verifier.create.assert_called_once_with( + "a", vtype="b", namespace="c", source="d", version="e", + system_wide=True, extra_settings={}) + + self.fake_api.verifier.get.assert_called_once_with(self.verifier_uuid) + mock_update_globals_file.assert_called_once_with( + envutils.ENV_VERIFIER, self.verifier_uuid) + + @mock.patch("rally.cli.commands.verify.fileutils.update_globals_file") + def test_use_verifier(self, mock_update_globals_file): + self.fake_api.verifier.get.return_value = self.verifier_data + self.verify.use_verifier(self.fake_api, self.verifier_uuid) + self.fake_api.verifier.get.assert_called_once_with(self.verifier_uuid) + mock_update_globals_file.assert_called_once_with( + envutils.ENV_VERIFIER, self.verifier_uuid) + + @mock.patch("rally.cli.commands.verify.cliutils.print_list") + def test_list_verifiers_empty_verifiers(self, mock_print_list): + self.fake_api.verifier.list.return_value = [] + self.verify.list_verifiers(self.fake_api) + + self.verify.list_verifiers(self.fake_api, "foo") + self.verify.list_verifiers(self.fake_api) + + self.fake_api.verifier.list.assert_has_calls([mock.call(None), + mock.call("foo")]) + + @mock.patch("rally.cli.commands.verify.cliutils.print_list") + def test_list_verifiers(self, mock_print_list): + self.fake_api.verifier.list.return_value = [self.verifier_data] + + additional_fields = ["UUID", "Name", "Type", "Namespace", "Created at", + "Updated at", "Status", "Version", "System-wide", + "Active"] + additional_keys = ["normalize_field_names", "sortby_index", + "formatters"] + self.verify.list_verifiers(self.fake_api) + # astarove: should be replaced on mock_print_list.assert_called_once() + self.assertTrue(1, len(mock_print_list.call_args())) + self.assertEqual(([self.verifier_data], additional_fields), + mock_print_list.call_args[0]) + self.assertEqual(additional_keys.sort(), + list(mock_print_list.call_args[1].keys()).sort()) + + @mock.patch("rally.cli.commands.verify.envutils.get_global") + def test_show_verifier(self, mock_get_global): + self.fake_api.verifier.get.return_value = self.verifier_data + self.verify._base_dir = mock.Mock(return_value="./verifiers/") # It is a hard task to mock default value of function argument, so we # need to apply this workaround @@ -423,7 +198,319 @@ class VerifyCommandsTestCase(test.TestCase): with mock.patch.object(verify.cliutils, "print_dict", new=print_dict): - self.verify.show(self.fake_api, "v_uuid", detailed=True) + self.verify.show_verifier(self.fake_api, self.verifier_uuid) + + self.assertEqual(1, len(print_dict_calls)) + + self.assertEqual( + "+---------------------------------------------+\n" + "| Verifier |\n" + "+----------------+----------------------------+\n" + "| UUID | my-verifier-uuid |\n" + "| Status | installed |\n" + "| Created at | 2016-01-01 17:00:03 |\n" + "| Updated at | 2016-01-01 17:01:05 |\n" + "| Active | - |\n" + "| Name | My Verifier |\n" + "| Description | The best tool in the world |\n" + "| Type | OldSchoolTestTool |\n" + "| Namespace | OpenStack |\n" + "| Source | https://example.com |\n" + "| Version | master |\n" + "| System-wide | False |\n" + "| Extra settings | - |\n" + "| Location | ./verifiers/repo |\n" + "| Venv location | ./verifiers/.venv |\n" + "+----------------+----------------------------+\n", + print_dict_calls[0].getvalue()) + + self.fake_api.verifier.get.assert_called_once_with(self.verifier_uuid) + + def test_delete_verifier(self): + self.verify.delete_verifier(self.fake_api, "v_id", "d_id", force=True) + self.fake_api.verifier.delete.assert_called_once_with( + "v_id", "d_id", True) + + def test_update_verifier(self): + self.verify.update_verifier(self.fake_api, self.verifier_uuid) + self.assertFalse(self.fake_api.verifier.update.called) + + self.verify.update_verifier(self.fake_api, self.verification_uuid, + update_venv=True, + system_wide=True) + self.assertFalse(self.fake_api.verifier.update.called) + + self.verify.update_verifier(self.fake_api, self.verification_uuid, + system_wide=True, + no_system_wide=True) + self.assertFalse(self.fake_api.verifier.update.called) + + self.verify.update_verifier(self.fake_api, self.verification_uuid, + version="a", + system_wide=True) + self.fake_api.verifier.update.assert_called_once_with( + self.verification_uuid, system_wide=True, + version="a", update_venv=None) + + @mock.patch("rally.cli.commands.verify.open", create=True) + @mock.patch("rally.cli.commands.verify.os.path.exists") + def test_configure_verifier(self, mock_exists, mock_open): + self.verify.configure_verifier(self.fake_api, self.verifier_uuid, + self.deployment_uuid, + new_configuration="/p/a/t/h", + reconfigure=True, + show=True) + self.assertFalse(self.fake_api.verifier.configure.called) + + mock_exists.return_value = False + self.verify.configure_verifier(self.fake_api, self.verifier_uuid, + self.deployment_uuid, + new_configuration="/p/a/t/h", + show=True) + self.assertFalse(self.fake_api.verifier.override_configuration.called) + + mock_exists.return_value = True + mock_open.return_value = mock.mock_open(read_data="data").return_value + self.verify.configure_verifier(self.fake_api, self.verifier_uuid, + self.deployment_uuid, + new_configuration="/p/a/t/h", + show=True) + mock_open.assert_called_once_with("/p/a/t/h") + self.fake_api.verifier.override_configuration(self.verifier_uuid, + self.deployment_uuid, + "data") + + tf = tempfile.NamedTemporaryFile() + with open(tf.name, "w") as f: + f.write("[DEFAULT]\nopt = val\n[foo]\nopt = val") + self.verify.configure_verifier(self.fake_api, self.verifier_uuid, + self.deployment_uuid, + extra_options=tf.name) + expected_options = {"foo": {"opt": "val"}, + "DEFAULT": {"opt": "val"}} + self.fake_api.verifier.configure.assert_called_once_with( + self.verifier_uuid, self.deployment_uuid, + extra_options=expected_options, reconfigure=False) + + self.verify.configure_verifier(self.fake_api, self.verifier_uuid, + self.deployment_uuid, + extra_options="{foo: {opt: val}, " + "DEFAULT: {opt: val}}") + self.fake_api.verifier.configure.assert_called_with( + self.verifier_uuid, self.deployment_uuid, + extra_options=expected_options, reconfigure=False) + + def test_list_verifier_tests(self): + self.fake_api.verifier.list_tests.return_value = ["test_1", "test_2"] + self.verify.list_verifier_tests(self.fake_api, self.verifier_uuid, "p") + + self.fake_api.verifier.list_tests.return_value = [] + self.verify.list_verifier_tests(self.fake_api, self.verifier_uuid, "p") + + self.fake_api.verifier.list_tests.assert_has_calls( + [mock.call(self.verifier_uuid, "p"), + mock.call(self.verifier_uuid, "p")]) + + def test_add_verifier_ext(self): + self.verify.add_verifier_ext(self.fake_api, self.verifier_uuid, + "a", "b", "c") + self.fake_api.verifier.add_extension.assert_called_once_with( + self.verifier_uuid, source="a", version="b", extra_settings="c") + + @mock.patch("rally.cli.commands.verify.cliutils.print_list") + @mock.patch("rally.cli.commands.verify.logging.is_debug", + return_value=True) + def test_list_verifier_exts_empty_list(self, + mock_is_debug, mock_print_list): + self.fake_api.verifier.list_extensions.return_value = [] + self.verify.list_verifier_exts(self.fake_api, self.verifier_uuid) + + self.verify.list_verifier_exts(self.fake_api, self.verifier_uuid) + + self.fake_api.verifier.list_extensions.assert_has_calls( + [mock.call(self.verifier_uuid), mock.call(self.verifier_uuid)]) + + @mock.patch("rally.cli.commands.verify.cliutils.print_list") + @mock.patch("rally.cli.commands.verify.logging.is_debug", + return_value=False) + def test_list_verifier_exts(self, mock_is_debug, mock_print_list): + ver_exts = self.fake_api.verifier.list_extensions + ver_exts.return_value = [mock.MagicMock()] + fields = ["Name", "Entry point"] + self.verify.list_verifier_exts(self.fake_api, self.verifier_uuid) + + self.verify.list_verifier_exts(self.fake_api, self.verifier_uuid) + + self.fake_api.verifier.list_extensions.assert_has_calls( + [mock.call(self.verifier_uuid), mock.call(self.verifier_uuid)]) + + mock_print_list.assert_called_with(ver_exts.return_value, + fields, + normalize_field_names=True) + + @mock.patch("rally.cli.commands.verify.cliutils.print_list") + @mock.patch("rally.cli.commands.verify.logging.is_debug", + return_value=True) + def test_list_verifier_exts_with_logging(self, + mock_is_debug, mock_print_list): + ver_exts = self.fake_api.verifier.list_extensions + ver_exts.return_value = [mock.MagicMock()] + fields = ["Name", "Entry point", "Location"] + self.verify.list_verifier_exts(self.fake_api, self.verifier_uuid) + + self.verify.list_verifier_exts(self.fake_api, self.verifier_uuid) + + self.fake_api.verifier.list_extensions.assert_has_calls( + [mock.call(self.verifier_uuid), mock.call(self.verifier_uuid)]) + + mock_print_list.assert_called_with(ver_exts.return_value, + fields, + normalize_field_names=True) + + def test_delete_verifier_ext(self): + self.verify.delete_verifier_ext(self.fake_api, self.verifier_uuid, + "ext_name") + self.fake_api.verifier.delete_extension.assert_called_once_with( + self.verifier_uuid, "ext_name") + + @mock.patch("rally.cli.commands.verify.fileutils.update_globals_file") + @mock.patch("rally.cli.commands.verify.os.path.exists") + def test_start(self, mock_exists, mock_update_globals_file): + self.verify.start(self.fake_api, self.verifier_uuid, + self.deployment_uuid, pattern="pattern", + load_list="load-list") + self.assertFalse(self.fake_api.verification.start.called) + + verification = self.verification_data + self.fake_api.verification.start.return_value = { + "verification": verification, + "totals": self.results_data["totals"], + "tests": self.results_data["tests"]} + self.fake_api.verification.get.return_value = verification + + mock_exists.return_value = False + self.verify.start(self.fake_api, self.verifier_uuid, + self.deployment_uuid, load_list="/p/a/t/h") + self.assertFalse(self.fake_api.verification.start.called) + + mock_exists.return_value = True + tf = tempfile.NamedTemporaryFile() + with open(tf.name, "w") as f: + f.write("test_1\ntest_2") + self.verify.start(self.fake_api, self.verifier_uuid, + self.deployment_uuid, tags=["foo"], + load_list=tf.name) + self.fake_api.verification.start.assert_called_once_with( + self.verifier_uuid, self.deployment_uuid, tags=["foo"], + load_list=["test_1", "test_2"]) + + mock_exists.return_value = False + self.fake_api.verification.start.reset_mock() + + self.verify.start(self.fake_api, self.verifier_uuid, + self.verifier_uuid, skip_list="/p/a/t/h") + self.assertFalse(self.fake_api.verification.start.called) + + tf = tempfile.NamedTemporaryFile() + with open(tf.name, "w") as f: + f.write("test_1:\ntest_2: Reason\n") + mock_exists.return_value = True + self.verify.start(self.fake_api, self.verifier_uuid, + self.deployment_uuid, skip_list=tf.name) + self.fake_api.verification.start.assert_called_once_with( + self.verifier_uuid, self.deployment_uuid, tags=None, + skip_list={"test_1": None, "test_2": "Reason"}) + + mock_exists.return_value = False + self.fake_api.verification.start.reset_mock() + self.verify.start(self.fake_api, self.verifier_uuid, + self.deployment_uuid, xfail_list="/p/a/t/h") + self.assertFalse(self.fake_api.verification.start.called) + + tf = tempfile.NamedTemporaryFile() + with open(tf.name, "w") as f: + f.write("test_1:\ntest_2: Reason\n") + mock_exists.return_value = True + self.verify.start(self.fake_api, self.verifier_uuid, + self.deployment_uuid, xfail_list=tf.name) + self.fake_api.verification.start.assert_called_once_with( + self.verifier_uuid, self.deployment_uuid, tags=None, + xfail_list={"test_1": None, "test_2": "Reason"}) + + self.fake_api.verification.get.assert_called_with( + self.verification_uuid) + mock_update_globals_file.assert_called_with( + envutils.ENV_VERIFICATION, self.verification_uuid) + + self.fake_api.verification.get.reset_mock() + mock_update_globals_file.reset_mock() + self.verify.start(self.fake_api, self.verifier_uuid, + self.deployment_uuid, detailed=True, + do_use=False) + self.assertFalse(self.fake_api.verification.get.called) + self.assertFalse(mock_update_globals_file.called) + + @mock.patch("rally.cli.commands.verify.os.path.exists") + @mock.patch("rally.cli.commands.verify.fileutils.update_globals_file") + def test_start_on_unfinished_deployment(self, mock_update_globals_file, + mock_exists): + deployment_id = self.deployment_uuid + deployment_name = self.deployment_name + exc = exceptions.DeploymentNotFinishedStatus( + name=deployment_name, + uuid=deployment_id, + status=consts.DeployStatus.DEPLOY_INIT) + self.fake_api.verification.start.side_effect = exc + self.assertEqual( + 1, self.verify.start(self.fake_api, + self.deployment_uuid, deployment_id)) + + @mock.patch("rally.cli.commands.verify.fileutils.update_globals_file") + def test_use(self, mock_update_globals_file): + self.fake_api.verification.get.return_value = self.verification_data + self.verify.use(self.fake_api, self.verification_uuid) + self.fake_api.verification.get.assert_called_once_with( + self.verification_uuid) + mock_update_globals_file.assert_called_once_with( + envutils.ENV_VERIFICATION, self.verification_uuid) + + @mock.patch("rally.cli.commands.verify.fileutils.update_globals_file") + def test_rerun(self, mock_update_globals_file): + self.fake_api.verification.rerun.return_value = { + "verification": self.verification_data, + "totals": self.results_data["totals"], + "tests": self.results_data["tests"]} + self.fake_api.verification.get.return_value = self.verification_data + + self.verify.rerun(self.fake_api, self.verification_uuid, + self.deployment_uuid, failed=True) + self.fake_api.verification.rerun.assert_called_once_with( + self.verification_uuid, concur=None, deployment="some-deploy-uuid", + failed=True, tags=None) + mock_update_globals_file.assert_called_once_with( + envutils.ENV_VERIFICATION, self.verification_uuid) + + def test_show(self): + + verification = self.verification_data + self.fake_api.verifier.get.return_value = self.verifier_data + self.fake_api.verification.get.return_value = verification + self.fake_api.deployment.get.return_value = { + "name": self.deployment_name, "uuid": self.deployment_uuid} + + # It is a hard task to mock default value of function argument, so we + # need to apply this workaround + original_print_dict = cliutils.print_dict + print_dict_calls = [] + + def print_dict(*args, **kwargs): + print_dict_calls.append(six.StringIO()) + kwargs["out"] = print_dict_calls[-1] + original_print_dict(*args, **kwargs) + + with mock.patch.object(verify.cliutils, "print_dict", + new=print_dict): + self.verify.show(self.fake_api, self.verifier_uuid, detailed=True) self.assertEqual(1, len(print_dict_calls)) @@ -475,21 +562,98 @@ class VerifyCommandsTestCase(test.TestCase): "+---------------------+------------------------------------------" "--------------------+\n", print_dict_calls[0].getvalue()) - self.fake_api.verification.get.assert_called_once_with("v_uuid") + self.fake_api.verification.get.assert_called_once_with( + self.verifier_uuid) + + with mock.patch.object(verify.cliutils, "print_dict", + new=print_dict): + self.verify.show(self.fake_api, self.verifier_uuid, detailed=False) + self.assertEqual(2, len(print_dict_calls)) + + self.assertEqual("+---------------------------------------------------" + "--------------------------------------+\n" + "| Verification " + " |\n" + "+---------------------+-----------------------------" + "--------------------------------------+\n" + "| UUID | uuuiiiiddd " + " |\n" + "| Status | success " + " |\n" + "| Started at | 2016-01-01 17:00:03 " + " |\n" + "| Finished at | 2016-01-01 17:01:05 " + " |\n" + "| Duration | 0:01:02 " + " |\n" + "| Run arguments | concurrency: 3 " + " |\n" + "| | load_list: (value is too lon" + "g, use 'detailed' flag to display it) |\n" + "| | skip_list: (value is too lon" + "g, use 'detailed' flag to display it) |\n" + "| Tags | bar, foo " + " |\n" + "| Verifier name | My Verifier (UUID: my-verifi" + "er-uuid) |\n" + "| Verifier type | OldSchoolTestTool (namespace" + ": OpenStack) |\n" + "| Deployment name | Some Deploy (UUID: some-depl" + "oy-uuid) |\n" + "| Tests count | 2 " + " |\n" + "| Tests duration, sec | 4 " + " |\n" + "| Success | 1 " + " |\n" + "| Skipped | 0 " + " |\n" + "| Expected failures | 0 " + " |\n" + "| Unexpected success | 0 " + " |\n" + "| Failures | 1 " + " |\n" + "+---------------------+-----------------------------" + "--------------------------------------+\n", + print_dict_calls[1].getvalue()) + + self.fake_api.verification.get.assert_called_with( + self.verifier_uuid) @mock.patch("rally.cli.commands.verify.cliutils.print_list") - def test_list(self, mock_print_list): - self.verify.list(self.fake_api, "v_id", "d_id") - + def test_list_empty_verifications(self, mock_print_list): self.fake_api.verification.list.return_value = [] - self.verify.list(self.fake_api, "v_id", "d_id", "foo", "bar") + self.verify.list(self.fake_api, self.verifier_uuid, + self.deployment_uuid) + + self.verify.list(self.fake_api, self.verifier_uuid, + self.deployment_uuid, "foo", "bar") self.verify.list(self.fake_api) self.fake_api.verification.list.assert_has_calls( - [mock.call("v_id", "d_id", None, None), - mock.call("v_id", "d_id", "foo", "bar"), + [mock.call(self.verifier_uuid, self.deployment_uuid, None, None), + mock.call(self.verifier_uuid, self.deployment_uuid, "foo", "bar"), mock.call(None, None, None, None)]) + @mock.patch("rally.cli.commands.verify.cliutils.print_list") + def test_list(self, mock_print_list): + self.fake_api.verification.list.return_value = [self.verification_data] + self.verify.list(self.fake_api, self.verifier_uuid, + self.deployment_uuid) + + additional_fields = ["UUID", "Tags", "Verifier name", + "Deployment name", "Started at", "Finished at", + "Duration", "Status"] + additional_keys = ["normalize_field_names", "sortby_index", + "formatters"] + # astarove: Should be replaced on mock_print_list.assert_called_once()) + self.assertTrue(1, len(mock_print_list.call_args())) + self.assertEqual(([self.verification_data], additional_fields), + mock_print_list.call_args[0]) + self.assertEqual(additional_keys.sort(), + list(mock_print_list.call_args[1].keys()).sort()) + def test_delete(self): self.verify.delete(self.fake_api, "v_uuid") self.fake_api.verification.delete.assert_called_once_with("v_uuid") @@ -509,10 +673,11 @@ class VerifyCommandsTestCase(test.TestCase): "files": {output_dest: content}, "open": output_dest} mock_os.path.exists.return_value = False - self.verify.report(self.fake_api, "v_uuid", output_type=output_type, + self.verify.report(self.fake_api, self.verifier_uuid, + output_type=output_type, output_dest=output_dest, open_it=True) self.fake_api.verification.report.assert_called_once_with( - ["v_uuid"], output_type, output_dest) + [self.verifier_uuid], output_type, output_dest) mock_open.assert_called_once_with(mock_os.path.abspath.return_value, "w") mock_os.makedirs.assert_called_once_with( @@ -526,7 +691,8 @@ class VerifyCommandsTestCase(test.TestCase): self.fake_api.verification.report.return_value = { "files": {output_dest: content}, "print": "foo"} - self.verify.report(self.fake_api, "v_uuid", output_type=output_type, + self.verify.report(self.fake_api, self.verifier_uuid, + output_type=output_type, output_dest=output_dest) self.assertFalse(mock_open_new_tab.called) @@ -537,29 +703,25 @@ class VerifyCommandsTestCase(test.TestCase): @mock.patch("rally.cli.commands.verify.os.path.exists") def test_import_results(self, mock_exists, mock_open, mock_use): mock_exists.return_value = False - self.verify.import_results(self.fake_api, "v_id", "d_id") + self.verify.import_results(self.fake_api, self.verifier_uuid, + self.deployment_uuid) self.assertFalse(self.fake_api.verification.import_results.called) - verification = mock.Mock(uuid="verification_uuid") - results = mock.Mock(totals={"tests_count": 2, - "tests_duration": 4, - "success": 2, - "skipped": 0, - "expected_failures": 0, - "unexpected_success": 0, - "failures": 0}) + verification = self.verification_data + results = self.results_data self.fake_api.verification.import_results.return_value = ( verification, results) mock_exists.return_value = True mock_open.return_value = mock.mock_open(read_data="data").return_value - self.verify.import_results(self.fake_api, "v_id", "d_id", + self.verify.import_results(self.fake_api, self.verifier_uuid, + self.deployment_uuid, file_to_parse="/p/a/t/h") mock_open.assert_called_once_with("/p/a/t/h", "r") self.fake_api.verification.import_results.assert_called_once_with( - "v_id", "d_id", "data") + self.verifier_uuid, self.deployment_uuid, "data") - mock_use.assert_called_with("verification_uuid") + mock_use.assert_called_with(self.fake_api, self.verification_uuid) mock_use.reset_mock() self.verify.import_results(self.fake_api, "v_id", "d_id", do_use=False) diff --git a/tests/unit/common/objects/test_verification.py b/tests/unit/common/objects/test_verification.py index 494a9691d3..849188301d 100644 --- a/tests/unit/common/objects/test_verification.py +++ b/tests/unit/common/objects/test_verification.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +import datetime as dt + import mock from rally.common import objects @@ -25,6 +27,7 @@ class VerificationTestCase(test.TestCase): super(VerificationTestCase, self).setUp() self.db_obj = {"uuid": "uuid-1"} + self._db_entry = {} @mock.patch("rally.common.objects.verification.db.verification_create") def test_init(self, mock_verification_create): @@ -33,6 +36,33 @@ class VerificationTestCase(test.TestCase): self.assertEqual(self.db_obj["uuid"], v.uuid) self.assertEqual(self.db_obj["uuid"], v["uuid"]) + def test_to_dict(self): + TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z" + data = {"created_at": dt.date(2017, 2, 3), + "updated_at": dt.date(2017, 3, 3), + "id": "v_id", + "deployment_uuid": "d_uuid", + "uuid": "v_uuid", + "verifier_uuid": "v_uuid", + "unexpected_success": "2", + "status": "False", + "tests": {"test1": "tdata1", + "test2": "tdata2"}, + "skipped": 2, + "tests_duration": "", + "tags": None, + "run_args": "args", + "success": 0, + "expected_failures": 2, + "tests_count": 3, + "failures": 2} + verification = objects.Verification("verification_id") + verification._db_entry = data + result = objects.Verification.to_dict(verification) + data["created_at"] = data["created_at"].strftime(TIME_FORMAT) + data["updated_at"] = data["updated_at"].strftime(TIME_FORMAT) + self.assertEqual(data, result) + @mock.patch("rally.common.objects.verification.db.verification_create") def test_create(self, mock_verification_create): objects.Verification.create("some-verifier", "some-deployment", [], {}) diff --git a/tests/unit/common/objects/test_verifier.py b/tests/unit/common/objects/test_verifier.py index aee3cc8da8..2e4a4e2a1a 100644 --- a/tests/unit/common/objects/test_verifier.py +++ b/tests/unit/common/objects/test_verifier.py @@ -26,6 +26,17 @@ class VerifierTestCase(test.TestCase): self.db_obj = {"uuid": "uuid-1"} + def test___str__(self): + v = objects.Verifier(self.db_obj) + self.db_obj["name"] = "name" + self.db_obj["uuid"] = "uuid" + self.assertEqual("'name' (UUID=uuid)", "%s" % v) + + def test_to_dict(self): + v = objects.Verifier(self.db_obj) + v._db_entry = mock.Mock() + self.assertIsInstance(v.to_dict(), dict) + @mock.patch("rally.common.objects.verifier.db.verifier_create") def test_init(self, mock_verifier_create): v = objects.Verifier(self.db_obj) diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 82f62b6a8c..b4e7795bb1 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -682,7 +682,7 @@ class VerifierAPITestCase(test.TestCase): def test_get(self, mock_verifier_get): uuid = "some" - self.assertEqual(mock_verifier_get.return_value, + self.assertEqual(mock_verifier_get.return_value.to_dict(), api._Verifier.get(uuid)) mock_verifier_get.assert_called_once_with(uuid) @@ -690,18 +690,20 @@ class VerifierAPITestCase(test.TestCase): @mock.patch("rally.api.objects.Verifier.list") def test_list(self, mock_verifier_list): status = "some_special_status" + mock_verifier_list.return_value = [mock.Mock()] - self.assertEqual(mock_verifier_list.return_value, - api._Verifier.list(status)) + self.assertEqual( + [i.to_dict() for i in mock_verifier_list.return_value], + api._Verifier.list(status)) mock_verifier_list.assert_called_once_with(status) @mock.patch("rally.api.objects.Verifier.create") - @mock.patch("rally.api._Verifier.get") + @mock.patch("rally.api._Verifier._get") @mock.patch("rally.api.vmanager.VerifierManager.get") - def test_create(self, mock_verifier_manager_get, mock___verifier_get, + def test_create(self, mock_verifier_manager_get, mock___verifier__get, mock_verifier_create): - mock___verifier_get.side_effect = exceptions.ResourceNotFound(id="1") + mock___verifier__get.side_effect = exceptions.ResourceNotFound(id="1") name = "SomeVerifier" vtype = "fake_verifier" @@ -720,7 +722,7 @@ class VerifierAPITestCase(test.TestCase): mock_verifier_manager_get.assert_called_once_with(vtype, namespace=None) - mock___verifier_get.assert_called_once_with(name) + mock___verifier__get.assert_called_once_with(name) mock_verifier_create.assert_called_once_with( name=name, source=None, system_wide=system_wide, version=version, vtype=vtype, namespace=None, extra_settings=extra_settings) @@ -734,10 +736,10 @@ class VerifierAPITestCase(test.TestCase): verifier_obj.manager.install.assert_called_once_with() @mock.patch("rally.api.objects.Verifier.create") - @mock.patch("rally.api._Verifier.get") + @mock.patch("rally.api._Verifier._get") @mock.patch("rally.api.vmanager.VerifierManager.get") def test_create_fails_on_existing_verifier( - self, mock_verifier_manager_get, mock___verifier_get, + self, mock_verifier_manager_get, mock___verifier__get, mock_verifier_create): name = "SomeVerifier" vtype = "fake_verifier" @@ -755,16 +757,16 @@ class VerifierAPITestCase(test.TestCase): mock_verifier_manager_get.assert_called_once_with(vtype, namespace=namespace) - mock___verifier_get.assert_called_once_with(name) + mock___verifier__get.assert_called_once_with(name) self.assertFalse(mock_verifier_create.called) @mock.patch("rally.api.objects.Verifier.create") - @mock.patch("rally.api._Verifier.get") + @mock.patch("rally.api._Verifier._get") @mock.patch("rally.api.vmanager.VerifierManager.get") def test_create_fails_on_install_step( - self, mock_verifier_manager_get, mock___verifier_get, + self, mock_verifier_manager_get, mock___verifier__get, mock_verifier_create): - mock___verifier_get.side_effect = exceptions.ResourceNotFound(id="1") + mock___verifier__get.side_effect = exceptions.ResourceNotFound(id="1") verifier_obj = mock_verifier_create.return_value verifier_obj.manager.install.side_effect = RuntimeError @@ -784,7 +786,7 @@ class VerifierAPITestCase(test.TestCase): mock_verifier_manager_get.assert_called_once_with(vtype, namespace=namespace) - mock___verifier_get.assert_called_once_with(name) + mock___verifier__get.assert_called_once_with(name) mock_verifier_create.assert_called_once_with( name=name, source=source, system_wide=system_wide, version=version, vtype=vtype, namespace=namespace, extra_settings=extra_settings) @@ -796,12 +798,12 @@ class VerifierAPITestCase(test.TestCase): @mock.patch("rally.api.objects.Verifier.delete") @mock.patch("rally.api._Verification.list") - @mock.patch("rally.api._Verifier.get") - def test_delete_no_verifications(self, mock___verifier_get, + @mock.patch("rally.api._Verifier._get") + def test_delete_no_verifications(self, mock___verifier__get, mock___verification_list, mock_verifier_delete): mock___verification_list.return_value = [] - verifier_obj = mock___verifier_get.return_value + verifier_obj = mock___verifier__get.return_value verifier_id = "uuuiiiddd" deployment_id = "deployment" @@ -830,13 +832,12 @@ class VerifierAPITestCase(test.TestCase): @mock.patch("rally.api.objects.Verifier.delete") @mock.patch("rally.api._Verification.delete") @mock.patch("rally.api._Verification.list") - @mock.patch("rally.api._Verifier.get") + @mock.patch("rally.api._Verifier._get") def test_delete_with_verifications( - self, mock___verifier_get, mock___verification_list, + self, mock___verifier__get, mock___verification_list, mock___verification_delete, mock_verifier_delete): - verifications = [mock.Mock(), mock.Mock()] + verifications = [{"uuid": "uuid_1"}, {"uuid": "uuid_2"}] mock___verification_list.return_value = verifications - verifier_id = "uuuiiiddd" self.assertRaises(exceptions.RallyException, api._Verifier.delete, @@ -848,13 +849,13 @@ class VerifierAPITestCase(test.TestCase): api._Verifier.delete(verifier_id, force=True) mock___verification_list.assert_called_once_with(verifier_id, None) - self.assertEqual([mock.call(v.uuid) for v in verifications], + self.assertEqual([mock.call(v["uuid"]) for v in verifications], mock___verification_delete.call_args_list) @mock.patch("rally.api.utils.BackupHelper") - @mock.patch("rally.api._Verifier.get") - def test_update_failed(self, mock___verifier_get, mock_backup_helper): - verifier_obj = mock___verifier_get.return_value + @mock.patch("rally.api._Verifier._get") + def test_update_failed(self, mock___verifier__get, mock_backup_helper): + verifier_obj = mock___verifier__get.return_value verifier_obj.system_wide = False uuid = "uuuuiiiidddd" @@ -882,9 +883,9 @@ class VerifierAPITestCase(test.TestCase): self.assertIn(msg, "%s" % e) @mock.patch("rally.api.utils.BackupHelper") - @mock.patch("rally.api._Verifier.get") - def test_update(self, mock___verifier_get, mock_backup_helper): - verifier_obj = mock___verifier_get.return_value + @mock.patch("rally.api._Verifier._get") + def test_update(self, mock___verifier__get, mock_backup_helper): + verifier_obj = mock___verifier__get.return_value verifier_obj.system_wide = False verifier_obj.status = consts.VerifierStatus.INSTALLED uuid = "uuuuiiiidddd" @@ -964,9 +965,18 @@ class VerifierAPITestCase(test.TestCase): verifier_obj.update_properties.assert_called_once_with( status=verifier_obj.status, system_wide=True) - @mock.patch("rally.api._Verifier.get") - def test_configure_with_wrong_state_of_verifier(self, mock___verifier_get): - verifier_obj = mock___verifier_get.return_value + # check switching from system-wide to system-wide + verifier_obj.system_wide = True + expected_calls = len(verifier_obj.update_status.call_args()) + api._Verifier.update(uuid, system_wide=True) + self.assertTrue(expected_calls, + len(verifier_obj.update_status.call_args())) + self.assertFalse(verifier_obj.manager.install_venv.called) + + @mock.patch("rally.api._Verifier._get") + def test_configure_with_wrong_state_of_verifier(self, + mock___verifier__get): + verifier_obj = mock___verifier__get.return_value verifier_id = "uuiiiidd" deployment_id = "deployment" for status in consts.VerifierStatus: @@ -978,12 +988,16 @@ class VerifierAPITestCase(test.TestCase): self.assertIn("because verifier is in '%s' status" % status, "%s" % e) + @mock.patch("rally.cli.commands.verify.logging.is_debug", + return_value=False) @mock.patch("rally.plugins.openstack.verification.tempest.manager." "os.path.exists") - @mock.patch("rally.api._Verifier.get") - def test_configure_when_it_is_already_configured(self, mock___verifier_get, - mock_exists): - verifier_obj = mock___verifier_get.return_value + @mock.patch("rally.api._Verifier._get") + def test_configure_when_it_is_already_configured(self, + mock___verifier__get, + mock_exists, + mock_is_debug): + verifier_obj = mock___verifier__get.return_value verifier_id = "uuiiiidd" deployment_id = "deployment" extra = {"key": "value"} @@ -1018,10 +1032,58 @@ class VerifierAPITestCase(test.TestCase): verifier_obj.manager.configure.asset_called_once_with( extra_options=extra) - @mock.patch("rally.api._Verifier.get") - def test_override_config_with_wrong_state_of_verifier(self, - mock___verifier_get): - verifier_obj = mock___verifier_get.return_value + verifier_obj.update_status.reset_mock() + verifier_obj.manager.extend_configuration.reset_mock() + + @mock.patch("rally.cli.commands.verify.logging.is_debug", + return_value=True) + @mock.patch("rally.plugins.openstack.verification.tempest.manager." + "os.path.exists") + @mock.patch("rally.api._Verifier._get") + def test_configure_when_it_is_already_configured_with_logging( + self, mock___verifier__get, mock_exists, mock_is_debug): + verifier_obj = mock___verifier__get.return_value + verifier_id = "uuiiiidd" + deployment_id = "deployment" + extra = {"key": "value"} + verifier_obj.status = consts.VerifierStatus.INSTALLED + + # no recreate and no extra options + self.assertEqual(verifier_obj.manager.get_configuration.return_value, + api._Verifier.configure(verifier_id, deployment_id, + reconfigure=False)) + self.assertFalse(verifier_obj.manager.extend_configuration.called) + self.assertFalse(verifier_obj.manager.configure.called) + self.assertFalse(verifier_obj.update_status.called) + + # no recreate, just extend existing configuration + self.assertEqual(verifier_obj.manager.get_configuration.return_value, + api._Verifier.configure(verifier_id, deployment_id, + reconfigure=False, + extra_options=extra)) + verifier_obj.manager.extend_configuration.assert_called_once_with( + extra) + self.assertFalse(verifier_obj.manager.configure.called) + + verifier_obj.update_status.reset_mock() + verifier_obj.manager.extend_configuration.reset_mock() + + # recreate with extra options + self.assertEqual(verifier_obj.manager.configure.return_value, + api._Verifier.configure(verifier_id, deployment_id, + reconfigure=True, + extra_options=extra)) + self.assertFalse(verifier_obj.manager.extend_configuration.called) + verifier_obj.manager.configure.asset_called_once_with( + extra_options=extra) + + verifier_obj.update_status.reset_mock() + verifier_obj.manager.extend_configuration.reset_mock() + + @mock.patch("rally.api._Verifier._get") + def test_override_config_with_wrong_state_of_verifier( + self, mock___verifier__get): + verifier_obj = mock___verifier__get.return_value verifier_id = "uuiiiidd" deployment_id = "deployment" new_content = {} @@ -1036,10 +1098,10 @@ class VerifierAPITestCase(test.TestCase): @mock.patch("rally.plugins.openstack.verification.tempest.manager." "os.path.exists") - @mock.patch("rally.api._Verifier.get") + @mock.patch("rally.api._Verifier._get") def test_override_config_when_it_is_already_configured( - self, mock___verifier_get, mock_exists): - verifier_obj = mock___verifier_get.return_value + self, mock___verifier__get, mock_exists): + verifier_obj = mock___verifier__get.return_value verifier_id = "uuiiiidd" deployment_id = "deployment" new_config = {"key": "value"} @@ -1050,9 +1112,9 @@ class VerifierAPITestCase(test.TestCase): verifier_obj.manager.override_configuration.assert_called_once_with( new_config) - @mock.patch("rally.api._Verifier.get") - def test_list_tests(self, mock___verifier_get): - verifier_obj = mock___verifier_get.return_value + @mock.patch("rally.api._Verifier._get") + def test_list_tests(self, mock___verifier__get): + verifier_obj = mock___verifier__get.return_value verifier_id = "uuiiiidd" pattern = "some" verifier_obj.status = consts.VerifierStatus.INIT @@ -1069,9 +1131,9 @@ class VerifierAPITestCase(test.TestCase): api._Verifier.list_tests(verifier_id, pattern)) verifier_obj.manager.list_tests.assert_called_once_with(pattern) - @mock.patch("rally.api._Verifier.get") - def test_add_extension(self, mock___verifier_get): - verifier_obj = mock___verifier_get.return_value + @mock.patch("rally.api._Verifier._get") + def test_add_extension(self, mock___verifier__get): + verifier_obj = mock___verifier__get.return_value verifier_id = "uuiiiidd" source = "example.com" version = 3.14159 @@ -1107,9 +1169,9 @@ class VerifierAPITestCase(test.TestCase): mock.call(verifier_obj.status)], verifier_obj.update_status.call_args_list) - @mock.patch("rally.api._Verifier.get") - def test_list_extensions(self, mock___verifier_get): - verifier_obj = mock___verifier_get.return_value + @mock.patch("rally.api._Verifier._get") + def test_list_extensions(self, mock___verifier__get): + verifier_obj = mock___verifier__get.return_value verifier_id = "uuiiiidd" for status in consts.VerifierStatus: @@ -1127,9 +1189,9 @@ class VerifierAPITestCase(test.TestCase): api._Verifier.list_extensions(verifier_id)) verifier_obj.manager.list_extensions.assert_called_once_with() - @mock.patch("rally.api._Verifier.get") - def test_delete_extension(self, mock___verifier_get): - verifier_obj = mock___verifier_get.return_value + @mock.patch("rally.api._Verifier._get") + def test_delete_extension(self, mock___verifier__get): + verifier_obj = mock___verifier__get.return_value verifier_id = "uuiiiidd" name = "some" @@ -1150,10 +1212,27 @@ class VerifierAPITestCase(test.TestCase): class VerificationAPITestCase(test.TestCase): + results_data = { + "totals": {"tests_count": 2, + "tests_duration": 4, + "success": 2, + "skipped": 0, + "expected_failures": 0, + "unexpected_success": 0, + "failures": 0}, + "tests": { + "test_1": { + "name": "test_1", + "status": "success", + "duration": 2, + "tags": []} + } + } + @mock.patch("rally.api.objects.Verification.get") def test_get(self, mock_verification_get): verification_uuid = "uuiiiidd" - self.assertEqual(mock_verification_get.return_value, + self.assertEqual(mock_verification_get.return_value.to_dict(), api._Verification.get(verification_uuid)) mock_verification_get.assert_called_once_with(verification_uuid) @@ -1171,10 +1250,11 @@ class VerificationAPITestCase(test.TestCase): tags = ["foo", "bar"] status = "some_status" - self.assertEqual(mock_verification_list.return_value, - api._Verification.list(verifier_id, - deployment_id=deployment_id, - tags=tags, status=status)) + mock_verification_list.return_value = [mock.Mock()] + self.assertEqual( + [i.to_dict() for i in mock_verification_list.return_value], + api._Verification.list(verifier_id, deployment_id=deployment_id, + tags=tags, status=status)) mock_verification_list.assert_called_once_with( verifier_id, deployment_id=deployment_id, tags=tags, status=status) @@ -1203,54 +1283,56 @@ class VerificationAPITestCase(test.TestCase): mock_verification_get.call_args_list) @mock.patch("rally.api.objects.Verification.create") - @mock.patch("rally.api._Verifier.get") - def test_import_results(self, mock___verifier_get, + @mock.patch("rally.api._Verifier._get") + def test_import_results(self, mock___verifier__get, mock_verification_create): verifier_id = "vuuuiiddd" deployment_id = "duuuiidd" data = "contest of file with results" run_args = {"set_name": "compute"} - verifier_obj = mock___verifier_get.return_value + verifier_obj = mock___verifier__get.return_value - averification, aresults = api._Verification.import_results( + results = api._Verification.import_results( verifier_id, deployment_id=deployment_id, data=data, **run_args) - self.assertEqual(mock_verification_create.return_value, averification) - self.assertEqual(verifier_obj.manager.parse_results.return_value, - aresults) - mock___verifier_get.assert_called_once_with(verifier_id) + verification = mock_verification_create.return_value + + self.assertEqual(verification.to_dict(), results["verification"]) + self.assertEqual( + verifier_obj.manager.parse_results.return_value.totals, + results["totals"]) + mock___verifier__get.assert_called_once_with(verifier_id) verifier_obj.set_deployment.assert_called_once_with(deployment_id) verifier_obj.manager.validate_args.assert_called_once_with(run_args) mock_verification_create.assert_called_once_with( verifier_id, deployment_id=deployment_id, run_args=run_args) - averification.update_status.assert_called_once_with( + verification.update_status.assert_called_once_with( consts.VerificationStatus.RUNNING) verifier_obj.manager.parse_results.assert_called_once_with(data) - averification.finish.assert_called_once_with(aresults.totals, - aresults.tests) + verification.finish.assert_called_once_with(results["totals"], + results["tests"]) # check setting failed - self.assertFalse(averification.set_failed.called) - averification.finish.reset_mock() + verification.finish.reset_mock() verifier_obj.manager.parse_results.side_effect = RuntimeError self.assertRaises(RuntimeError, api._Verification.import_results, verifier_id, deployment_id=deployment_id, data=data, **run_args) - self.assertFalse(averification.finish.called) - self.assertTrue(averification.set_failed.called) + self.assertFalse(verification.finish.called) + self.assertTrue(verification.set_failed.called) - @mock.patch("rally.api._Verifier.get") + @mock.patch("rally.api._Verifier._get") @mock.patch("rally.api.objects.Deployment.get", return_value=fakes.FakeDeployment( uuid="deployment_uuid", status=consts.DeployStatus.DEPLOY_FINISHED)) def test_start_failed_due_to_wrong_status_of_verifier( - self, mock_deployment_get, mock___verifier_get): + self, mock_deployment_get, mock___verifier__get): verifier_id = "vuuuiiddd" deployment_id = "duuuiidd" - verifier_obj = mock___verifier_get.return_value + verifier_obj = mock___verifier__get.return_value for status in consts.VerifierStatus: if status != consts.VerifierStatus.INSTALLED: @@ -1265,17 +1347,17 @@ class VerificationAPITestCase(test.TestCase): @mock.patch("rally.api.objects.Verification.create") @mock.patch("rally.api._Verifier.configure") - @mock.patch("rally.api._Verifier.get") + @mock.patch("rally.api._Verifier._get") @mock.patch("rally.api.objects.Deployment.get", return_value=fakes.FakeDeployment( uuid="deployment_uuid", status=consts.DeployStatus.DEPLOY_FINISHED)) def test_start_with_configuring(self, mock_deployment_get, - mock___verifier_get, mock_configure, + mock___verifier__get, mock_configure, mock_verification_create): verifier_id = "vuuuiiddd" deployment_id = "duuuiidd" - verifier_obj = mock___verifier_get.return_value + verifier_obj = mock___verifier__get.return_value verifier_obj.status = consts.VerifierStatus.INSTALLED verifier_obj.manager.is_configured.return_value = False @@ -1286,18 +1368,18 @@ class VerificationAPITestCase(test.TestCase): @mock.patch("rally.api.objects.Verification.create") @mock.patch("rally.api._Verifier.configure") - @mock.patch("rally.api._Verifier.get") + @mock.patch("rally.api._Verifier._get") @mock.patch("rally.api.objects.Deployment.get", return_value=fakes.FakeDeployment( uuid="deployment_uuid", status=consts.DeployStatus.DEPLOY_FINISHED)) - def test_start(self, mock_deployment_get, mock___verifier_get, + def test_start(self, mock_deployment_get, mock___verifier__get, mock_configure, mock_verification_create): verifier_id = "vuuuiiddd" deployment_id = "duuuiidd" tags = ["foo", "bar"] run_args = {"arg": "value"} - verifier_obj = mock___verifier_get.return_value + verifier_obj = mock___verifier__get.return_value verifier_obj.status = consts.VerifierStatus.INSTALLED verification_obj = mock_verification_create.return_value @@ -1341,19 +1423,19 @@ class VerificationAPITestCase(test.TestCase): @mock.patch("rally.api.objects.Verification.create") @mock.patch("rally.api._Verifier.configure") - @mock.patch("rally.api._Verifier.get") + @mock.patch("rally.api._Verifier._get") @mock.patch("rally.api.objects.Deployment.get", return_value=fakes.FakeDeployment( uuid="deployment_uuid", status=consts.DeployStatus.DEPLOY_FINISHED)) def test_start_failed_to_run(self, mock_deployment_get, - mock___verifier_get, mock_configure, + mock___verifier__get, mock_configure, mock_verification_create): verifier_id = "vuuuiiddd" deployment_id = "duuuiidd" tags = ["foo", "bar"] run_args = {"arg": "value"} - verifier_obj = mock___verifier_get.return_value + verifier_obj = mock___verifier__get.return_value verifier_obj.status = consts.VerifierStatus.INSTALLED verification_obj = mock_verification_create.return_value verifier_obj.manager.run.side_effect = RuntimeError @@ -1381,48 +1463,57 @@ class VerificationAPITestCase(test.TestCase): @mock.patch("rally.api._Verification.start") @mock.patch("rally.api._Deployment.get") - @mock.patch("rally.api._Verification.get") - def test_rerun(self, mock___verification_get, mock___deployment_get, - mock___verification_start): + @mock.patch("rally.api._Verification._get") + def test_rerun(self, mock___verification__get, mock___deployment_get, + mock_start): + tests = {"test_1": {"status": "success"}, "test_2": {"status": "fail"}} - mock___verification_get.return_value = mock.Mock( + mock___verification__get.return_value = mock.Mock( uuid="uuid", verifier_uuid="v_uuid", deployment_uuid="d_uuid", tests=tests) mock___deployment_get.return_value = {"name": "d_name", "uuid": "d_uuid"} + mock_start.return_value = ( + mock___verification__get.return_value, + mock.Mock(totals=self.results_data["totals"], + tests=self.results_data["tests"])) api._Verification.rerun("uuid") - mock___verification_start.assert_called_once_with( + mock_start.assert_called_once_with( "v_uuid", "d_uuid", load_list=tests.keys(), tags=None) @mock.patch("rally.api._Verification.start") @mock.patch("rally.api._Deployment.get") - @mock.patch("rally.api._Verification.get") + @mock.patch("rally.api._Verification._get") def test_rerun_failed_tests( - self, mock___verification_get, mock___deployment_get, - mock___verification_start): + self, mock___verification__get, mock___deployment_get, + mock_start): tests = {"test_1": {"status": "success"}, "test_2": {"status": "fail"}, "test_3": {"status": "fail"}} - mock___verification_get.return_value = mock.Mock( + mock___verification__get.return_value = mock.Mock( uuid="uuid", verifier_uuid="v_uuid", deployment_uuid="d_uuid", tests=tests) mock___deployment_get.return_value = {"name": "d_name", "uuid": "d_uuid"} + mock_start.return_value = ( + mock___verification__get.return_value, + mock.Mock(totals=self.results_data["totals"], + tests=self.results_data["tests"])) api._Verification.rerun("uuid", failed=True) expected_tests = [t for t, r in tests.items() if r["status"] == "fail"] - mock___verification_start.assert_called_once_with( + mock_start.assert_called_once_with( "v_uuid", "d_uuid", load_list=expected_tests, tags=None) - @mock.patch("rally.api._Verification.get") + @mock.patch("rally.api._Verification._get") def test_rerun_failed_tests_raise_exc( - self, mock___verification_get): + self, mock___verification__get): tests = {"test_1": {"status": "success"}, "test_2": {"status": "success"}, "test_3": {"status": "skip"}} - mock___verification_get.return_value = mock.Mock( + mock___verification__get.return_value = mock.Mock( uuid="uuid", verifier_uuid="v_uuid", deployment_uuid="d_uuid", tests=tests)