diff --git a/doc/samples/plugins/context/context_plugin.py b/doc/samples/plugins/context/context_plugin.py index 6b15308736..20e587b279 100644 --- a/doc/samples/plugins/context/context_plugin.py +++ b/doc/samples/plugins/context/context_plugin.py @@ -1,4 +1,3 @@ -from oslo.config import cfg from rally.benchmark.context import base from rally import log as logging @@ -6,7 +5,6 @@ from rally import osclients from rally import utils LOG = logging.getLogger(__name__) -CONF = cfg.CONF @base.context(name="create_flavor", order=1000) @@ -56,7 +54,7 @@ class CreateFlavorContext(base.Context): LOG.debug("Flavor with id '%s'" % self.context["flavor"]["id"]) except Exception as e: msg = "Can't create flavor: %s" % e.message - if CONF.debug: + if logging.is_debug(): LOG.exception(msg) else: LOG.warning(msg) @@ -69,7 +67,7 @@ class CreateFlavorContext(base.Context): LOG.debug("Flavor '%s' deleted" % self.context["flavor"]["id"]) except Exception as e: msg = "Can't delete flavor: %s" % e.message - if CONF.debug: + if logging.is_debug(): LOG.exception(msg) else: LOG.warning(msg) diff --git a/rally/aas/rest/app.py b/rally/aas/rest/app.py index 97705b20be..011d80d963 100644 --- a/rally/aas/rest/app.py +++ b/rally/aas/rest/app.py @@ -13,11 +13,9 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.config import cfg import pecan - -CONF = cfg.CONF +from rally import log as logging def setup_app(config): @@ -29,19 +27,19 @@ def setup_app(config): :return: A normal WSGI application, an instance of :class:`pecan.Pecan`. """ - app = pecan.Pecan(config.app.root, debug=CONF.debug) + app = pecan.Pecan(config.app.root, debug=logging.is_debug()) return app def make_app(): config = { - 'app': { - 'root': 'rally.aas.rest.controllers.root.RootController', - 'modules': ['rally.aas.rest'], - 'debug': CONF.debug, + "app": { + "root": "rally.aas.rest.controllers.root.RootController", + "modules": ["rally.aas.rest"], + "debug": logging.is_debug(), }, - 'wsme': { - 'debug': CONF.debug, + "wsme": { + "debug": logging.is_debug(), }, } app = pecan.load_app(config) diff --git a/rally/benchmark/context/cleanup/manager.py b/rally/benchmark/context/cleanup/manager.py index f628a7e395..8e5c16e5cc 100644 --- a/rally/benchmark/context/cleanup/manager.py +++ b/rally/benchmark/context/cleanup/manager.py @@ -15,8 +15,6 @@ import time -from oslo.config import cfg - from rally.benchmark.context.cleanup import base from rally import broker from rally.i18n import _ @@ -25,7 +23,6 @@ from rally import osclients from rally import utils as rutils -CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -87,7 +84,7 @@ class SeekAndDestroy(object): _("Resource deletion failed, max retries exceeded for " "%(service)s.%(resource)s: %(uuid)s. Reason: %(reason)s") % msg_kw) - if CONF.debug: + if logging.is_debug(): LOG.exception(e) else: started = time.time() diff --git a/rally/benchmark/runners/base.py b/rally/benchmark/runners/base.py index a52fb80bf5..18a4fc91c5 100644 --- a/rally/benchmark/runners/base.py +++ b/rally/benchmark/runners/base.py @@ -18,7 +18,6 @@ import collections import random import jsonschema -from oslo.config import cfg from rally.benchmark.scenarios import base as scenario_base from rally.benchmark import types @@ -73,7 +72,7 @@ def _run_scenario_once(args): method_name)(**kwargs) or scenario_output except Exception as e: error = utils.format_exc(e) - if cfg.CONF.debug: + if logging.is_debug(): LOG.exception(e) finally: status = "Error %s: %s" % tuple(error[0:2]) if error else "OK" diff --git a/rally/broker.py b/rally/broker.py index 7cbba6dfd8..e28ebf285e 100644 --- a/rally/broker.py +++ b/rally/broker.py @@ -17,13 +17,10 @@ import collections import threading import time -from oslo.config import cfg - from rally.i18n import _ from rally import log as logging -CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -50,7 +47,7 @@ def _consumer(consume, queue, is_published): except Exception as e: LOG.warning(_("Failed to consume a task from the queue: " "%s") % e) - if CONF.debug: + if logging.is_debug(): LOG.exception(e) elif is_published.isSet(): break @@ -73,7 +70,7 @@ def _publisher(publish, queue, is_published): publish(queue) except Exception as e: LOG.warning(_("Failed to publish a task to the queue: %s") % e) - if CONF.debug: + if logging.is_debug(): LOG.exception(e) finally: is_published.set() diff --git a/rally/cmd/cliutils.py b/rally/cmd/cliutils.py index f2259b7952..f26ac21728 100644 --- a/rally/cmd/cliutils.py +++ b/rally/cmd/cliutils.py @@ -298,12 +298,12 @@ def run(argv, categories): ret = fn(*fn_args, **fn_kwargs) return(ret) except (IOError, TypeError, exceptions.DeploymentNotFound) as e: - if CONF.debug: + if logging.is_debug(): raise print(e) return 1 except exceptions.TaskNotFound as e: - if CONF.debug: + if logging.is_debug(): LOG.exception(e) print(e) return 1 diff --git a/rally/cmd/commands/task.py b/rally/cmd/commands/task.py index 25079c6761..cb9d0badb6 100644 --- a/rally/cmd/commands/task.py +++ b/rally/cmd/commands/task.py @@ -21,7 +21,6 @@ import os import pprint import webbrowser -from oslo.config import cfg import yaml from rally.benchmark.processing import plot @@ -33,6 +32,7 @@ from rally import consts from rally import db from rally import exceptions from rally.i18n import _ +from rally import log as logging from rally import objects from rally.openstack.common import cliutils as common_cliutils from rally.orchestrator import api @@ -205,7 +205,7 @@ class TaskCommands(object): print("-" * 80) verification = yaml.safe_load(task["verification_log"]) - if not cfg.CONF.debug: + if not logging.is_debug(): print(verification[0]) print(verification[1]) print() diff --git a/rally/log.py b/rally/log.py index e7fcbd72bc..763558e39f 100644 --- a/rally/log.py +++ b/rally/log.py @@ -65,3 +65,7 @@ class RallyContextAdapter(oslogging.ContextAdapter): def debug(self, msg, *args, **kwargs): self.log(logging.RDEBUG, msg, *args, **kwargs) + + +def is_debug(): + return CONF.debug or CONF.rally_debug diff --git a/rally/orchestrator/api.py b/rally/orchestrator/api.py index f882eb38e7..906cb6f233 100644 --- a/rally/orchestrator/api.py +++ b/rally/orchestrator/api.py @@ -14,7 +14,6 @@ # under the License. import jsonschema -from oslo.config import cfg from rally.benchmark import engine from rally import consts @@ -25,7 +24,6 @@ from rally import log as logging from rally import objects from rally.verification.verifiers.tempest import tempest -CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -39,7 +37,7 @@ def create_deploy(config, name): try: deployment = objects.Deployment(name=name, config=config) except exceptions.DeploymentNameExists as e: - if CONF.debug: + if logging.is_debug(): LOG.exception(e) raise diff --git a/rally/osclients.py b/rally/osclients.py index a8f8a6e567..2808da9444 100644 --- a/rally/osclients.py +++ b/rally/osclients.py @@ -34,6 +34,7 @@ from zaqarclient.queues import client as zaqar from rally import consts from rally import exceptions +from rally import log as logging CONF = cfg.CONF @@ -144,7 +145,7 @@ class Clients(object): region_name=self.endpoint.region_name) client = nova.Client(version, auth_token=kc.auth_token, - http_log_debug=CONF.debug, + http_log_debug=logging.is_debug(), timeout=CONF.openstack_client_http_timeout, insecure=CONF.https_insecure, cacert=CONF.https_cacert) @@ -203,7 +204,7 @@ class Clients(object): def cinder(self, version='1'): """Return cinder client.""" client = cinder.Client(version, None, None, - http_log_debug=CONF.debug, + http_log_debug=logging.is_debug(), timeout=CONF.openstack_client_http_timeout, insecure=CONF.https_insecure, cacert=CONF.https_cacert) diff --git a/rally/ui/utils.py b/rally/ui/utils.py index 02f588d34e..2d67af1e14 100644 --- a/rally/ui/utils.py +++ b/rally/ui/utils.py @@ -35,10 +35,15 @@ def get_template(template_path): def main(*args): - if len(args) != 2 or args[0] != "render": - exit("Usage: utils.py render ") + if len(args) < 2 or args[0] != "render": + exit("Usage: \n\t" + "utils.py render " + "= =\n" + "where key-1,value-1 and key-2,value-2 are key pairs of template") try: - print(get_template(sys.argv[2]).render()) + render_kwargs = dict([arg.split("=") for arg in args[2:]]) + + print(get_template(sys.argv[2]).render(**render_kwargs)) except mako.exceptions.TopLevelLookupException as e: exit(e) diff --git a/rally/utils.py b/rally/utils.py index 224145b6df..3ff85c110a 100644 --- a/rally/utils.py +++ b/rally/utils.py @@ -23,7 +23,6 @@ import StringIO import sys import time -from oslo.config import cfg from oslo.utils import importutils from sphinx.util import docstrings @@ -31,7 +30,6 @@ from rally import exceptions from rally.i18n import _ from rally import log as logging -CONF = cfg.CONF LOG = logging.getLogger(__name__) JSON_SCHEMA = 'http://json-schema.org/draft-04/schema' @@ -201,7 +199,7 @@ def load_plugins(directory): LOG.warning( "\t Failed to load module with plugins %(path)s.py: %(e)s" % {"path": fullpath, "e": e}) - if CONF.debug: + if logging.is_debug(): LOG.exception(e) diff --git a/rally/verification/verifiers/tempest/subunit2json.py b/rally/verification/verifiers/tempest/subunit2json.py index 40c9706fb2..6993950b69 100644 --- a/rally/verification/verifiers/tempest/subunit2json.py +++ b/rally/verification/verifiers/tempest/subunit2json.py @@ -45,10 +45,9 @@ class JsonOutput(testtools.TestResult): self.results_file = results_file def _format_result(self, name, time, status, output, failure=None): - if status == STATUS_SKIP: - # We do not need `setUpClass' in skipped test name - if name[:12] == "setUpClass (" and name[-1] == ")": - name = name[12:-1] + # We do not need `setUpClass' in test name + if name[:12] == "setUpClass (" and name[-1] == ")": + name = name[12:-1] self.test_cases[name] = {"name": name, "status": status, "time": time, "output": output} diff --git a/rally/verification/verifiers/tempest/tempest.py b/rally/verification/verifiers/tempest/tempest.py index 0331cd900a..993482c8d4 100644 --- a/rally/verification/verifiers/tempest/tempest.py +++ b/rally/verification/verifiers/tempest/tempest.py @@ -44,7 +44,8 @@ def check_output(*args, **kwargs): LOG.debug("error output: '%s'" % e.output) raise - LOG.debug(output) + if logging.is_debug(): + print(output) class Tempest(object): diff --git a/tests/ci/rally-gate/index_verify.mako b/tests/ci/rally-gate/index_verify.mako new file mode 100644 index 0000000000..20b3159c77 --- /dev/null +++ b/tests/ci/rally-gate/index_verify.mako @@ -0,0 +1,136 @@ +## -*- coding: utf-8 -*- +<%inherit file="/base.mako"/> + +<%block name="title_text">Rally Verification job results + +<%block name="css"> + li { margin:2px 0 } + a, a:visited { color:#039 } + code { padding:0 15px; color:#888; display: block } + .columns li { position:relative } + .columns li > :first-child { display:block } + .columns li > :nth-child(2) { display:block; position:static; left:165px; top:0; white-space:nowrap } + .fail {color: red; text-transform: uppercase} + .pass {color: green; display: none; text-transform: uppercase} + + +<%block name="css_content_wrap">margin:0 auto; padding:0 5px + +<%block name="media_queries"> + @media only screen and (min-width: 320px) { .content-wrap { width:400px } } + @media only screen and (min-width: 520px) { .content-wrap { width:500px } } + @media only screen and (min-width: 620px) { .content-wrap { width:90% } .columns li > :nth-child(2) { position:absolute } } + @media only screen and (min-width: 720px) { .content-wrap { width:70% } } + + +<%block name="header_text">Verify job results + +<%block name="content"> +

Job Logs and Job Result files

+ + +

Job Steps and Results

+

Introduction

+
    +
  • Install tempest
    • +
  • Launch two verifications ("compute" set is used)
    • +
  • List all verifications
    • +
  • Compare two verification results
    • +
+ + Each job step has output in all supported formats. + +

Details

+ [${install}] + Tempest installation + $ rally-manage tempest install + +
First verification run +
    +
  1. + [${v1}] + Launch of verification + $ rally verify start --set compute +
    2. +
  2. + [${vr_1_html}] + Display raw results in HTML + $ rally verify results --html +
    3. +
  3. + [${vr_1_json}] + Display raw results in JSON + $ rally verify results --json +
    4. +
  4. + [${vs_1}] + Display results table of the verification + $ rally verify show +
    5. +
  5. + [${vsd_1}] + Display results table of the verification with detailed errors
    + $ rally verify show --detailed or $ rally verify detailed +
    6. +
+ + + Second verification run +
    +
  1. + [${v2}] + Launch of verification + $ rally verify start --set compute +
    2. +
  2. + [${vr_2_html}] + Display results in HTML + $ rally verify results --html +
    3. +
  3. + [${vr_2_json}] + Display results in JSON + $ rally verify results --json +
    4. +
  4. + [${vs_2}] + Display table results of the verification + $ rally verify show +
    5. +
  5. + [${vsd_2}] + Display table results of the verification with detailed errors
    + $ rally verify show --detailed or $ rally verify detailed +
    6. +
+ + [${l}] + List of all verifications + $ rally verify list + + [${c_html}] + Compare two verification and display results in HTML + $ rally verify compare --uuid-1 <uuid-1> --uuid-2 <uuid-2> --html + + [${c_json}] + Compare two verifications and display results in JSON + $ rally verify compare --uuid-1 <uuid-1> --uuid-2 <uuid-2> --json + + [${c_csv}] + Compare two verifications and display results in CSV + $ rally verify compare --uuid-1 <uuid-1> --uuid-2 <uuid-2> --csv + +

About Rally

+

Rally is benchmarking and verification system for OpenStack:

+ + diff --git a/tests/ci/rally-verify.sh b/tests/ci/rally-verify.sh new file mode 100755 index 0000000000..3d067e5b68 --- /dev/null +++ b/tests/ci/rally-verify.sh @@ -0,0 +1,96 @@ +#!/bin/bash -ex +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compli$OUT with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script is executed by post_test_hook function in desvstack gate. + +RESULTS_DIR="rally-verify" + +env +set -o pipefail +set +e + +mkdir -p ${RESULTS_DIR}/extra + + +# Check deployment +rally use deployment --deployment devstack +rally deployment check + +function do_status { + if [[ ${1} != 0 ]] + then + echo "fail" + else + echo "pass" + fi +} + +declare -a RESULTS + +rally-manage --rally-debug tempest install > ${RESULTS_DIR}/tempest_installation.txt 2>&1 +RESULTS+="install=$(do_status $?) " + +gzip -9 ${RESULTS_DIR}/tempest_installation.txt + +# Run to verification for one SET_NAME and then compare them. +SET_NAME="compute" + +function do_verification { + OUTPUT_FILE=${RESULTS_DIR}/${1}_verification_${SET_NAME}_set.txt + rally --rally-debug verify start --set ${SET_NAME} > ${OUTPUT_FILE} 2>&1 + RESULTS+="v${1}=$(do_status $?) " + gzip -9 ${OUTPUT_FILE} + source ~/.rally/globals && VERIFICATIONS[${1}]=${RALLY_VERIFICATION} + + # Check different "rally verify" commands, which displays verification results + for OUTPUT_FORMAT in "html" "json" + do + OUTPUT_FILE=${RESULTS_DIR}/${1}_verify_results.${OUTPUT_FORMAT} + rally verify results --uuid ${RALLY_VERIFICATION} --${OUTPUT_FORMAT} --output-file ${OUTPUT_FILE} + RESULTS+="vr_${1}_${OUTPUT_FORMAT}=$(do_status $?) " + gzip -9 ${OUTPUT_FILE} + done + + rally verify show --uuid ${RALLY_VERIFICATION} > ${RESULTS_DIR}/${1}_verify_show.txt + RESULTS+="vs_${1}=$(do_status $?) " + gzip -9 ${RESULTS_DIR}/${1}_verify_show.txt + + rally verify show --uuid ${RALLY_VERIFICATION} --detailed > ${RESULTS_DIR}/${1}_verify_show_detailed.txt + RESULTS+="vsd_${1}=$(do_status $?) " + gzip -9 ${RESULTS_DIR}/${1}_verify_show_detailed.txt +} + +do_verification 1 +do_verification 2 + +rally verify list > ${RESULTS_DIR}/verify_list.txt +RESULTS+="l=$(do_status $?) " +gzip -9 ${RESULTS_DIR}/verify_list.txt + +# Compare and save results in different formats +for OUTPUT_FORMAT in "csv" "html" "json" +do + OUTPUT_FILE=${RESULTS_DIR}/compare_results.${OUTPUT_FORMAT} + rally --rally-debug verify compare --uuid-1 ${VERIFICATIONS[1]} --uuid-2 ${VERIFICATIONS[2]} --${OUTPUT_FORMAT} --output-file ${OUTPUT_FILE} + RESULTS+="c_${OUTPUT_FORMAT}=$(do_status $?) " + gzip -9 ${OUTPUT_FILE} +done + +python $BASE/new/rally/rally/ui/utils.py render\ + tests/ci/rally-gate/index_verify.mako ${RESULTS[*]}> ${RESULTS_DIR}/extra/index.html + +if [[ ${RESULTS[*]} == *"fail"* ]] +then + return 1 +fi diff --git a/tests/functional/test_cli_verify.py b/tests/functional/test_cli_verify.py index 631df542dd..903c1c7590 100644 --- a/tests/functional/test_cli_verify.py +++ b/tests/functional/test_cli_verify.py @@ -13,35 +13,17 @@ # License for the specific language governing permissions and limitations # under the License. -import json -import unittest +############################################################################## +# +# THIS MODULE IS DEPRECATED. +# DON'T ADD TESTS FOR "rally verify" HERE. +# +# This module is no longer used for testing "rally verify" command. +# Functional testing for this command is moved to separate job. +# https://review.openstack.org/#/c/137232 +# +# Please look at tests/ci/rally-verify.sh for more details. +# +############################################################################## -from tests.functional import utils - - -class VerifyTestCase(unittest.TestCase): - - def setUp(self): - super(VerifyTestCase, self).setUp() - self.rally = utils.Rally() - - def _verify_start_and_get_results_in_json(self, set_name): - self.rally("verify start %s" % set_name) - results = json.loads(self.rally("verify results --json")) - - failed_tests = results["failures"] * 100.0 / results["tests"] - if failed_tests >= 50: - self.fail("Number of failed tests more than 50%.") - - show_output = self.rally("verify show") - - total_raw = show_output.split("\n").pop(5)[1:-1].replace(" ", "") - total = total_raw.split('|') - - self.assertEqual(set_name, total[2]) - self.assertEqual(results["tests"], int(total[3])) - self.assertEqual(results["failures"], int(total[4])) - self.assertEqual("finished", total[6]) - - def test_image_set(self): - self._verify_start_and_get_results_in_json("image") +pass diff --git a/tests/hacking/README.rst b/tests/hacking/README.rst index 256811fddc..1a1d4881cf 100644 --- a/tests/hacking/README.rst +++ b/tests/hacking/README.rst @@ -14,6 +14,7 @@ Rally Specific Commandments * [N310-N314] - Reserved for rules related to logging * [N310] - Ensure that ``rally.log`` is used instead of ``rally.openstack.common.log`` * [N311] - Validate that debug level logs are not translated + * [N312] - Validate correctness of debug on check. * [N32x] - Reserved for rules related to assert* methods * [N320] - Ensure that ``assertTrue(isinstance(A, B))`` is not used * [N321] - Ensure that ``assertEqual(type(A), B)`` is not used diff --git a/tests/hacking/checks.py b/tests/hacking/checks.py index 20df5468c1..6f25d04827 100644 --- a/tests/hacking/checks.py +++ b/tests/hacking/checks.py @@ -138,6 +138,24 @@ def no_translate_debug_logs(logical_line): yield(0, "N311 Don't translate debug level logs") +def no_use_conf_debug_check(logical_line, filename): + """Check for 'cfg.CONF.debug' + + Rally has two DEBUG level: + - Full DEBUG, which include all debug-messages from all OpenStack services + - Rally DEBUG, which include only Rally debug-messages + so we should use custom check to know debug-mode, instead of CONF.debug + + N312 + """ + excluded_files = ["./rally/log.py"] + + point = logical_line.find("CONF.debug") + if point != -1 and filename not in excluded_files: + yield(point, "N312 Don't use `CONF.debug`. " + "Function `rally.log.is_debug` should be used instead.") + + def assert_true_instance(logical_line): """Check for assertTrue(isinstance(a, b)) sentences @@ -209,6 +227,7 @@ def factory(register): register(check_assert_methods_from_mock) register(check_import_of_logging) register(no_translate_debug_logs) + register(no_use_conf_debug_check) register(assert_true_instance) register(assert_equal_type) register(assert_equal_none) diff --git a/tests/unit/test_hacking.py b/tests/unit/test_hacking.py index 75e053d47d..f922eca025 100644 --- a/tests/unit/test_hacking.py +++ b/tests/unit/test_hacking.py @@ -93,6 +93,16 @@ class HackingTestCase(test.TestCase): self.assertEqual(len(list(checks.no_translate_debug_logs( "LOG.info(_('foo'))"))), 0) + def test_no_use_conf_debug_check(self): + self.assertEqual(len(list(checks.no_use_conf_debug_check( + "if CONF.debug:", "fakefile"))), 1) + + self.assertEqual(len(list(checks.no_use_conf_debug_check( + "if cfg.CONF.debug", "fakefile"))), 1) + + self.assertEqual(len(list(checks.no_use_conf_debug_check( + "if logging.is_debug()", "fakefile"))), 0) + def test_assert_true_instance(self): self.assertEqual(len(list(checks.assert_true_instance( "self.assertTrue(isinstance(e, "