Configure gate-rally-dsvm-verify

Recently, we added new job "gate-rally-dsvm-verify" for Rally[1].
This job implied functional testing integration Rally with Tempest[2].

This patch adds tests/ci/rally-verify.sh script, which is used by new job.

rally-verify.sh does:
- tempest installation
- run "rally verify start" twice and print results
- compare results of two verifications
- list verifications
- generate html page based on results

To implement gate-rally-dsvm-verify some changes were requered to
existing Rally code:
- Added ability for rally/ui/utils.py to accept arguments to render html-pages
- Fixed logging debug-messages in tempest verifier
- Fixed check "is debug mode turned on or not"(also, added hacking rule for it)

TODO for future patches:
- add launch of rally task for Tempest
- add launch of random test set
- add check for successful tests

[1] https://review.openstack.org/#/c/137232
[2] https://www.mirantis.com/blog/rally-openstack-tempest-testing-made-simpler

Closes-Bug: #1400465
Closes-Bug: #1400518

Change-Id: I8e1fbab22c2da109bbc442f040fe259e5d22a62a

tests/functional/test_cli_verify.py

@@ -13,35 +13,17 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
-import unittest
+##############################################################################
+#
+#                        THIS MODULE IS DEPRECATED.
+#                 DON'T ADD TESTS FOR "rally verify" HERE.
+#
+# This module is no longer used for testing "rally verify" command.
+# Functional testing for this command is moved to separate job.
+# https://review.openstack.org/#/c/137232
+#
+# Please look at tests/ci/rally-verify.sh for more details.
+#
+##############################################################################
 
-from tests.functional import utils
-
-
-class VerifyTestCase(unittest.TestCase):
-
-    def setUp(self):
-        super(VerifyTestCase, self).setUp()
-        self.rally = utils.Rally()
-
-    def _verify_start_and_get_results_in_json(self, set_name):
-        self.rally("verify start %s" % set_name)
-        results = json.loads(self.rally("verify results --json"))
-
-        failed_tests = results["failures"] * 100.0 / results["tests"]
-        if failed_tests >= 50:
-            self.fail("Number of failed tests more than 50%.")
-
-        show_output = self.rally("verify show")
-
-        total_raw = show_output.split("\n").pop(5)[1:-1].replace(" ", "")
-        total = total_raw.split('|')
-
-        self.assertEqual(set_name, total[2])
-        self.assertEqual(results["tests"], int(total[3]))
-        self.assertEqual(results["failures"], int(total[4]))
-        self.assertEqual("finished", total[6])
-
-    def test_image_set(self):
-        self._verify_start_and_get_results_in_json("image")
+pass

tests/hacking/README.rst

@@ -14,6 +14,7 @@ Rally Specific Commandments
 * [N310-N314] - Reserved for rules related to logging
  * [N310] - Ensure that ``rally.log`` is used instead of ``rally.openstack.common.log``
  * [N311] - Validate that debug level logs are not translated
+ * [N312] - Validate correctness of debug on check.
 * [N32x] - Reserved for rules related to assert* methods
  * [N320] - Ensure that ``assertTrue(isinstance(A, B))``  is not used
  * [N321] - Ensure that ``assertEqual(type(A), B)`` is not used

tests/hacking/checks.py

@@ -138,6 +138,24 @@ def no_translate_debug_logs(logical_line):
         yield(0, "N311 Don't translate debug level logs")
 
 
+def no_use_conf_debug_check(logical_line, filename):
+    """Check for 'cfg.CONF.debug'
+
+    Rally has two DEBUG level:
+     - Full DEBUG, which include all debug-messages from all OpenStack services
+     - Rally DEBUG, which include only Rally debug-messages
+    so we should use custom check to know debug-mode, instead of CONF.debug
+
+    N312
+    """
+    excluded_files = ["./rally/log.py"]
+
+    point = logical_line.find("CONF.debug")
+    if point != -1 and filename not in excluded_files:
+        yield(point, "N312 Don't use `CONF.debug`. "
+                     "Function `rally.log.is_debug` should be used instead.")
+
+
 def assert_true_instance(logical_line):
     """Check for assertTrue(isinstance(a, b)) sentences
 
@@ -209,6 +227,7 @@ def factory(register):
     register(check_assert_methods_from_mock)
     register(check_import_of_logging)
     register(no_translate_debug_logs)
+    register(no_use_conf_debug_check)
     register(assert_true_instance)
     register(assert_equal_type)
     register(assert_equal_none)

tests/unit/test_hacking.py

@@ -93,6 +93,16 @@ class HackingTestCase(test.TestCase):
         self.assertEqual(len(list(checks.no_translate_debug_logs(
             "LOG.info(_('foo'))"))), 0)
 
+    def test_no_use_conf_debug_check(self):
+        self.assertEqual(len(list(checks.no_use_conf_debug_check(
+            "if CONF.debug:", "fakefile"))), 1)
+
+        self.assertEqual(len(list(checks.no_use_conf_debug_check(
+            "if cfg.CONF.debug", "fakefile"))), 1)
+
+        self.assertEqual(len(list(checks.no_use_conf_debug_check(
+            "if logging.is_debug()", "fakefile"))), 0)
+
     def test_assert_true_instance(self):
         self.assertEqual(len(list(checks.assert_true_instance(
             "self.assertTrue(isinstance(e, "