diff --git a/fuelweb_test/helpers/patching.py b/fuelweb_test/helpers/patching.py index 8a94dc53c..1dc51b95d 100644 --- a/fuelweb_test/helpers/patching.py +++ b/fuelweb_test/helpers/patching.py @@ -135,6 +135,11 @@ def map_test(target): env_distro, target) + if 'rally' in errata.keys(): + if len(errata['rally']) > 0: + settings.PATCHING_RUN_RALLY = True + settings.RALLY_TAGS = errata['rally'] + if settings.PATCHING_CUSTOM_TEST: deployment_test = settings.PATCHING_CUSTOM_TEST settings.PATCHING_SNAPSHOT = \ diff --git a/fuelweb_test/settings.py b/fuelweb_test/settings.py index e48c2a54a..7bdb0a3d3 100644 --- a/fuelweb_test/settings.py +++ b/fuelweb_test/settings.py @@ -428,6 +428,7 @@ PATCHING_PKGS = os.environ.get("PATCHING_PKGS", None) PATCHING_SNAPSHOT = os.environ.get("PATCHING_SNAPSHOT", None) PATCHING_CUSTOM_TEST = os.environ.get("PATCHING_CUSTOM_TEST", None) PATCHING_DISABLE_UPDATES = get_var_as_bool('PATCHING_DISABLE_UPDATES', False) +PATCHING_RUN_RALLY = get_var_as_bool("PATCHING_RUN_RALLY", False) DOWNLOAD_LINK = os.environ.get( 'DOWNLOAD_LINK', 'http://releases.ubuntu.com/14.04.2/' diff --git a/fuelweb_test/tests/tests_patching/test_patching.py b/fuelweb_test/tests/tests_patching/test_patching.py index 92ba80490..b93880cf3 100644 --- a/fuelweb_test/tests/tests_patching/test_patching.py +++ b/fuelweb_test/tests/tests_patching/test_patching.py @@ -24,6 +24,8 @@ from fuelweb_test import logger from fuelweb_test import settings from fuelweb_test.helpers import patching from fuelweb_test.helpers.decorators import log_snapshot_after_test +from fuelweb_test.helpers.rally import RallyBenchmarkTest +from fuelweb_test.helpers.rally import RallyResult from fuelweb_test.helpers.utils import install_pkg from fuelweb_test.tests.base_test_case import TestBasic @@ -99,7 +101,18 @@ class PatchingTests(TestBasic): assert_is_not_none(cluster_id, 'Environment for patching not found.') # Step #2 - # Run Rally benchmarks, coming soon... + if settings.PATCHING_RUN_RALLY: + rally_benchmarks = {} + benchmark_results1 = {} + for tag in set(settings.RALLY_TAGS): + rally_benchmarks[tag] = RallyBenchmarkTest( + container_repo=settings.RALLY_DOCKER_REPO, + environment=self.env, + cluster_id=cluster_id, + test_type=tag + ) + benchmark_results1[tag] = rally_benchmarks[tag].run() + logger.debug(benchmark_results1[tag].show()) # Step #3 patching_repos = patching.add_remote_repositories( @@ -137,8 +150,23 @@ class PatchingTests(TestBasic): self.fuel_web.run_ostf(cluster_id=cluster_id) # Step #8 - # Run Rally benchmarks, compare new results with previous, - # coming soon... + if settings.PATCHING_RUN_RALLY: + benchmark_results2 = {} + for tag in set(settings.RALLY_TAGS): + benchmark_results2[tag] = rally_benchmarks[tag].run() + logger.debug(benchmark_results2[tag].show()) + + rally_benchmarks_passed = True + + for tag in set(settings.RALLY_TAGS): + if not RallyResult.compare(benchmark_results1[tag], + benchmark_results2[tag], + deviation=0.2): + rally_benchmarks_passed = False + + assert_true(rally_benchmarks_passed, + "Rally benchmarks show performance degradation " + "after packages patching.") @test(groups=["patching_master_tests"])