diff --git a/.zuul.d/rally-tox-functional.yaml b/.zuul.d/rally-tox-functional.yaml new file mode 100644 index 00000000..f483dd56 --- /dev/null +++ b/.zuul.d/rally-tox-functional.yaml @@ -0,0 +1,21 @@ +- job: + name: rally-dsvm-tox-functional + parent: devstack + description: + Run functional test for rally-openstack project. + Uses tox with the ``functional`` environment. + required-projects: + - name: openstack-dev/devstack + - name: openstack-infra/devstack-gate + - name: openstack/rally-openstack + # NOTE(andreykurilin): it is a required project to fetch the latest + # version and test master of rally-openstack with master of rally + - name: openstack/rally + timeout: 7200 + roles: + - zuul: openstack-dev/devstack + vars: + devstack_plugins: + rally-openstack: https://git.openstack.org/openstack/rally-openstack + rally_tox_env: "functional" + run: tests/ci/playbooks/run-rally-tox.yaml diff --git a/.zuul.d/zuul.yaml b/.zuul.d/zuul.yaml index b08a66b1..7d5db7d0 100644 --- a/.zuul.d/zuul.yaml +++ b/.zuul.d/zuul.yaml @@ -48,6 +48,7 @@ - rally-tox-py36 - rally-tox-py37 - rally-tox-cover + - rally-dsvm-tox-functional - rally-docker-check - rally-task-simple-job - rally-task-basic-with-existing-users: @@ -99,6 +100,7 @@ - rally-tox-py36 - rally-tox-py37 - rally-tox-cover + - rally-dsvm-tox-functional - rally-docker-check - rally-task-simple-job #- rally-task-heat diff --git a/tests/check_samples/__init__.py b/tests/check_samples/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/check_samples/utils.py b/tests/check_samples/utils.py deleted file mode 100644 index baf34213..00000000 --- a/tests/check_samples/utils.py +++ /dev/null @@ -1,32 +0,0 @@ -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import subprocess - -from tests.functional import utils - - -DEPLOYMENT_FILE = "/tmp/rally_functests_main_deployment.json" - - -class Rally(utils.Rally): - - def __init__(self, force_new_db=False): - self._DEPLOYMENT_CREATE_ARGS = " --file %s" % DEPLOYMENT_FILE - if not os.path.exists(DEPLOYMENT_FILE): - subprocess.call(["rally", "--log-file", "/dev/null", - "deployment", "config"], - stdout=open(DEPLOYMENT_FILE, "w")) - super(Rally, self).__init__(force_new_db=force_new_db) diff --git a/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml b/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml index 05a86da7..dcbdf99a 100644 --- a/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml +++ b/tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml @@ -175,10 +175,16 @@ nova flavor-create m1.nano 42 64 0 1 fi +- name: Check the existence of rally_task + stat: + path: '{{ zuul.project.src_dir }}/{{ rally_task }}' + register: rally_task_file_stat + - name: Copy task file become: True become_user: stack command: cp "{{ zuul.project.src_dir }}/{{ rally_task }}" "{{ rally_home_dir }}/task.yaml" + when: rally_task_file_stat.stat.exists == True - name: Check the existence of task_args_file stat: diff --git a/tests/ci/playbooks/roles/run-rally-tox/tasks/main.yaml b/tests/ci/playbooks/roles/run-rally-tox/tasks/main.yaml new file mode 100644 index 00000000..0776542e --- /dev/null +++ b/tests/ci/playbooks/roles/run-rally-tox/tasks/main.yaml @@ -0,0 +1,10 @@ +- name: Run tox command + become: True + become_user: stack + shell: + executable: /bin/sh + cmd: | + set -e + cd {{ zuul.project.src_dir }} + export REQUESTS_CA_BUNDLE=/opt/stack/data/ca-bundle.pem + tox -e {{ rally_tox_env }} \ No newline at end of file diff --git a/tests/ci/playbooks/run-rally-tox.yaml b/tests/ci/playbooks/run-rally-tox.yaml new file mode 100644 index 00000000..5088c939 --- /dev/null +++ b/tests/ci/playbooks/run-rally-tox.yaml @@ -0,0 +1,15 @@ +- hosts: all + roles: + - run-devstack + +- hosts: controller + vars: + rally_home_dir: '/opt/stack/.rally' + rally_fake_image_path: '{{ rally_home_dir }}/extra/fake-image.img' + rally_use_existing_users: false + rally_existing_users_config: '{{ rally_home_dir }}/with-existing-users-config' + rally_task_args_file: "100-percent-not-exist-file" + rally_task: "100-percent-not-exist-file" + roles: + - prepare-for-rally-task + - run-rally-tox \ No newline at end of file diff --git a/tests/ci/rally_functional_job.sh b/tests/ci/rally_functional_job.sh new file mode 100755 index 00000000..9ff3da20 --- /dev/null +++ b/tests/ci/rally_functional_job.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +LOCAL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +DB_CONNECTION="$(rally db show)" + +if [[ $DB_CONNECTION == sqlite* ]]; then + CONCURRENCY=0 +else + # in case of not sqlite db backends we cannot launch tests in parallel due + # to possible conflicts + CONCURRENCY=1 + # currently, RCI_KEEP_DB variable is used to not create new databases per + # each test + export RCI_KEEP_DB=1 +fi + +python $LOCAL_DIR/pytest_launcher.py "tests/functional" --concurrency $CONCURRENCY --posargs=$1 diff --git a/tests/functional/extra/fake_dir1/fake_plugin1.py b/tests/functional/extra/fake_dir1/fake_plugin1.py deleted file mode 100644 index 4d2c1a49..00000000 --- a/tests/functional/extra/fake_dir1/fake_plugin1.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2015: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from rally.task import scenario - - -@scenario.configure(name="FakeScenarioPlugin1.list") -class FakeScenarioPlugin1(scenario.Scenario): - - def run(self): - """Sample fake scenario.""" - pass diff --git a/tests/functional/extra/fake_dir2/fake_plugin2.py b/tests/functional/extra/fake_dir2/fake_plugin2.py deleted file mode 100644 index e380842c..00000000 --- a/tests/functional/extra/fake_dir2/fake_plugin2.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2015: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from rally.task import scenario - - -@scenario.configure(name="FakeScenarioPlugin2.list") -class FakeScenarioPlugin2(scenario.Scenario): - - def run(self): - """Sample fake scenario.""" - pass diff --git a/tests/functional/extra/fake_platforms.py b/tests/functional/extra/fake_platforms.py deleted file mode 100644 index 1f015379..00000000 --- a/tests/functional/extra/fake_platforms.py +++ /dev/null @@ -1,43 +0,0 @@ -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from rally.env import platform - - -@platform.configure(name="good", platform="fake") -class GoodPlatform(platform.Platform): - - CONFIG_SCHEMA = {} - - def create(self): - return {}, {} - - def destroy(self): - pass - - def cleanup(self, task_uuid=None): - return { - "message": "Coming soon!", - "discovered": 0, - "deleted": 0, - "failed": 0, - "resources": {}, - "errors": [] - } - - def check_health(self): - return {"available": True} - - def info(self): - return {"info": {"a": 1}} diff --git a/tests/functional/extra/test_fake_scenario.json b/tests/functional/extra/test_fake_scenario.json deleted file mode 100644 index e831ae7e..00000000 --- a/tests/functional/extra/test_fake_scenario.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "FakeScenarioPlugin1.list": [ - { - "runner": { - "type": "constant", - "times": 5 - } - } - ], - "FakeScenarioPlugin2.list": [ - { - "runner": { - "type": "constant", - "times": 5 - } - } - ] -} \ No newline at end of file diff --git a/tests/check_samples/test_certification_task.py b/tests/functional/test_certification_task.py similarity index 89% rename from tests/check_samples/test_certification_task.py rename to tests/functional/test_certification_task.py index fdd0a73a..b46ea85d 100644 --- a/tests/check_samples/test_certification_task.py +++ b/tests/functional/test_certification_task.py @@ -19,18 +19,14 @@ import traceback import unittest import rally_openstack -from tests.check_samples import utils +from tests.functional import utils class TestPreCreatedTasks(unittest.TestCase): - def test_check_success(self): - rally = utils.Rally() - rally("deployment check") - def test_task_samples_is_valid(self): - rally = utils.Rally() + rally = utils.RallyWithSpecifiedDeployment() full_path = os.path.join( os.path.dirname(rally_openstack.__file__), os.pardir, "tasks", "openstack") diff --git a/tests/functional/test_cli_deployment.py b/tests/functional/test_cli_deployment.py index 99d8efb6..68bc03b4 100644 --- a/tests/functional/test_cli_deployment.py +++ b/tests/functional/test_cli_deployment.py @@ -51,11 +51,6 @@ class DeploymentTestCase(unittest.TestCase): "--filename /tmp/.tmp.deployment") self.assertIn("t_create_file", rally("deployment list")) - def test_create_empty(self): - rally = utils.Rally() - rally("deployment create --name t_empty") - self.assertEqual("{}", rally("deployment config").strip()) - def test_destroy(self): rally = utils.Rally() rally.env.update(TEST_ENV) @@ -120,9 +115,6 @@ class DeploymentTestCase(unittest.TestCase): rally.env) self.assertEqual(uuid, current_deployment) - # TODO(andreykurilin): Do not forget to move thes tests while splitting - # rally to main framework and openstack plugins - def test_create_from_env_openstack_deployment(self): rally = utils.Rally() rally.env.update(TEST_ENV) diff --git a/tests/functional/test_cli_env.py b/tests/functional/test_cli_env.py deleted file mode 100644 index 1102706c..00000000 --- a/tests/functional/test_cli_env.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2013: ITLook, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import os -import tempfile - -import unittest - -from tests.functional import utils - - -class EnvTestCase(unittest.TestCase): - - def test_create_no_spec(self): - rally = utils.Rally() - rally("env create --name empty --description de") - self.assertIn("empty", rally("env list")) - env_data = rally("env show --json", getjson=True) - self.assertEqual("empty", env_data["name"]) - self.assertEqual("de", env_data["description"]) - self.assertEqual({}, env_data["extras"]) - self.assertEqual({}, env_data["platforms"]) - - def _create_spec(self, spec): - f = tempfile.NamedTemporaryFile(delete=False) - - def unlink(): - os.unlink(f.name) - - self.addCleanup(unlink) - - f.write(json.dumps(spec, indent=2)) - f.close() - return f.name - - def test_create_check_info_destroy_delete_with_spec(self): - rally = utils.Rally(plugin_path="tests/functional/extra") - - spec = self._create_spec({"good@fake": {}}) - rally("env create --name real --spec %s" % spec) - env = rally("env show --json", getjson=True) - self.assertIn("fake", env["platforms"]) - - env_info = rally("env info --json", getjson=True) - self.assertEqual({"good@fake": {"info": {"a": 1}}}, env_info) - - rally("env check --json") - - def test_list_empty(self): - rally = utils.Rally() - # TODO(boris-42): Clean this up somehow - rally("env destroy MAIN") - rally("env delete MAIN") - self.assertEqual([], rally("env list --json", getjson=True)) - self.assertIn("There are no environments", rally("env list")) - - def test_list(self): - rally = utils.Rally() - envs = rally("env list --json", getjson=True) - self.assertEqual(1, len(envs)) - self.assertEqual("MAIN", envs[0]["name"]) - self.assertIn("MAIN", rally("env list")) - - def test_use(self): - - def show_helper(): - return rally("env show --json", getjson=True) - - rally = utils.Rally() - self.assertEqual("MAIN", show_helper()["name"]) - empty_uuid = rally("env create --name empty --json", - getjson=True)["uuid"] - self.assertEqual("empty", show_helper()["name"]) - rally("env use MAIN") - self.assertEqual("MAIN", show_helper()["name"]) - rally("env use %s" % empty_uuid) - self.assertEqual("empty", show_helper()["name"]) - rally("env create --name empty2 --description de --no-use") - self.assertEqual("empty", show_helper()["name"]) diff --git a/tests/functional/test_cli_plugin.py b/tests/functional/test_cli_plugin.py deleted file mode 100644 index 5218ee81..00000000 --- a/tests/functional/test_cli_plugin.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2015: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import unittest - -from tests.functional import utils - - -class PluginTestCase(unittest.TestCase): - - def test_show_one(self): - rally = utils.Rally() - result = rally("plugin show Dummy.dummy") - self.assertIn("NAME", result) - self.assertIn("PLATFORM", result) - self.assertIn("Dummy.dummy", result) - self.assertIn("MODULE", result) - - def test_show_multiple(self): - rally = utils.Rally() - result = rally("plugin show Dummy") - self.assertIn("Multiple plugins found:", result) - self.assertIn("Dummy.dummy", result) - self.assertIn("Dummy.dummy_exception", result) - self.assertIn("Dummy.dummy_random_fail_in_atomic", result) - - def test_show_not_found(self): - rally = utils.Rally() - name = "Dummy666666" - result = rally("plugin show %s" % name) - self.assertIn("Plugin %s not found" % name, result) - - def test_show_not_found_in_specific_platform(self): - rally = utils.Rally() - name = "Dummy" - platform = "non_existing" - result = rally( - "plugin show --name %(name)s --platform %(platform)s" - % {"name": name, "platform": platform}) - self.assertIn( - "Plugin %(name)s@%(platform)s not found" - % {"name": name, "platform": platform}, - result) - - def test_list(self): - rally = utils.Rally() - result = rally("plugin list Dummy") - self.assertIn("Dummy.dummy", result) - self.assertIn("Dummy.dummy_exception", result) - self.assertIn("Dummy.dummy_random_fail_in_atomic", result) - - def test_list_not_found_platform(self): - rally = utils.Rally() - result = rally("plugin list --platform some") - self.assertIn("Platform some not found", result) - - def test_list_not_found_name(self): - rally = utils.Rally() - result = rally("plugin list Dummy2222") - self.assertIn("Plugin Dummy2222 not found", result) diff --git a/tests/functional/test_cli_task.py b/tests/functional/test_cli_task.py deleted file mode 100644 index 8a6cb517..00000000 --- a/tests/functional/test_cli_task.py +++ /dev/null @@ -1,1424 +0,0 @@ -# Copyright 2013: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import os -import re -import threading -import time -import unittest - -import jsonschema -import mock - -from rally import api -from tests.functional import utils - - -FAKE_TASK_UUID = "87ab639d-4968-4638-b9a1-07774c32484a" - - -class TaskTestCase(unittest.TestCase): - - def _get_sample_task_config(self): - return { - "Dummy.dummy_random_fail_in_atomic": [ - { - "runner": { - "type": "constant", - "times": 20, - "concurrency": 5 - }, - "sla": { - "failure_rate": {"max": 100} - } - } - ] - } - - def _get_sample_task_config_v2(self): - return { - "version": 2, - "title": "Dummy task", - "tags": ["dummy", "functional_test"], - "subtasks": [ - { - "title": "first-subtask", - "group": "Dummy group", - "description": "The first subtask in dummy task", - "tags": ["dummy", "functional_test"], - "workloads": [ - { - "scenario": { - "Dummy.dummy": {"sleep": 0}}, - "runner": { - "constant": { - "times": 10, - "concurrency": 2 - } - } - } - ] - }, - { - "title": "second-subtask", - "description": "The second subtask in dummy task", - "tags": ["dummy", "functional_test"], - "scenario": { - "Dummy.dummy": {"sleep": 1}}, - "runner": { - "constant": { - "times": 10, - "concurrency": 2 - } - } - } - ] - } - - def _get_deployment_uuid(self, output): - return re.search( - r"Using deployment: (?P[0-9a-f\-]{36})", - output).group("uuid") - - def _get_task_uuid(self, output): - return re.search( - r"\trally task report (?P[0-9a-f\-]{36})", - output).group("uuid") - - def test_status(self): - rally = utils.Rally() - cfg = self._get_sample_task_config() - config = utils.TaskConfig(cfg) - rally("task start --task %s" % config.filename) - self.assertIn("finished", rally("task status")) - - def test_detailed(self): - rally = utils.Rally() - cfg = self._get_sample_task_config() - config = utils.TaskConfig(cfg) - rally("task start --task %s" % config.filename) - detailed = rally("task detailed") - self.assertIn("Dummy.dummy_random_fail_in_atomic", detailed) - self.assertIn("dummy_fail_test (x2)", detailed) - detailed_iterations_data = rally("task detailed --iterations-data") - self.assertIn(". dummy_fail_test (x2)", detailed_iterations_data) - self.assertNotIn("n/a", detailed_iterations_data) - - def test_detailed_with_errors(self): - rally = utils.Rally() - cfg = { - "Dummy.dummy_exception": [ - { - "runner": { - "type": "constant", - "times": 1, - "concurrency": 1 - }, - "sla": { - "failure_rate": {"max": 100} - } - } - ] - } - config = utils.TaskConfig(cfg) - output = rally("task start --task %s" % config.filename) - uuid = re.search( - r"(?P[0-9a-f\-]{36}): started", output).group("uuid") - output = rally("task detailed") - self.assertIn("Task %s has 1 error(s)" % uuid, output) - - def test_detailed_no_atomic_actions(self): - rally = utils.Rally() - cfg = { - "Dummy.dummy": [ - { - "runner": { - "type": "constant", - "times": 100, - "concurrency": 5 - } - } - ] - } - config = utils.TaskConfig(cfg) - rally("task start --task %s" % config.filename) - detailed = rally("task detailed") - self.assertIn("Dummy.dummy", detailed) - detailed_iterations_data = rally("task detailed --iterations-data") - self.assertNotIn("n/a", detailed_iterations_data) - - def test_start_with_empty_config(self): - rally = utils.Rally() - config = utils.TaskConfig(None) - with self.assertRaises(utils.RallyCliError) as err: - rally("task start --task %s" % config.filename) - self.assertIn("Input task is empty", err.exception.output) - - def test_results(self): - rally = utils.Rally() - cfg = self._get_sample_task_config() - config = utils.TaskConfig(cfg) - rally("task start --task %s" % config.filename) - self.assertIn("result", rally("task results")) - - def test_results_with_wrong_task_id(self): - rally = utils.Rally() - self.assertRaises(utils.RallyCliError, - rally, "task results --uuid %s" % FAKE_TASK_UUID) - - def test_import_results(self): - rally = utils.Rally() - cfg = self._get_sample_task_config() - config = utils.TaskConfig(cfg) - rally("task start --task %s" % config.filename) - json_report = rally.gen_report_path(extension="json") - with open(json_report, "w+") as f: - f.write(rally("task results", no_logs=True)) - import_print = rally("task import --file %s" % json_report) - task_uuid = re.search("UUID:\s([a-z0-9\-]+)", import_print).group(1) - self.assertIn("Dummy.dummy_random_fail_in_atomic", - rally("task results --uuid %s" % task_uuid)) - - # new json report - rally("task report --json --out %s" % json_report, no_logs=True) - import_print = rally("task import --file %s" % json_report) - task_uuid = re.search("UUID:\s([a-z0-9\-]+)", import_print).group(1) - self.assertIn("Dummy.dummy_random_fail_in_atomic", - rally("task report --uuid %s --json" % task_uuid)) - - def test_abort_with_wrong_task_id(self): - rally = utils.Rally() - self.assertRaises(utils.RallyCliError, - rally, "task abort --uuid %s" % FAKE_TASK_UUID) - - def test_delete_with_wrong_task_id(self): - rally = utils.Rally() - self.assertRaises(utils.RallyCliError, - rally, "task delete --uuid %s" % FAKE_TASK_UUID) - - def test_detailed_with_wrong_task_id(self): - rally = utils.Rally() - self.assertRaises(utils.RallyCliError, - rally, "task detailed --uuid %s" % FAKE_TASK_UUID) - - def test_report_with_wrong_task_id(self): - rally = utils.Rally() - self.assertRaises(utils.RallyCliError, - rally, "task report --tasks %s" % FAKE_TASK_UUID) - self.assertRaises(utils.RallyCliError, - rally, "task report --uuid %s" % FAKE_TASK_UUID) - - def test_sla_check_with_wrong_task_id(self): - rally = utils.Rally() - self.assertRaises(utils.RallyCliError, - rally, "task sla-check --uuid %s" % FAKE_TASK_UUID) - - def test_status_with_wrong_task_id(self): - rally = utils.Rally() - self.assertRaises(utils.RallyCliError, - rally, "task status --uuid %s" % FAKE_TASK_UUID) - - def _assert_html_report_libs_are_embedded(self, file_path, expected=True): - - embedded_signatures = ["Copyright (c) 2011-2014 Novus Partners, Inc.", - "AngularJS v1.3.3", - "Copyright (c) 2010-2015, Michael Bostock"] - external_signatures = ["