diff --git a/utils/fuel-tasklib/LICENSE b/utils/fuel-tasklib/LICENSE new file mode 100644 index 0000000000..e06d208186 --- /dev/null +++ b/utils/fuel-tasklib/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/utils/fuel-tasklib/README.rst b/utils/fuel-tasklib/README.rst new file mode 100644 index 0000000000..55d7a61510 --- /dev/null +++ b/utils/fuel-tasklib/README.rst @@ -0,0 +1,24 @@ +fuel-tasklib +============ + +Project resources +----------------- + + +Project status, bugs, and blueprints are tracked on Launchpad: +https://launchpad.net/fuel + +Development documentation is hosted here: +http://docs.mirantis.com/fuel-dev + +User guide can be found here: +http://docs.mirantis.com + +Any additional information can be found on the Fuel's project wiki: +https://wiki.openstack.org/wiki/Fuel + +Anyone wishing to contribute to fuel-tasklib follow the general +OpenStack process. A good reference for it can be found here: + +* https://wiki.openstack.org/wiki/How_To_Contribute +* http://docs.openstack.org/infra/manual/developers.html diff --git a/utils/fuel-tasklib/requirements.txt b/utils/fuel-tasklib/requirements.txt new file mode 100644 index 0000000000..33bc83329a --- /dev/null +++ b/utils/fuel-tasklib/requirements.txt @@ -0,0 +1,4 @@ +networkx>=1.8,<1.10 +jsonschema>=2.3.0 +PyYAML>=3.11 +argparse>=1.2.1 diff --git a/utils/fuel-tasklib/run_tests.sh b/utils/fuel-tasklib/run_tests.sh new file mode 100755 index 0000000000..9f4e3c2fce --- /dev/null +++ b/utils/fuel-tasklib/run_tests.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -e + +ROOT=$(dirname $(readlink -f $0)) +UTILS_DIR="${ROOT}" +TOX_PYENVS=${TOX_PYENVS:-"py26"} + +certain_tests=() +flake8_checks=1 +only_flake8_checks=0 +python_tests=1 +testropts="" + + + +print_error() { + echo >&2 "Error: $1" +} + +check_tox() { + type tox >/dev/null 2>&1 || { print_error "Tox is required to be installed to run tests."; exit 1; } +} + + +usage() { + echo "Usage: $0 [OPTIONS] [-- TESTR OPTIONS]" + echo "Run Python tests for fuel-tasklib" + echo "" + echo " -p, --pep8 Run only flake8 checks." + echo " -P, --no-pep8 Do not run flake8 tests" + echo " -t, --tests Select tests to run. Could be specified" + echo " multiple times to select multiple tests" + echo " -h, --help Print this usage message and exit." + exit +} + + +process_options() { + local TEMP=$(getopt \ + -o hpPt: \ + --long help,pep8,no-pep8,tests: \ + -n 'run_tests.sh' -- "$@") + + + eval set -- "$TEMP" + + while true ; do + case "$1" in + -p|--pep8) only_flake8_checks=1; shift 1;; + -P|--no-pep8) flake8_checks=0; shift 1;; + -h|--help) usage; shift 1;; + -t|--tests) certain_tests+=("$2"); shift 2;; + # All parameters and alien options will be passed to testr + --) shift 1; testropts+="$@"; + break;; + *) print_error "Internal error: got \"$1\" argument."; + usage; exit 1 + esac + + done + + # Check that specified test file/dir exists. Fail otherwise. + if [[ ${#certain_tests[@]} -ne 0 ]]; then + for test in ${certain_tests[@]}; do + local file_name=${test%:*} + + if [[ ! -f $file_name ]] && [[ ! -d $file_name ]]; then + print_error "Specified tests were not found." + exit 1 + fi + done + fi + +} + + +run_cleanup() { + echo "Doing a clean up." + find . -type f -name "*.pyc" -delete +} + + +# run pep8 checks +run_flake8() { + echo "Starting flake8 checks..." + + tox -e pep8 -- $testropts +} + + +# run tests +run_tests() { + echo "Starting Python tests..." + + local tests="${ROOT}/tasklib/tests" + + if [[ ${#certain_tests[@]} -ne 0 ]]; then + tests=${certain_tests[@]} + fi + + tox -e $TOX_PYENVS -- $testropts $tests +} + + +run() { + local errors="" + + run_cleanup + + #if flake8 is not disabled or only flake8 + if [[ $only_flake8_checks -eq 1 ]]; then + flake8_checks=1 + python_tests=0 + fi + + if [[ -n "${certain_tests[@]}" ]]; then + flake8_checks=0 + python_tests=1 + fi + + if [[ $flake8_checks -eq 1 ]]; then + run_flake8 || errors+=" flake8" + fi + + if [[ $python_tests -eq 1 ]]; then + run_tests || errors+=" Python" + fi + + # print failed tests + if [[ -n "$errors" ]]; then + echo Failed tests: $errors + exit 1 + fi +} + +check_tox +process_options $@ +run diff --git a/utils/fuel-tasklib/setup.cfg b/utils/fuel-tasklib/setup.cfg new file mode 100644 index 0000000000..da833e08c5 --- /dev/null +++ b/utils/fuel-tasklib/setup.cfg @@ -0,0 +1,33 @@ +[metadata] +name = fuel-tasklib +version = 6.1.0 +summary = Tasks validation for Fuel +author = Mirantis Inc. +author-email = product@mirantis.com +home-page = http://mirantis.com +description-file = + README.rst +classifier = + Intended Audience :: Developers + Intended Audience :: Information Technology + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.6 + +[entry_points] +console_scripts = + tasks-validator = tasklib.validator:main + +[files] +packages = + tasklib + +[global] +setup-hooks = + pbr.hooks.setup_hook + tasklib.setup_hooks.fix_version + +[wheel] +universal = 1 diff --git a/utils/fuel-tasklib/setup.py b/utils/fuel-tasklib/setup.py new file mode 100644 index 0000000000..581c4ce8b1 --- /dev/null +++ b/utils/fuel-tasklib/setup.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import setuptools + + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/utils/fuel-tasklib/tasklib/__init__.py b/utils/fuel-tasklib/tasklib/__init__.py new file mode 100644 index 0000000000..86beb98b9f --- /dev/null +++ b/utils/fuel-tasklib/tasklib/__init__.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/utils/fuel-tasklib/tasklib/graph.py b/utils/fuel-tasklib/tasklib/graph.py new file mode 100644 index 0000000000..c92d1863bb --- /dev/null +++ b/utils/fuel-tasklib/tasklib/graph.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import networkx as nx + + +class DeploymentGraph(object): + + def __init__(self, tasks): + self.tasks = tasks + self.graph = self._create_graph() + + def _create_graph(self): + """Create graph from tasks + + :return: directed graph + """ + graph = nx.DiGraph() + for task in self.tasks: + task_id = task['id'] + graph.add_node(task_id, **task) + if 'required_for' in task: + for req in task['required_for']: + graph.add_edge(task_id, req) + if 'requires' in task: + for req in task['requires']: + graph.add_edge(req, task_id) + + if 'groups' in task: + for req in task['groups']: + graph.add_edge(task_id, req) + if 'tasks' in task: + for req in task['tasks']: + graph.add_edge(req, task_id) + + return graph + + def find_cycles(self): + """Find cycles in graph. + + :return: list of cycles in graph + """ + cycles = [] + for cycle in nx.simple_cycles(self.graph): + cycles.append(cycle) + + return cycles + + def is_connected(self): + """Check if graph is connected. + + :return: bool + """ + return nx.is_weakly_connected(self.graph) + + def find_empty_nodes(self): + """Find empty nodes in graph. + + :return: list of empty nodes in graph + """ + empty_nodes = [] + for node_name, node in self.graph.node.items(): + if node == {}: + empty_nodes.append(node_name) + return empty_nodes diff --git a/utils/fuel-tasklib/tasklib/schemas.py b/utils/fuel-tasklib/tasklib/schemas.py new file mode 100644 index 0000000000..ba41c3a53d --- /dev/null +++ b/utils/fuel-tasklib/tasklib/schemas.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class BaseTasksSchema(object): + + base_task_schema = { + '$schema': 'http://json-schema.org/draft-04/schema#', + 'type': 'object', + 'required': ['type', 'id'], + 'properties': { + 'id': {'type': 'string', + 'minLength': 1}, + 'type': {'enum': [], + 'type': 'string'}, + 'parameters': {'type': 'object'}, + 'required_for': {'type': 'array'}, + 'requires': {'type': 'array'}, + }} + + base_tasks_schema = { + '$schema': 'http://json-schema.org/draft-04/schema#', + 'type': 'array', + 'items': ''} + + types = {'enum': ['puppet', 'shell'], + 'type': 'string'}, + + @property + def task_schema(self): + self.base_task_schema['properties']['type'] = self.types + return self.base_task_schema + + @property + def tasks_schema(self): + self.base_tasks_schema['items'] = self.task_schema + return self.base_tasks_schema + + +class TasksSchema61(BaseTasksSchema): + + types = {'enum': ['puppet', 'shell', 'group', 'stage', 'copy_files', + 'sync', 'upload_file'], + 'type': 'string'} + + +VERSIONS_SCHEMAS_MAP = { + "6.1": TasksSchema61, + "last": TasksSchema61, +} diff --git a/utils/fuel-tasklib/tasklib/setup_hooks.py b/utils/fuel-tasklib/tasklib/setup_hooks.py new file mode 100644 index 0000000000..20bcb84f30 --- /dev/null +++ b/utils/fuel-tasklib/tasklib/setup_hooks.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def fix_version(config): + """Monkeypatch for PBR to do not add commit id to package version.""" + import pbr + import pbr.packaging + + pbr.packaging._get_version_from_git = lambda pre_version: pre_version diff --git a/utils/fuel-tasklib/tasklib/tests/.gitkeep b/utils/fuel-tasklib/tasklib/tests/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/utils/fuel-tasklib/tasklib/tests/__init__.py b/utils/fuel-tasklib/tasklib/tests/__init__.py new file mode 100644 index 0000000000..1f6e00f845 --- /dev/null +++ b/utils/fuel-tasklib/tasklib/tests/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/utils/fuel-tasklib/tasklib/tests/test_graph.py b/utils/fuel-tasklib/tasklib/tests/test_graph.py new file mode 100644 index 0000000000..2531b817ac --- /dev/null +++ b/utils/fuel-tasklib/tasklib/tests/test_graph.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from unittest2.case import TestCase + +from tasklib import graph + + +class TestGraphs(TestCase): + + def test_connectability(self): + tasks = [ + {'id': 'pre_deployment_start', + 'type': 'stage'}, + {'id': 'pre_deployment_end', + 'type': 'stage', + 'requires': ['pre_deployment_start']}, + {'id': 'deploy_start', + 'type': 'stage'}] + tasks_graph = graph.DeploymentGraph(tasks) + self.assertFalse(tasks_graph.is_connected()) + + def test_cyclic(self): + tasks = [ + {'id': 'pre_deployment_start', + 'type': 'stage'}, + {'id': 'pre_deployment_end', + 'type': 'stage', + 'requires': ['pre_deployment_start']}, + {'id': 'deploy_start', + 'type': 'stage', + 'requires': ['pre_deployment_end'], + 'required_for': ['pre_deployment_start']}] + tasks_graph = graph.DeploymentGraph(tasks) + cycles = tasks_graph.find_cycles() + self.assertEqual(len(cycles), 1) + self.assertItemsEqual(cycles[0], ['deploy_start', + 'pre_deployment_start', + 'pre_deployment_end']) + + def test_empty_nodes(self): + tasks = [ + {'id': 'pre_deployment_start', + 'type': 'stage', + 'requires': ['empty_node']}, + {'id': 'pre_deployment_end', + 'type': 'stage', + 'requires': ['pre_deployment_start']}, + {'id': 'deploy_start', + 'type': 'stage', + 'requires': ['empty_node_2']}] + tasks_graph = graph.DeploymentGraph(tasks) + self.assertItemsEqual(tasks_graph.find_empty_nodes(), ['empty_node', + 'empty_node_2']) diff --git a/utils/fuel-tasklib/tasklib/tests/test_validator.py b/utils/fuel-tasklib/tasklib/tests/test_validator.py new file mode 100644 index 0000000000..4343861c21 --- /dev/null +++ b/utils/fuel-tasklib/tasklib/tests/test_validator.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import jsonschema +import mock + +from unittest2.case import TestCase + +from tasklib import validator + + +TASKS = [ + {'id': 'pre_deployment_start', + 'type': 'stage'}, + {'id': 'pre_deployment_end', + 'requires': ['pre_deployment_start'], + 'type': 'stage'}, + {'id': 'deploy_start', + 'requires': ['pre_deployment_end'], + 'type': 'stage'}, + {'id': 'deploy_end', + 'requires': ['deploy_start'], + 'type': 'stage'}, + {'id': 'post_deployment_start', + 'requires': ['deploy_end'], + 'type': 'stage'}, + {'id': 'post_deployment_end', + 'requires': ['post_deployment_start'], + 'type': 'stage'}] + + +class TestValidator61(TestCase): + + def setUp(self): + self.tasks = copy.deepcopy(TASKS) + + def test_validate_schema(self): + valid_tasks = validator.TasksValidator(self.tasks, "6.1") + valid_tasks.validate_schema() + + def test_wrong_schema(self): + self.tasks.append({'id': 'wrong', + 'type': 'non existing'}) + valid_tasks = validator.TasksValidator(self.tasks, "6.1") + self.assertRaises(jsonschema.ValidationError, + valid_tasks.validate_schema) + + def test_empty_id_schema(self): + self.tasks.append({'id': '', + 'type': 'stage'}) + valid_tasks = validator.TasksValidator(self.tasks, "6.1") + self.assertRaises(jsonschema.ValidationError, + valid_tasks.validate_schema) + + def test_validate_graph(self): + valid_tasks = validator.TasksValidator(self.tasks, "6.1") + valid_tasks.validate_graph() + + def test_validate_cyclic_graph(self): + self.tasks.append({'id': 'post_deployment_part', + 'type': 'stage', + 'requires': ['post_deployment_start'], + 'required_for': ['pre_deployment_start']}) + valid_tasks = validator.TasksValidator(self.tasks, "6.1") + self.assertRaises(ValueError, + valid_tasks.validate_graph) + + def test_validate_not_connected_graph(self): + self.tasks.append({'id': 'post_deployment_part', + 'type': 'stage'}) + valid_tasks = validator.TasksValidator(self.tasks, "6.1") + self.assertRaises(ValueError, + valid_tasks.validate_graph) + + def test_validate_duplicated_tasks(self): + self.tasks.append({'id': 'pre_deployment_start', + 'type': 'stage'}) + valid_tasks = validator.TasksValidator(self.tasks, "6.1") + self.assertRaises(ValueError, + valid_tasks.validate_unique_tasks) + + def test_validate_empty_nodes(self): + self.tasks.append({'id': 'some_task', + 'type': 'stage', + 'requires': ['empty_node', + 'post_deployment_start']}) + valid_tasks = validator.TasksValidator(self.tasks, "6.1") + self.assertRaises(ValueError, + valid_tasks.validate_graph) + + +class TestValidatorClient(TestCase): + + def test_no_dir(self): + args = ['script/name'] + try: + validator.main(args) + except SystemExit as pars_error: + pass + self.assertEqual(pars_error[0], 2) + + @mock.patch('tasklib.validator.get_tasks') + @mock.patch('tasklib.validator.TasksValidator') + def test_passing_params(self, mock_validator, mock_file): + mock_file.return_value = TASKS + args = ['/usr/local/bin/tasks-validator', '-d', + './path', '-v', '6.1'] + validator.main(args) + mock_file.called_with('./path') + mock_validator.assert_called_with(TASKS, '6.1') diff --git a/utils/fuel-tasklib/tasklib/validator.py b/utils/fuel-tasklib/tasklib/validator.py new file mode 100644 index 0000000000..7605fb1232 --- /dev/null +++ b/utils/fuel-tasklib/tasklib/validator.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import argparse +from fnmatch import fnmatch +import jsonschema +import logging +import os +import sys +import yaml + +from tasklib import graph +from tasklib.schemas import VERSIONS_SCHEMAS_MAP + +logging.basicConfig(level=logging.ERROR) +LOG = logging.getLogger(__name__) + + +class TasksValidator(object): + + def __init__(self, tasks, version): + self.version = version + self.tasks = tasks + self.graph = graph.DeploymentGraph(tasks) + + def validate_schema(self): + '''Validate tasks schema + + :raise: jsonschema.ValidationError + ''' + checker = jsonschema.FormatChecker() + schema = VERSIONS_SCHEMAS_MAP.get(self.version)().tasks_schema + jsonschema.validate(self.tasks, schema, format_checker=checker) + + def validate_unique_tasks(self): + '''Check if all tasks have unique ids + + :raise: ValueError when ids are duplicated + ''' + if not len(self.tasks) == len(set(task['id'] for task in self.tasks)): + raise ValueError("All tasks should have unique id") + + def validate_graph(self): + '''Validate graph if is executable completely by fuel nailgun + + :raise: ValueEror when one of requirements is not satisfied + ''' + msgs = [] + + # deployment graph should be without cycles + cycles = self.graph.find_cycles() + if len(cycles): + msgs.append('Graph is not acyclic. Cycles: {0}'.format(cycles)) + + # graph should be connected to execute all tasks + if not self.graph.is_connected(): + msgs.append('Graph is not connected.') + + # deployment graph should have filled all nodes + empty_nodes = self.graph.find_empty_nodes() + if len(empty_nodes): + msgs.append('Graph have empty nodes: {0}'.format(empty_nodes)) + + if msgs: + raise ValueError('Graph validation fail: {0}'.format(msgs)) + + +def get_files(base_dir, file_pattern='*tasks.yaml'): + for root, _dirs, files in os.walk(base_dir): + for file_name in files: + if fnmatch(file_name, file_pattern): + yield os.path.join(root, file_name) + + +def get_tasks(base_dir): + tasks = [] + for file_path in get_files(base_dir): + with open(file_path) as f: + LOG.debug('Reading tasks from file %s', file_path) + tasks.extend(yaml.load(f.read())) + return tasks + + +def main(args=sys.argv): + parser = argparse.ArgumentParser( + description='''Validator of tasks, gather all yaml files with name + contains tasks and read and validate tasks from them''') + parser.add_argument('-d', '--dir', dest='dir', required=True, + help='directory where tasks are localised') + parser.add_argument('-v', '--version', dest='ver', default='last', + metavar='VERSION', help='version of fuel for which ' + 'tasks should be validated') + parser.add_argument("--debug", dest="debug", action="store_true", + default=False, help="Enable debug mode") + args, _ = parser.parse_known_args(args) + + tasks = get_tasks(args.dir) + if args.debug: + LOG.setLevel(logging.DEBUG) + + if tasks: + t_validator = TasksValidator(tasks, args.ver) + t_validator.validate_schema() + t_validator.validate_unique_tasks() + t_validator.validate_graph() + else: + print("No tasks in the provided directory %s" % args.dir) + sys.exit(1) diff --git a/utils/fuel-tasklib/test-requirements.txt b/utils/fuel-tasklib/test-requirements.txt new file mode 100644 index 0000000000..a06846d961 --- /dev/null +++ b/utils/fuel-tasklib/test-requirements.txt @@ -0,0 +1,3 @@ +pytest>=2.6.4 +unittest2>=0.5.1 +mock==1.0 \ No newline at end of file diff --git a/utils/jenkins/tasks-test-requirements.txt b/utils/jenkins/tasks-test-requirements.txt index b5d5a59617..3586e3613c 100644 --- a/utils/jenkins/tasks-test-requirements.txt +++ b/utils/jenkins/tasks-test-requirements.txt @@ -1 +1,2 @@ --e git://git.openstack.org/stackforge/fuel-tasklib@9e41ce127f51361274a8d8e9af63da6f832ced71#egg=fuel-tasklib +#Relative to fuel-library base dir +-e file:./../fuel-tasklib#egg=tasklib diff --git a/utils/jenkins/tasks_run_tests.sh b/utils/jenkins/tasks_run_tests.sh index 27446d8d05..e58407ce07 100755 --- a/utils/jenkins/tasks_run_tests.sh +++ b/utils/jenkins/tasks_run_tests.sh @@ -16,10 +16,10 @@ set -eu -DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -WORK_DIR="${DIR}/../../deployment" - -type tasks-validator >/dev/null 2>&1 || { echo >&2 "fuel-tasks-validator is required to be installed to run tests."; exit 1; } - +ROOT=$(dirname $(readlink -f $0)) +WORK_DIR="${ROOT}/../../deployment" +UTILS_DIR="${ROOT}" +TOX_CONF="${ROOT}/tox.ini" +TOX_PYENVS=${TOX_PYENVS:-"tasks"} echo "Starting tasks-validator..." -tasks-validator -d $WORK_DIR +tox -e $TOX_PYENVS -c $TOX_CONF diff --git a/utils/jenkins/tox.ini b/utils/jenkins/tox.ini index 8c209d19a8..0390bb7014 100644 --- a/utils/jenkins/tox.ini +++ b/utils/jenkins/tox.ini @@ -5,7 +5,7 @@ [tox] minversion = 1.8 -envlist = pep8 +envlist = pep8,tasks skipsdist = True [testenv] @@ -16,6 +16,11 @@ commands = [testenv:venv] commands = {posargs:} +[testenv:tasks] +deps = -r{toxinidir}/tasks-test-requirements.txt +commands = + tasks-validator {posargs:-d ../../deployment} + [testenv:pep8] deps = hacking == 0.10.1 usedevelop = False