diff --git a/lower-constraints.txt b/lower-constraints.txt index 75b89b9ddd..a0c6381016 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -1,6 +1,7 @@ alabaster==0.7.10 alembic==0.8.10 amqp==2.1.1 +ansible-runner==1.4.2 aodhclient==0.9.0 appdirs==1.3.0 asn1crypto==0.23.0 diff --git a/test-ansible-requirements.txt b/test-ansible-requirements.txt new file mode 100644 index 0000000000..3e87cc19c0 --- /dev/null +++ b/test-ansible-requirements.txt @@ -0,0 +1,12 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +yaql>=1.1.3 # Apache 2.0 License +ansible>=2.8.6 # GPL +ansible-runner>=1.4.2 # Apache +ansi2html>=1.5.2 # GPL (soft-dependency of pytest-html) +pytest>=5.2.2 # MIT +pytest-ansible-playbook-runner>=0.0.2 # Apache-2.0 +pytest-cov>=2.8.1 # MIT +pytest-html>=1.22.0 # MPL 2.0 +pytest-xdist>=1.30.0 # MIT diff --git a/test-requirements.txt b/test-requirements.txt index c19bd2dc84..3365805b46 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -18,3 +18,4 @@ testtools>=2.2.0 # MIT mock>=2.0.0 # BSD oslotest>=3.2.0 # Apache-2.0 yaql>=1.1.3 # Apache 2.0 License +ansible-runner>=1.4.2 # Apache diff --git a/tools/render-ansible-tasks.py b/tools/render-ansible-tasks.py new file mode 100755 index 0000000000..1c3d0c3660 --- /dev/null +++ b/tools/render-ansible-tasks.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import argparse +import errno +import json +import os +import sys +import yaml +import yaql + + +def parse_opts(argv): + parser = argparse.ArgumentParser( + description='Render the Ansible tasks based in the role and the tags selected.' + 'Those tasks can be used for debugging or linting purposes.') + subp = parser.add_mutually_exclusive_group(required=True) + + parser.add_argument('--output', required=True, metavar='', + help="The folder to store the rendered tasks", + ) + + parser.add_argument('--ansible-tasks', nargs="+", required=True, + metavar='', + help='THT tags to filter the Ansible rendering ' + 'i.e. update_tasks') + + subp.add_argument('--roles-list', nargs="+", metavar='', + help='Composable roles to filter the Ansible rendering ' + 'i.e. Controller Compute') + + subp.add_argument('--all', action='store_true', + help='Process all services in the resource registry at once, ' + 'this allows to test all services templates avoiding ' + 'reading and generating all the files.') + + opts = parser.parse_args(argv[1:]) + return opts + + +def main(): + opts = parse_opts(sys.argv) + engine = yaql.factory.YaqlFactory().create() + output = opts.output + # We open the resource registry once + resource_registry = "./overcloud-resource-registry-puppet.yaml" + resource_reg = yaml.load(open(os.path.join(resource_registry), 'r')) + + if (opts.all): + # This means we will parse all the services defined + # by default in the resource registry + roles_list = ["overcloud-resource-registry-puppet"] + else: + roles_list = opts.roles_list + + for role in roles_list: + # We open the role file only once. + if (opts.all): + # The service definition will be the same resource registry + role_resources = resource_reg + else: + role_resources = yaml.load(open(os.path.join("./roles/", role + ".yaml"), 'r')) + + for section_task in opts.ansible_tasks: + if(opts.all): + # We get all the services in the resource_registry section + expression = engine( + "$.resource_registry" + ) + else: + expression = engine( + "$.ServicesDefault.flatten().distinct()" + ) + heat_resources = expression.evaluate(data=role_resources) + role_ansible_tasks = [] + + for resource in heat_resources: + if(opts.all): + # If we use the resource registry as the source of the + # data we need to split the service name of the + # service config definition + resource = resource.split(' ')[0] + expression = engine( + "$.resource_registry.get('" + resource + "')" + ) + config_file = expression.evaluate(data=resource_reg) + if(config_file is not None): + if('::' in config_file): + print("This is a nested Heat resource") + else: + data_source = yaml.load(open("./" + config_file, 'r')) + expression = engine( + "$.outputs.role_data.value.get(" + section_task + ").flatten().distinct()" + ) + try: + ansible_tasks = expression.evaluate(data=data_source) + print(ansible_tasks) + role_ansible_tasks = role_ansible_tasks + ansible_tasks + except Exception as e: + print("There are no tasks in the configuration file") + if (role_ansible_tasks != []): + tasks_output_file = os.path.join(output, role + "_" + section_task + ".yml") + if not os.path.exists(os.path.dirname(tasks_output_file)): + try: + os.makedirs(os.path.dirname(tasks_output_file)) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + save = open(tasks_output_file, 'w+') + yaml.dump(yaml.load(json.dumps(role_ansible_tasks)), save, default_flow_style=False) + +if __name__ == '__main__': + main() diff --git a/tox.ini b/tox.ini index d82378bd1c..1a869c2ed5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] minversion = 2.0 skipsdist = True -envlist = pep8,py27,py37 +envlist = pep8,py27,py37,tht [testenv] usedevelop = True @@ -91,3 +91,33 @@ deps = -c{toxinidir}/lower-constraints.txt -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt + +[testenv:tht] +basepython = python3 +usedevelop = True +setenv = + ANSIBLE_FORCE_COLOR=1 + ANSIBLE_INVENTORY={toxinidir}/test/hosts.ini + ANSIBLE_THT_FOLDER={toxinidir} + ANSIBLE_NOCOWS=1 + ANSIBLE_RETRY_FILES_ENABLED=0 + ANSIBLE_STDOUT_CALLBACK=debug + PY_COLORS=1 + VIRTUAL_ENV={envdir} + # Avoid 2020-01-01 warnings: https://github.com/pypa/pip/issues/6207 + PYTHONWARNINGS=ignore:DEPRECATION::pip._internal.cli.base_command + PIP_DISABLE_PIP_VERSION_CHECK=1 +passenv = + ANSIBLE_* +deps = + -r{toxinidir}/test-ansible-requirements.txt +whitelist_externals = + bash +commands_pre = + pip install -q bindep + bindep test +commands = + pytest --color=no \ + --html={envlogdir}/reports.html \ + --self-contained-html \ + {toxinidir}/tripleo_heat_templates/tests/test_tht_ansible_syntax.py diff --git a/tripleo_heat_templates/tests/test_tht_ansible_syntax.py b/tripleo_heat_templates/tests/test_tht_ansible_syntax.py new file mode 100644 index 0000000000..c9359cca27 --- /dev/null +++ b/tripleo_heat_templates/tests/test_tht_ansible_syntax.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import ansible_runner + + +def test_tht_ansible_syntax(pytestconfig): + + tht_root = str(pytestconfig.invocation_params.dir) + role_path = os.path.join(tht_root, + "tripleo_heat_templates/tests/roles/tripleo-ansible/tripleo-ansible/tripleo_ansible/roles") + play_path = os.path.join(tht_root, + "tripleo_heat_templates/tests/test_tht_ansible_syntax.yml") + + os.environ["ANSIBLE_ROLES_PATH"] = role_path + + run = ansible_runner.run( + playbook=play_path, + extravars={'tht_root': tht_root} + ) + + try: + assert run.rc == 0 + finally: + print("{}: {}".format(run.status, run.rc)) diff --git a/tripleo_heat_templates/tests/test_tht_ansible_syntax.yml b/tripleo_heat_templates/tests/test_tht_ansible_syntax.yml new file mode 100644 index 0000000000..360b6c6a9b --- /dev/null +++ b/tripleo_heat_templates/tests/test_tht_ansible_syntax.yml @@ -0,0 +1,109 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +- name: Run check + hosts: localhost + name: Render Ansible tasks for roles templates + vars: + # This variable is set to 9999, so we will not execute actually + # any task. Still, the Ansible interpreter will check for syntax + # issues wich is the intention of this playbook. + step: 9999 + # In the future this list should be extended to + # ANSIBLE_TASKS_SECTIONS defined in yaml-validate.py + # including also deployment tasks + tasks_list: > + update_tasks + post_update_tasks + external_update_tasks + upgrade_tasks + post_upgrade_tasks + external_upgrade_tasks + fast_forward_upgrade_tasks + fast_forward_post_upgrade_tasks + # In the future this list shoud be extended to support + # automatically any role definition in t-h-t/roles/* + # Currently we have a --all option check allservices + # in the resource registry + roles_list: > + Compute + tasks: + - name: set basic tht folder path + fail: + msg: >- + The variable `tht_root` set this option and try again. On + the CLI this can be defined with "-e tht_root=/path/to/tht" + when: + - tht_root is undefined + + - name: Set temp dir var + set_fact: + tmp_folder: "{{ tht_root }}/../tht-rendered" + + - name: Get Ansible Galaxy roles + command: >- + ansible-galaxy install + --roles-path {{ tht_root }}/tripleo_heat_templates/tests/roles/ + -fr + {{ tht_root }}/tripleo_heat_templates/tests/tht-role-requirements.yml + + - name: Create temp folder + file: + state: directory + path: "{{ tmp_folder }}" + + # This task will render all the jinja templates in t-h-t. + - name: Process templates + command: > + python {{ tht_root }}/tools/process-templates.py \ + -r {{ tht_root }}/roles_data.yaml \ + -o {{ tmp_folder }} + args: + chdir: "{{ tht_root }}" + + # This task will call the render tool based on the tasks list + # using all the services defined by default in the resource registry + - name: Render the ansible tasks per role + command: > + python {{ tht_root }}/tools/render-ansible-tasks.py \ + --output {{ tmp_folder }}/rendered-tasks/ \ + --ansible-tasks {{ tasks_list}} \ + --all + args: + chdir: "{{ tmp_folder }}" + + # To check changes in the tool we check the script passing 1 task type and + # 1 role + - name: Render the ansible tasks for 1 role and 1 task type as an example + command: > + python {{ tht_root }}/tools/render-ansible-tasks.py \ + --output {{ tmp_folder }}/rendered-tasks/ \ + --ansible-tasks update_tasks \ + --roles-list Compute + args: + chdir: "{{ tmp_folder }}" + + - name: Find rendered Ansible tasks + find: + paths: "{{ tmp_folder }}/rendered-tasks/" + patterns: "*.yml" + recurse: false + register: find_result + + - name: Import rendered Ansible tasks + include_tasks: "{{ item.path }}" + with_items: "{{ find_result.files }}" diff --git a/tripleo_heat_templates/tests/tht-role-requirements.yml b/tripleo_heat_templates/tests/tht-role-requirements.yml new file mode 100644 index 0000000000..b07642f4cf --- /dev/null +++ b/tripleo_heat_templates/tests/tht-role-requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: tripleo-ansible + scm: git + src: https://opendev.org/openstack/tripleo-ansible + version: master + trackbranch: master diff --git a/zuul.d/layout.yaml b/zuul.d/layout.yaml index d41f493a3c..89848fafec 100644 --- a/zuul.d/layout.yaml +++ b/zuul.d/layout.yaml @@ -14,6 +14,7 @@ - tripleo-ci-centos-7-containers-multinode: dependencies: &deps_unit_lint - openstack-tox-pep8 + - openstack-tox-tht - tripleo-ci-centos-7-undercloud-containers: dependencies: *deps_unit_lint - tripleo-ci-centos-7-standalone: @@ -87,6 +88,7 @@ dependencies: *deps_unit_lint - tripleo-ci-centos-7-containerized-undercloud-upgrades: dependencies: *deps_unit_lint + - openstack-tox-tht gate: queue: tripleo jobs: @@ -135,3 +137,17 @@ - ^deployed-server/.*$ - ^common/.*$ - zuul.d/* +- job: + name: openstack-tox-tht + parent: openstack-tox + description: Runs syntax tht tests. Uses tox with the ``tht`` environment. + success-url: "reports.html" + failure-url: "reports.html" + voting: true + vars: + tox_envlist: tht + bindep_profile: test tht + test_setup_skip: true + files: + - ^((docker|puppet)/services|deployment)/.*$ + - tools/*