openstack
/
tripleo-heat-templates
Code Issues Proposed changes

Merge "Ansible lint check in THT"

This commit is contained in:
Zuul 2019-11-07 05:08:38 +00:00 committed by Gerrit Code Review
parent aff1b2ba49 0de387ee3c
commit 8fad46f0d3
9 changed files with 355 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lower-constraints.txt
View File

@ -1,6 +1,7 @@
alabaster==0.7.10
alembic==0.8.10
amqp==2.1.1
ansible-runner==1.4.2
aodhclient==0.9.0
appdirs==1.3.0
asn1crypto==0.23.0

12
test-ansible-requirements.txt Normal file
View File

@ -0,0 +1,12 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
yaql>=1.1.3 # Apache 2.0 License
ansible>=2.8.6 # GPL
ansible-runner>=1.4.2 # Apache
ansi2html>=1.5.2 # GPL (soft-dependency of pytest-html)
pytest>=5.2.2 # MIT
pytest-ansible-playbook-runner>=0.0.2 # Apache-2.0
pytest-cov>=2.8.1 # MIT
pytest-html>=1.22.0 # MPL 2.0
pytest-xdist>=1.30.0 # MIT

1
test-requirements.txt
View File

@ -18,3 +18,4 @@ testtools>=2.2.0 # MIT
mock>=2.0.0 # BSD
oslotest>=3.2.0 # Apache-2.0
yaql>=1.1.3 # Apache 2.0 License
ansible-runner>=1.4.2 # Apache

127
tools/render-ansible-tasks.py Executable file
View File

@ -0,0 +1,127 @@
#!/usr/bin/env python
#
# Copyright 2019 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import argparse
import errno
import json
import os
import sys
import yaml
import yaql
def parse_opts(argv):
parser = argparse.ArgumentParser(
description='Render the Ansible tasks based in the role and the tags selected.'
'Those tasks can be used for debugging or linting purposes.')
subp = parser.add_mutually_exclusive_group(required=True)
parser.add_argument('--output', required=True, metavar='<tasks directory output>',
help="The folder to store the rendered tasks",
)
parser.add_argument('--ansible-tasks', nargs="+", required=True,
metavar='<ansible tasks to be rendered>',
help='THT tags to filter the Ansible rendering '
'i.e. update_tasks')
subp.add_argument('--roles-list', nargs="+", metavar='<list of roles to render>',
help='Composable roles to filter the Ansible rendering '
'i.e. Controller Compute')
subp.add_argument('--all', action='store_true',
help='Process all services in the resource registry at once, '
'this allows to test all services templates avoiding '
'reading and generating all the files.')
opts = parser.parse_args(argv[1:])
return opts
def main():
opts = parse_opts(sys.argv)
engine = yaql.factory.YaqlFactory().create()
output = opts.output
# We open the resource registry once
resource_registry = "./overcloud-resource-registry-puppet.yaml"
resource_reg = yaml.load(open(os.path.join(resource_registry), 'r'))
if (opts.all):
# This means we will parse all the services defined
# by default in the resource registry
roles_list = ["overcloud-resource-registry-puppet"]
else:
roles_list = opts.roles_list
for role in roles_list:
# We open the role file only once.
if (opts.all):
# The service definition will be the same resource registry
role_resources = resource_reg
else:
role_resources = yaml.load(open(os.path.join("./roles/", role + ".yaml"), 'r'))
for section_task in opts.ansible_tasks:
if(opts.all):
# We get all the services in the resource_registry section
expression = engine(
"$.resource_registry"
)
else:
expression = engine(
"$.ServicesDefault.flatten().distinct()"
)
heat_resources = expression.evaluate(data=role_resources)
role_ansible_tasks = []
for resource in heat_resources:
if(opts.all):
# If we use the resource registry as the source of the
# data we need to split the service name of the
# service config definition
resource = resource.split(' ')[0]
expression = engine(
"$.resource_registry.get('" + resource + "')"
)
config_file = expression.evaluate(data=resource_reg)
if(config_file is not None):
if('::' in config_file):
print("This is a nested Heat resource")
else:
data_source = yaml.load(open("./" + config_file, 'r'))
expression = engine(
"$.outputs.role_data.value.get(" + section_task + ").flatten().distinct()"
)
try:
ansible_tasks = expression.evaluate(data=data_source)
print(ansible_tasks)
role_ansible_tasks = role_ansible_tasks + ansible_tasks
except Exception as e:
print("There are no tasks in the configuration file")
if (role_ansible_tasks != []):
tasks_output_file = os.path.join(output, role + "_" + section_task + ".yml")
if not os.path.exists(os.path.dirname(tasks_output_file)):
try:
os.makedirs(os.path.dirname(tasks_output_file))
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
save = open(tasks_output_file, 'w+')
yaml.dump(yaml.load(json.dumps(role_ansible_tasks)), save, default_flow_style=False)
if __name__ == '__main__':
main()

32
tox.ini
View File

@ -1,7 +1,7 @@
[tox]
minversion = 2.0
skipsdist = True
envlist = pep8,py27,py37
envlist = pep8,py27,py37,tht
[testenv]
usedevelop = True
@ -91,3 +91,33 @@ deps =
-c{toxinidir}/lower-constraints.txt
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/requirements.txt
[testenv:tht]
basepython = python3
usedevelop = True
setenv =
ANSIBLE_FORCE_COLOR=1
ANSIBLE_INVENTORY={toxinidir}/test/hosts.ini
ANSIBLE_THT_FOLDER={toxinidir}
ANSIBLE_NOCOWS=1
ANSIBLE_RETRY_FILES_ENABLED=0
ANSIBLE_STDOUT_CALLBACK=debug
PY_COLORS=1
VIRTUAL_ENV={envdir}
# Avoid 2020-01-01 warnings: https://github.com/pypa/pip/issues/6207
PYTHONWARNINGS=ignore:DEPRECATION::pip._internal.cli.base_command
PIP_DISABLE_PIP_VERSION_CHECK=1
passenv =
ANSIBLE_*
deps =
-r{toxinidir}/test-ansible-requirements.txt
whitelist_externals =
bash
commands_pre =
pip install -q bindep
bindep test
commands =
pytest --color=no \
--html={envlogdir}/reports.html \
--self-contained-html \
{toxinidir}/tripleo_heat_templates/tests/test_tht_ansible_syntax.py

37
tripleo_heat_templates/tests/test_tht_ansible_syntax.py Normal file
View File

@ -0,0 +1,37 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import ansible_runner
def test_tht_ansible_syntax(pytestconfig):
tht_root = str(pytestconfig.invocation_params.dir)
role_path = os.path.join(tht_root,
"tripleo_heat_templates/tests/roles/tripleo-ansible/tripleo-ansible/tripleo_ansible/roles")
play_path = os.path.join(tht_root,
"tripleo_heat_templates/tests/test_tht_ansible_syntax.yml")
os.environ["ANSIBLE_ROLES_PATH"] = role_path
run = ansible_runner.run(
playbook=play_path,
extravars={'tht_root': tht_root}
)
try:
assert run.rc == 0
finally:
print("{}: {}".format(run.status, run.rc))

109
tripleo_heat_templates/tests/test_tht_ansible_syntax.yml Normal file
View File

@ -0,0 +1,109 @@
---
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: Run check
hosts: localhost
name: Render Ansible tasks for roles templates
vars:
# This variable is set to 9999, so we will not execute actually
# any task. Still, the Ansible interpreter will check for syntax
# issues wich is the intention of this playbook.
step: 9999
# In the future this list should be extended to
# ANSIBLE_TASKS_SECTIONS defined in yaml-validate.py
# including also deployment tasks
tasks_list: >
update_tasks
post_update_tasks
external_update_tasks
upgrade_tasks
post_upgrade_tasks
external_upgrade_tasks
fast_forward_upgrade_tasks
fast_forward_post_upgrade_tasks
# In the future this list shoud be extended to support
# automatically any role definition in t-h-t/roles/*
# Currently we have a --all option check allservices
# in the resource registry
roles_list: >
Compute
tasks:
- name: set basic tht folder path
fail:
msg: >-
The variable `tht_root` set this option and try again. On
the CLI this can be defined with "-e tht_root=/path/to/tht"
when:
- tht_root is undefined
- name: Set temp dir var
set_fact:
tmp_folder: "{{ tht_root }}/../tht-rendered"
- name: Get Ansible Galaxy roles
command: >-
ansible-galaxy install
--roles-path {{ tht_root }}/tripleo_heat_templates/tests/roles/
-fr
{{ tht_root }}/tripleo_heat_templates/tests/tht-role-requirements.yml
- name: Create temp folder
file:
state: directory
path: "{{ tmp_folder }}"
# This task will render all the jinja templates in t-h-t.
- name: Process templates
command: >
python {{ tht_root }}/tools/process-templates.py \
-r {{ tht_root }}/roles_data.yaml \
-o {{ tmp_folder }}
args:
chdir: "{{ tht_root }}"
# This task will call the render tool based on the tasks list
# using all the services defined by default in the resource registry
- name: Render the ansible tasks per role
command: >
python {{ tht_root }}/tools/render-ansible-tasks.py \
--output {{ tmp_folder }}/rendered-tasks/ \
--ansible-tasks {{ tasks_list}} \
--all
args:
chdir: "{{ tmp_folder }}"
# To check changes in the tool we check the script passing 1 task type and
# 1 role
- name: Render the ansible tasks for 1 role and 1 task type as an example
command: >
python {{ tht_root }}/tools/render-ansible-tasks.py \
--output {{ tmp_folder }}/rendered-tasks/ \
--ansible-tasks update_tasks \
--roles-list Compute
args:
chdir: "{{ tmp_folder }}"
- name: Find rendered Ansible tasks
find:
paths: "{{ tmp_folder }}/rendered-tasks/"
patterns: "*.yml"
recurse: false
register: find_result
- name: Import rendered Ansible tasks
include_tasks: "{{ item.path }}"
with_items: "{{ find_result.files }}"

21
tripleo_heat_templates/tests/tht-role-requirements.yml Normal file
View File

@ -0,0 +1,21 @@
---
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: tripleo-ansible
scm: git
src: https://opendev.org/openstack/tripleo-ansible
version: master
trackbranch: master

16
zuul.d/layout.yaml
View File

@ -14,6 +14,7 @@
- tripleo-ci-centos-7-containers-multinode:
dependencies: &deps_unit_lint
- openstack-tox-pep8
- openstack-tox-tht
- tripleo-ci-centos-7-undercloud-containers:
dependencies: *deps_unit_lint
- tripleo-ci-centos-7-standalone:
@ -87,6 +88,7 @@
dependencies: *deps_unit_lint
- tripleo-ci-centos-7-containerized-undercloud-upgrades:
dependencies: *deps_unit_lint
- openstack-tox-tht
gate:
queue: tripleo
jobs:
@ -135,3 +137,17 @@
- ^deployed-server/.*$
- ^common/.*$
- zuul.d/*
- job:
name: openstack-tox-tht
parent: openstack-tox
description: Runs syntax tht tests. Uses tox with the ``tht`` environment.
success-url: "reports.html"
failure-url: "reports.html"
voting: true
vars:
tox_envlist: tht
bindep_profile: test tht
test_setup_skip: true
files:
- ^((docker|puppet)/services|deployment)/.*$
- tools/*