Merge "Ansible lint check in THT"

lower-constraints.txt

@@ -1,6 +1,7 @@
 alabaster==0.7.10
 alembic==0.8.10
 amqp==2.1.1
+ansible-runner==1.4.2
 aodhclient==0.9.0
 appdirs==1.3.0
 asn1crypto==0.23.0

test-ansible-requirements.txt

@@ -0,0 +1,12 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+yaql>=1.1.3 # Apache 2.0 License
+ansible>=2.8.6 # GPL
+ansible-runner>=1.4.2 # Apache
+ansi2html>=1.5.2  # GPL (soft-dependency of pytest-html)
+pytest>=5.2.2 # MIT
+pytest-ansible-playbook-runner>=0.0.2 # Apache-2.0
+pytest-cov>=2.8.1 # MIT
+pytest-html>=1.22.0 # MPL 2.0
+pytest-xdist>=1.30.0 # MIT

test-requirements.txt

@@ -18,3 +18,4 @@ testtools>=2.2.0 # MIT
 mock>=2.0.0 # BSD
 oslotest>=3.2.0 # Apache-2.0
 yaql>=1.1.3 # Apache 2.0 License
+ansible-runner>=1.4.2 # Apache

tools/render-ansible-tasks.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+#
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+import argparse
+import errno
+import json
+import os
+import sys
+import yaml
+import yaql
+
+
+def parse_opts(argv):
+    parser = argparse.ArgumentParser(
+        description='Render the Ansible tasks based in the role and the tags selected.'
+                    'Those tasks can be used for debugging or linting purposes.')
+    subp = parser.add_mutually_exclusive_group(required=True)
+
+    parser.add_argument('--output', required=True, metavar='<tasks directory output>',
+                        help="The folder to store the rendered tasks",
+                        )
+
+    parser.add_argument('--ansible-tasks', nargs="+", required=True,
+                        metavar='<ansible tasks to be rendered>',
+                        help='THT tags to filter the Ansible rendering '
+                             'i.e. update_tasks')
+
+    subp.add_argument('--roles-list', nargs="+", metavar='<list of roles to render>',
+                      help='Composable roles to filter the Ansible rendering '
+                           'i.e. Controller Compute')
+
+    subp.add_argument('--all', action='store_true',
+                      help='Process all services in the resource registry at once, '
+                           'this allows to test all services templates avoiding '
+                             'reading and generating all the files.')
+
+    opts = parser.parse_args(argv[1:])
+    return opts
+
+
+def main():
+    opts = parse_opts(sys.argv)
+    engine = yaql.factory.YaqlFactory().create()
+    output = opts.output
+    # We open the resource registry once
+    resource_registry = "./overcloud-resource-registry-puppet.yaml"
+    resource_reg = yaml.load(open(os.path.join(resource_registry), 'r'))
+
+    if (opts.all):
+        # This means we will parse all the services defined
+        # by default in the resource registry
+        roles_list = ["overcloud-resource-registry-puppet"]
+    else:
+        roles_list = opts.roles_list
+
+    for role in roles_list:
+        # We open the role file only once.
+        if (opts.all):
+            # The service definition will be the same resource registry
+            role_resources = resource_reg
+        else:
+            role_resources = yaml.load(open(os.path.join("./roles/", role + ".yaml"), 'r'))
+
+        for section_task in opts.ansible_tasks:
+            if(opts.all):
+                # We get all the services in the resource_registry section
+                expression = engine(
+                    "$.resource_registry"
+                )
+            else:
+                expression = engine(
+                    "$.ServicesDefault.flatten().distinct()"
+                )
+            heat_resources = expression.evaluate(data=role_resources)
+            role_ansible_tasks = []
+
+            for resource in heat_resources:
+                if(opts.all):
+                    # If we use the resource registry as the source of the
+                    # data we need to split the service name of the
+                    # service config definition
+                    resource = resource.split(' ')[0]
+                expression = engine(
+                  "$.resource_registry.get('" + resource + "')"
+                )
+                config_file = expression.evaluate(data=resource_reg)
+                if(config_file is not None):
+                    if('::' in config_file):
+                        print("This is a nested Heat resource")
+                    else:
+                        data_source = yaml.load(open("./" + config_file, 'r'))
+                        expression = engine(
+                          "$.outputs.role_data.value.get(" + section_task + ").flatten().distinct()"
+                        )
+                        try:
+                            ansible_tasks = expression.evaluate(data=data_source)
+                            print(ansible_tasks)
+                            role_ansible_tasks = role_ansible_tasks + ansible_tasks
+                        except Exception as e:
+                            print("There are no tasks in the configuration file")
+            if (role_ansible_tasks != []):
+                tasks_output_file = os.path.join(output, role + "_" + section_task + ".yml")
+                if not os.path.exists(os.path.dirname(tasks_output_file)):
+                    try:
+                        os.makedirs(os.path.dirname(tasks_output_file))
+                    except OSError as exc:
+                        if exc.errno != errno.EEXIST:
+                            raise
+                save = open(tasks_output_file, 'w+')
+                yaml.dump(yaml.load(json.dumps(role_ansible_tasks)), save, default_flow_style=False)
+
+if __name__ == '__main__':
+    main()

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 minversion = 2.0
 skipsdist = True
-envlist = pep8,py27,py37
+envlist = pep8,py27,py37,tht
 
 [testenv]
 usedevelop = True
@@ -91,3 +91,33 @@ deps =
   -c{toxinidir}/lower-constraints.txt
   -r{toxinidir}/test-requirements.txt
   -r{toxinidir}/requirements.txt
+
+[testenv:tht]
+basepython = python3
+usedevelop = True
+setenv =
+    ANSIBLE_FORCE_COLOR=1
+    ANSIBLE_INVENTORY={toxinidir}/test/hosts.ini
+    ANSIBLE_THT_FOLDER={toxinidir}
+    ANSIBLE_NOCOWS=1
+    ANSIBLE_RETRY_FILES_ENABLED=0
+    ANSIBLE_STDOUT_CALLBACK=debug
+    PY_COLORS=1
+    VIRTUAL_ENV={envdir}
+    # Avoid 2020-01-01 warnings: https://github.com/pypa/pip/issues/6207
+    PYTHONWARNINGS=ignore:DEPRECATION::pip._internal.cli.base_command
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+passenv =
+    ANSIBLE_*
+deps =
+  -r{toxinidir}/test-ansible-requirements.txt
+whitelist_externals =
+    bash
+commands_pre =
+    pip install -q bindep
+    bindep test
+commands =
+    pytest --color=no \
+           --html={envlogdir}/reports.html \
+           --self-contained-html \
+           {toxinidir}/tripleo_heat_templates/tests/test_tht_ansible_syntax.py

tripleo_heat_templates/tests/test_tht_ansible_syntax.py

@@ -0,0 +1,37 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import ansible_runner
+
+
+def test_tht_ansible_syntax(pytestconfig):
+
+    tht_root = str(pytestconfig.invocation_params.dir)
+    role_path = os.path.join(tht_root,
+                             "tripleo_heat_templates/tests/roles/tripleo-ansible/tripleo-ansible/tripleo_ansible/roles")
+    play_path = os.path.join(tht_root,
+                             "tripleo_heat_templates/tests/test_tht_ansible_syntax.yml")
+
+    os.environ["ANSIBLE_ROLES_PATH"] = role_path
+
+    run = ansible_runner.run(
+        playbook=play_path,
+        extravars={'tht_root': tht_root}
+    )
+
+    try:
+        assert run.rc == 0
+    finally:
+        print("{}: {}".format(run.status, run.rc))

tripleo_heat_templates/tests/test_tht_ansible_syntax.yml

@@ -0,0 +1,109 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Run check
+  hosts: localhost
+  name: Render Ansible tasks for roles templates
+  vars:
+    # This variable is set to 9999, so we will not execute actually
+    # any task. Still, the Ansible interpreter will check for syntax
+    # issues wich is the intention of this playbook.
+    step: 9999
+    # In the future this list should be extended to
+    # ANSIBLE_TASKS_SECTIONS defined in yaml-validate.py
+    # including also deployment tasks
+    tasks_list: >
+      update_tasks
+      post_update_tasks
+      external_update_tasks
+      upgrade_tasks
+      post_upgrade_tasks
+      external_upgrade_tasks
+      fast_forward_upgrade_tasks
+      fast_forward_post_upgrade_tasks
+    # In the future this list shoud be extended to support
+    # automatically any role definition in t-h-t/roles/*
+    # Currently we have a --all option check allservices
+    # in the resource registry
+    roles_list: >
+      Compute
+  tasks:
+  - name: set basic tht folder path
+    fail:
+      msg: >-
+        The variable `tht_root` set this option and try again. On
+        the CLI this can be defined with "-e tht_root=/path/to/tht"
+    when:
+      - tht_root is undefined
+
+  - name: Set temp dir var
+    set_fact:
+      tmp_folder: "{{ tht_root }}/../tht-rendered"
+
+  - name: Get Ansible Galaxy roles
+    command: >-
+      ansible-galaxy install
+      --roles-path {{ tht_root }}/tripleo_heat_templates/tests/roles/
+      -fr
+      {{ tht_root }}/tripleo_heat_templates/tests/tht-role-requirements.yml
+
+  - name: Create temp folder
+    file:
+      state: directory
+      path: "{{ tmp_folder }}"
+
+  # This task will render all the jinja templates in t-h-t.
+  - name: Process templates
+    command: >
+      python {{ tht_root }}/tools/process-templates.py \
+          -r {{ tht_root }}/roles_data.yaml \
+          -o {{ tmp_folder }}
+    args:
+      chdir: "{{ tht_root }}"
+
+  # This task will call the render tool based on the tasks list
+  # using all the services defined by default in the resource registry
+  - name: Render the ansible tasks per role
+    command: >
+      python {{ tht_root }}/tools/render-ansible-tasks.py \
+          --output {{ tmp_folder }}/rendered-tasks/ \
+          --ansible-tasks {{ tasks_list}} \
+          --all
+    args:
+      chdir: "{{ tmp_folder }}"
+
+  # To check changes in the tool we check the script passing 1 task type and
+  # 1 role
+  - name: Render the ansible tasks for 1 role and 1 task type as an example
+    command: >
+      python {{ tht_root }}/tools/render-ansible-tasks.py \
+          --output {{ tmp_folder }}/rendered-tasks/ \
+          --ansible-tasks update_tasks \
+          --roles-list Compute
+    args:
+      chdir: "{{ tmp_folder }}"
+
+  - name: Find rendered Ansible tasks
+    find:
+      paths: "{{ tmp_folder }}/rendered-tasks/"
+      patterns: "*.yml"
+      recurse: false
+    register: find_result
+
+  - name: Import rendered Ansible tasks
+    include_tasks: "{{ item.path }}"
+    with_items: "{{ find_result.files }}"

tripleo_heat_templates/tests/tht-role-requirements.yml

@@ -0,0 +1,21 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: tripleo-ansible
+  scm: git
+  src: https://opendev.org/openstack/tripleo-ansible
+  version: master
+  trackbranch: master

zuul.d/layout.yaml

@@ -14,6 +14,7 @@
         - tripleo-ci-centos-7-containers-multinode:
             dependencies: &deps_unit_lint
               - openstack-tox-pep8
+              - openstack-tox-tht
         - tripleo-ci-centos-7-undercloud-containers:
             dependencies: *deps_unit_lint
         - tripleo-ci-centos-7-standalone:
@@ -87,6 +88,7 @@
             dependencies: *deps_unit_lint
         - tripleo-ci-centos-7-containerized-undercloud-upgrades:
             dependencies: *deps_unit_lint
+        - openstack-tox-tht
     gate:
       queue: tripleo
       jobs:
@@ -135,3 +137,17 @@
               - ^deployed-server/.*$
               - ^common/.*$
               - zuul.d/*
+- job:
+    name: openstack-tox-tht
+    parent: openstack-tox
+    description: Runs syntax tht tests. Uses tox with the ``tht`` environment.
+    success-url: "reports.html"
+    failure-url: "reports.html"
+    voting: true
+    vars:
+      tox_envlist: tht
+      bindep_profile: test tht
+      test_setup_skip: true
+    files:
+      - ^((docker|puppet)/services|deployment)/.*$
+      - tools/*