Test install-deps.sh script using pytest/molecule

Adds one molecule scenario that deploys install-deps.sh on targeted
platforms and runs it there.

Job uses pytest for orchestration but developers have the freedom
of calling molecule directly if they want: `molecule test --all`.

Change-Id: I465891dd600ca0353a4124f46adc13259a527aa0
Task: https://tree.taiga.io/project/tripleo-ci-board/task/1021
Depends-On: https://review.opendev.org/#/c/663599/

molecule/Dockerfile.j2

@@ -0,0 +1,21 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]

molecule/install-deps/molecule.yml

@@ -0,0 +1,38 @@
+---
+driver:
+  name: docker
+log: true
+platforms:
+
+  - name: centos7-master
+    hostname: centos7-master
+    image: centos:7
+    dockerfile: ../Dockerfile.j2
+    pkg_extras: python-setuptools
+    easy_install:
+      - pip
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+
+  - name: fedora28-master
+    hostname: fedora28-master
+    image: fedora:28
+    dockerfile: ../Dockerfile.j2
+    pkg_extras: python*-setuptools
+    environment:
+      <<: *env
+
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - destroy
+lint:
+  enabled: false

molecule/install-deps/playbook.yml

@@ -0,0 +1,18 @@
+---
+
+- name: Converge
+  hosts: all
+  tasks:
+
+    - name: deploy install-deps.sh
+      copy:
+        src: ../../install-deps.sh
+        dest: install-deps.sh
+        mode: 0744
+
+    - name: "runs install-deps.sh"
+      shell: bash -x install-deps.sh
+      args:
+        warn: false
+      tags:
+        - skip_ansible_lint

setup.cfg

@@ -43,3 +43,7 @@ skip_changelog = True
 show-source = True
 ignore = E123,E125
 builtins = _
+
+[tool:pytest]
+ignore = .tox dist
+addopts = -x

tests/hosts.ini

@@ -0,0 +1 @@
+localhost

tests/test_molecule.py

@@ -0,0 +1,57 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import os
+import subprocess
+
+import docker
+
+
+client = docker.from_env(timeout=5)
+if not client.ping():
+    raise Exception("Failed to ping docker server.")
+
+try:
+    import selinux  # noqa
+except Exception as e:
+    logging.error(
+        "It appears that you are trying to use "
+        "molecule with a Python interpreter that does not have the libselinux "
+        "python bindings installed. These can only be installed using your "
+        "distro package manager and are specific to each python version. "
+        "Common package names: libselinux-python python2-libselinux "
+        "python3-libselinux")
+    raise e
+
+
+def pytest_generate_tests(metafunc):
+    # detects all molecule scenarios inside the project
+    matches = []
+    for filename in subprocess.check_output(
+            "find . -path '*/.*' -prune -o -name 'molecule.yml' -print",
+            shell=True, universal_newlines=True).split():
+        role_path = os.path.abspath(os.path.join(filename, os.pardir))
+        x = os.path.basename(role_path)
+        role_path = os.path.abspath(os.path.join(role_path,
+                                                 os.pardir,
+                                                 os.pardir))
+        matches.append([role_path, x])
+    metafunc.parametrize('testdata', matches)
+
+
+def test_molecule(testdata):
+    cwd, scenario = testdata
+    cmd = ['python', '-m', 'molecule', 'test', '-s', scenario]
+    print("running: %s (from %s)" % (" " .join(cmd), cwd))
+    r = subprocess.call(cmd, cwd=cwd)
+    assert r == 0

tox.ini

@@ -1,12 +1,32 @@
 [tox]
-minversion = 2.0
-envlist = docs, linters
-skipdist = True
+envlist = docs, linters, molecule
+minversion = 3.4.0
+ignore_basepython_conflict = True
+skip_missing_interpreters = True
+skipsdist = True
 
 [testenv]
 usedevelop = True
 install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt} {opts} {packages}
-setenv = VIRTUAL_ENV={envdir}
+setenv =
+    ANSIBLE_FORCE_COLOR={env:ANSIBLE_FORCE_COLOR:1}
+    ANSIBLE_INVENTORY={toxinidir}/tests/hosts.ini
+    ANSIBLE_NOCOWS=1
+    ANSIBLE_RETRY_FILES_ENABLED=0
+    ANSIBLE_STDOUT_CALLBACK={env:ANSIBLE_STDOUT_CALLBACK:debug}
+    ANSIBLE_VERBOSITY={env:ANSIBLE_VERBOSITY:1}
+    PY_COLORS={env:PY_COLORS:1}
+    VIRTUAL_ENV={envdir}
+    # pip: Avoid 2020-01-01 warnings: https://github.com/pypa/pip/issues/6207
+    # paramiko CryptographyDeprecationWarning: https://github.com/ansible/ansible/issues/52598
+    PYTHONWARNINGS=ignore:DEPRECATION::pip._internal.cli.base_command,ignore::UserWarning
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+passenv =
+    ANSIBLE_*
+    DOCKER_*
+    MOLECULE_*
+    SSH_AUTH_SOCK
+    TERM
 deps = -r{toxinidir}/test-requirements.txt
 whitelist_externals =
     bash
@@ -30,6 +50,24 @@ envdir = {toxworkdir}/linters
 commands =
     python -m pre_commit run bashate -a
 
+[testenv:molecule]
+# basepython forces tox to use system python and avoid other python versions like pyenv ones
+# because those are unlikely to have the required libselinux library available.
+basepython=python
+deps =
+    ansi2html
+    docker>=4.0.1
+    paramiko>=2.5.0
+    pytest
+    pytest-cov
+    pytest-html
+    pytest-xdist
+    mock
+    molecule>=2.22rc1
+    selinux>=0.1.5rc1
+commands =
+    python -m pytest --color=yes --html={envlogdir}/reports.html --self-contained-html {tty:-s} {posargs:tests}
+
 [testenv:pep8]
 envdir = {toxworkdir}/linters
 commands =

zuul.d/layout.yaml

@@ -7,6 +7,7 @@
       - publish-openstack-docs-pti
       - release-notes-jobs-python3
       - tripleo-standalone-scenarios-full
+      - openstack-tox-molecule
     check:
       jobs:
         - tripleo-ci-centos-7-scenario007-multinode-oooq-container: