Override tox requirments with zuul git repos

It's a common desire to be able to have a job that runs unittests
against the git checkout of another repository, but getting the tox
environment into that shape is a bunch of extra work.

Now that we have a defined place where repos go and they're always on
the build node, we can look in the source dir for git repos that contain
python packages and ask setup.py for the name of the python package they
provide.

We can then see what packages tox decided to install for this
environment, see if we have any matching ones in the source code repos
we've put on disk and if so we can re-install those depends from the
source location.

That way we can cause a tox job to use a second repo for cross-repo
unittesting simply by adding that project to required_projects.

Add a flag to disable the behavior ... although the easiest way to
disable the behavior is to just not list other projects in
required_projects.

Change-Id: Ia5250c11b1d73baaa70ea1cef7ea1ba4d5bab821
Story: 2001136
Task: 4852

playbooks/tox/pre.yaml

@@ -1,3 +1,5 @@
 - hosts: all
   roles:
     - ensure-tox
+    - role: tox-siblings
+      when: tox_install_siblings

roles/tox-siblings/README.rst

@@ -0,0 +1,24 @@
+Installs python packages from other Zuul repos into a tox environment.
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_workdir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory to run tox in.
+
+.. zuul:rolevar:: tox_envlist
+   :default: venv
+
+   Which tox environment to run. Defaults to 'venv'.
+
+.. zuul:rolevar:: tox_executable
+   :default: tox
+
+   Location of the tox executable. Defaults to 'tox'.
+
+.. zuul:rolevar:: tox_install_siblings
+   :default: true
+
+   Flag controlling whether to attempt to install python packages from any
+   other source code repos zuul has checked out. Defaults to True.

roles/tox-siblings/defaults/main.yaml

@@ -0,0 +1,6 @@
+---
+tox_envlist: venv
+tox_executable: tox
+tox_install_siblings: true
+
+zuul_work_dir: "{{ zuul.project.src_dir }}"

roles/tox-siblings/library/tox_install_sibling_packages.py

@@ -0,0 +1,162 @@
+#!/usr/bin/python
+
+# Copyright (c) 2017 Red Hat
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+module: tox_install_sibling_packages
+short_description: Install packages needed by tox that have local git versions
+author: Monty Taylor (@mordred)
+description:
+  - Looks for git repositories that zuul has placed on the system that provide
+    python packages needed by package tox is testing. If if finds any, it will
+    install them into the tox virtualenv so that subsequent runs of tox will
+    use the provided git versions.
+requirements:
+  - "python >= 3.5"
+options:
+  tox_envlist:
+    description:
+      - The tox environment to operate in.
+    required: true
+    type: str
+  project_dir:
+    description:
+      - The directory in which the project we care about is in.
+    required: true
+    type: str
+  projects:
+    description:
+      - A list of project dicts that zuul knows about
+    required: true
+    type: list
+'''
+
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
+import os
+import pip
+import subprocess
+import tempfile
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def get_sibling_python_packages(projects):
+    '''Finds all python packages that zuul has cloned.
+
+    If someone does a require_project: and then runs a tox job, it can be
+    assumed that what they want to do is to test the two together.
+    '''
+    packages = {}
+
+    for project in projects:
+        root = project['src_dir']
+        setup_cfg = os.path.join(root, 'setup.cfg')
+        if os.path.exists(setup_cfg):
+            c = configparser.ConfigParser()
+            c.read(setup_cfg)
+            package_name = c.get('metadata', 'name')
+            packages[package_name] = root
+    return packages
+
+
+def get_installed_packages(tox_python):
+    with tempfile.NamedTemporaryFile(delete=False) as tmp_requirements:
+        tmp_requirements.write(subprocess.check_output(
+            [tox_python, '-m', 'pip', 'freeze']))
+        tmp_requirements.file.flush()
+        return pip.req.req_file.parse_requirements(
+            tmp_requirements.name, session=pip.download.PipSession())
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            tox_envlist=dict(required=True, type='str'),
+            project_dir=dict(required=True, type='str'),
+            projects=dict(required=True, type='list'),
+        )
+    )
+    envlist = module.params['tox_envlist']
+    project_dir = module.params['project_dir']
+    projects = module.params['projects']
+
+    tox_python = '{project_dir}/.tox/{envlist}/bin/python'.format(
+        project_dir=project_dir, envlist=envlist)
+    # Write a log file into the .tox dir so that it'll get picked up
+    log_file = '{project_dir}/.tox/{envlist}/log/siblings.txt'.format(
+        project_dir=project_dir, envlist=envlist)
+    # Who are we?
+    package_name = subprocess.check_output(
+        ['.tox/{envlist}/bin/python'.format(envlist=envlist),
+         'setup.py', '--name'], cwd=project_dir).strip()
+
+    log = list()
+    log.append(
+        "Processing siblings for {name} from {project_dir}".format(
+            name=package_name,
+            project_dir=project_dir))
+
+    changed = False
+
+    sibling_python_packages = get_sibling_python_packages(projects)
+    for name, root in sibling_python_packages.items():
+        log.append("Sibling {name} at {root}".format(name=name, root=root))
+    for package in get_installed_packages(tox_python):
+        log.append(
+            "Found {name} python package installed".format(name=package.name))
+        if package.name == package_name:
+            # We don't need to re-process ourself. We've filtered ourselves
+            # from the source dir list, but let's be sure nothing is weird.
+            log.append(
+                "Skipping {name} because it's us".format(name=package.name))
+            continue
+        if package.name in sibling_python_packages:
+            log.append(
+                "Package {name} on system in {root}".format(
+                    name=package.name,
+                    root=sibling_python_packages[package.name]))
+            changed = True
+
+            log.append("Uninstalling {name}".format(name=package.name))
+            uninstall_output = subprocess.check_output(
+                [tox_python, '-m', 'pip', 'uninstall', '-y', package.name],
+                stderr=subprocess.STDOUT)
+            log.extend(uninstall_output.decode('utf-8').split('\n'))
+
+            # TODO(mordred) Account for upper-constraints during this install
+            log.append(
+                "Installing {name} from {root}".format(
+                    name=package.name,
+                    root=sibling_python_packages[package.name]))
+            install_output = subprocess.check_output(
+                [tox_python, '-m', 'pip', 'install',
+                 '-e', sibling_python_packages[package.name]])
+            log.extend(install_output.decode('utf-8').split('\n'))
+
+    log_text = "\n".join(log)
+    module.append_to_file(log_file, log_text)
+    module.exit_json(changed=changed, msg=log_text)
+
+
+if __name__ == '__main__':
+    main()

roles/tox-siblings/tasks/main.yaml

@@ -0,0 +1,17 @@
+- name: Require tox_envlist variable
+  fail:
+    msg: tox_envlist is required for this role
+  when: tox_envlist is not defined
+
+- name: Run tox without tests
+  command: "{{ tox_executable }} --notest -e{{ tox_envlist }}"
+  args:
+    chdir: "{{ zuul_work_dir }}"
+  when: tox_install_siblings
+
+- name: Install any sibling python packages
+  tox_install_sibling_packages:
+    tox_envlist: "{{ tox_envlist }}"
+    project_dir: "{{ zuul_work_dir }}"
+    projects: "{{ zuul.projects }}"
+  when: tox_install_siblings

zuul.yaml

@@ -39,6 +39,12 @@
 
          Path to an upper constraints file. Will be provided to tox via
          UPPER_CONSTRAINTS_FILE environment variable if it exists.
+
+      .. zuul:jobvar: tox_install_siblings
+         :default: true
+
+         Override tox requirements that have corresponding zuul git repos
+         on the node by installing the git versions into the tox virtualenv.
     run: playbooks/tox/run
     pre-run: playbooks/tox/pre
     post-run: playbooks/tox/post