Implement global constraints

In the previous repo build process, we had global constraints which
override upper constraints and anything set in the roles. This was
essential for two purposes:

1. To enable us to pin things that were not in upper constraints. eg: pip,
   setuptools, wheel
2. To enable us to pin things which were in upper constraints, but broken.
   This would usually be a temporary measure until upper constraints was
   fixed.

This patch implements a new variable 'venv_build_global_constraints' which
is a list of constraints to be applied globally for all venvs. This list
will be used to produce a file in the venv suffixed with
'-global-constraints.txt' and will be used on the pip command line when
building the wheels and when installing packages.

We also ensure that all constraints are used when both building and
installing pip, setuptools and wheel into the venv.

Change-Id: I9ae3ef19c863b9237a51d2fcd6f4ebce1a9ebad7

defaults/main.yml

@@ -66,6 +66,11 @@ venv_pip_upgrade_noconf: false
 # or installing python packages.
 venv_build_constraints: []
 
+# A list of pip constraints to be applied as global
+# constraints ahead of the list in venv_build_constraints.
+# This is useful for global pins across all venvs.
+venv_build_global_constraints: []
+
 # Arguments to pass to pip when building the wheels
 venv_pip_build_args: ""
 

tasks/python_venv_install.yml

@@ -73,6 +73,15 @@
       {% endfor %}
   register: _requirement_file
 
+- name: Build global constraints file for the venv
+  copy:
+    dest: "{{ venv_install_destination_path }}/global-constraints.txt"
+    content: |
+      {% for item in venv_build_global_constraints %}
+      {{ item }}
+      {% endfor %}
+  register: _global_constraint_file
+
 - name: Build constraints file for the venv
   copy:
     dest: "{{ venv_install_destination_path }}/constraints.txt"
@@ -97,6 +106,8 @@
     state: "{{ venv_pip_package_state }}"
     virtualenv: "{{ venv_install_destination_path }}"
     extra_args: >-
+      --constraint {{ venv_install_destination_path }}/global-constraints.txt
+      --constraint {{ venv_install_destination_path }}/constraints.txt
       --log /var/log/python_venv_build.log
       {{ venv_default_pip_install_args }}
       {{ venv_pip_install_args }}
@@ -116,12 +127,13 @@
         state: "{{ venv_pip_package_state }}"
         virtualenv: "{{ venv_install_destination_path }}"
         extra_args: >-
+          --constraint {{ venv_install_destination_path }}/global-constraints.txt
           --constraint {{ venv_install_destination_path }}/constraints.txt
           --pre
           --log /var/log/python_venv_build.log
           {{ venv_default_pip_install_args }}
           {{ venv_pip_install_args }}
-      when: (_requirement_file is changed) or (_constraint_file is changed)
+      when: (_requirement_file is changed) or (_global_constraint_file is changed) or (_constraint_file is changed)
       register: _install_venv_pip_packages
       until: _install_venv_pip_packages is success
       retries: 5
@@ -134,6 +146,7 @@
         state: absent
       with_items:
         - "{{ venv_install_destination_path }}/constraints.txt"
+        - "{{ venv_install_destination_path }}/global-constraints.txt"
         - "{{ venv_install_destination_path }}/requirements.txt"
     - fail:
         msg: >

tasks/python_venv_wheel_build.yml

@@ -47,6 +47,7 @@
       with_items:
         - "{{ venv_build_host_wheel_path }}"
         - "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt"
+        - "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-global-constraints.txt"
         - "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt"
         - "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-constraints.txt"
       when:
@@ -84,6 +85,15 @@
           {% endfor %}
       register: _requirement_file
 
+    - name: Build global constraints file for the venv
+      copy:
+        dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-global-constraints.txt"
+        content: |
+          {% for item in venv_build_global_constraints %}
+          {{ item }}
+          {% endfor %}
+      register: _global_constraint_file
+
     - name: Build constraints file for the venv
       copy:
         dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt"
@@ -102,6 +112,7 @@
         state: "{{ venv_pip_package_state }}"
         virtualenv: "{{ venv_build_host_venv_path }}"
         extra_args: >-
+          --constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-global-constraints.txt
           --constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt
           --find-links {{ venv_build_host_wheel_path }}/
           --log /var/log/python_venv_build.log
@@ -114,7 +125,7 @@
       delay: 2
 
     - name: Build wheels and constraints file
-      when: (_requirement_file is changed) or (_constraint_file is changed)
+      when: (_requirement_file is changed) or (_global_constraint_file is changed) or (_constraint_file is changed)
       block:
         - name: Clean up temporary wheel build path
           file:
@@ -125,6 +136,7 @@
           command: >-
             {{  venv_build_host_venv_path }}/bin/pip wheel
             --requirement {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt
+            --constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-global-constraints.txt
             --constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt
             --wheel-dir /tmp/{{ venv_install_destination_path | basename }}/
             --find-links {{ venv_build_host_wheel_path }}/