CI: fix kolla-ansible installation after cryptography 3.4 release

Installing kolla-ansible system-wide on CentOS 8 fails with:

    ModuleNotFoundError: No module named 'setuptools_rust'

This error appeared following the release of cryptography 3.4, which now
includes Rust code. It can be installed without Rust using a Python
wheel, but only with more recent pip than version 9.0.3 available as RPM
on CentOS 8. The cryptography bug report [1] recommends pip>=19.1.1.

This change switches to using pip --user when installing kolla-ansible.

Also fixes an issue with ansible-lint which was failing on
etc/kolla/globals.yml due to a missing space before comments.

This Victoria backport also includes
I47473de6f71c422db2238d653c2d8f379c55e79b, which fixes a similar issue
with ceph-ansible.
(cherry picked from commit 5fc7707938)

This Victoria backport also includes
I4c65a428facdf6d4ce28d97a868589aeae4c856e, which drops lower-constraints
testing.  Per our PTG resolution [2] and general OpenStack resolution
[3], lower-constraints are not worth the extra work and confusion they
introduce.  This patch drops them along with all mentions.
(cherry picked from commit 604d85b6d2)

[1] https://github.com/pyca/cryptography/issues/5753
[2] http://lists.openstack.org/pipermail/openstack-discuss/2020-October/018445.html
[3] http://lists.openstack.org/pipermail/openstack-discuss/2020-December/019521.html

Change-Id: Ifaf1948ed5d42eebaa62d7bad375bbfc12b134d5
(cherry picked from commit 3dd6834a61)
Closes-Bug: #1915141

lower-constraints.txt

@@ -1,87 +0,0 @@
-alabaster==0.7.10
-ansible==2.9.0
-appdirs==1.4.3
-asn1crypto==0.24.0
-Babel==2.5.3
-bandit==1.1.0
-bashate==0.5.1
-beautifulsoup4==4.6.0
-certifi==2018.1.18
-cffi==1.13.0
-chardet==3.0.4
-cliff==2.11.0
-cmd2==0.8.1
-coverage==4.0
-cryptography==2.1
-debtcollector==1.19.0
-decorator==4.2.1
-deprecation==2.0
-doc8==0.6.0
-docker-pycreds==0.2.2
-docker==2.4.2
-docutils==0.14
-dogpile.cache==0.6.5
-dulwich==0.19.0
-extras==1.0.0
-fixtures==3.0.0
-gitdb2==2.0.3
-GitPython==2.1.8
-idna==2.6
-imagesize==1.0.0
-iso8601==0.1.12
-Jinja2==2.10
-jmespath==0.9.3
-jsonpatch==1.21
-jsonpointer==2.0
-jsonschema==2.6.0
-keystoneauth1==3.4.0
-linecache2==1.0.0
-MarkupSafe==1.1.0
-monotonic==1.4
-msgpack==0.5.6
-munch==2.2.0
-netaddr==0.7.18
-netifaces==0.10.6
-openstacksdk==0.12.0
-os-client-config==1.29.0
-os-service-types==1.2.0
-osc-lib==1.10.0
-oslo.config==5.2.0
-oslo.context==2.20.0
-oslo.i18n==3.20.0
-oslo.log==3.36.0
-oslo.serialization==2.25.0
-oslo.utils==3.33.0
-oslotest==3.2.0
-packaging==17.1
-pbr==2.0.0
-prettytable==0.7.1
-pycparser==2.18
-Pygments==2.2.0
-pyinotify==0.9.6
-pyOpenSSL==17.5.0
-pyparsing==2.2.0
-pyperclip==1.6.0
-python-dateutil==2.7.0
-python-mimeparse==1.6.0
-python-subunit==1.2.0
-pytz==2013.6
-PyYAML==3.12
-requests==2.18.4
-requestsexceptions==1.4.0
-restructuredtext-lint==1.1.3
-rfc3986==1.2.0
-simplejson==3.13.2
-smmap2==2.0.3
-snowballstemmer==1.2.1
-sphinxcontrib-svg2pdfconverter==0.1.0
-stevedore==1.28.0
-stestr==2.0.0
-testscenarios==0.4
-testtools==2.2.0
-traceback2==1.4.0
-unittest2==1.1.0
-urllib3==1.22
-warlock==1.3.0
-websocket-client==0.47.0
-wrapt==1.10.11

tests/deploy-ceph-ansible.sh

@@ -9,6 +9,9 @@ export PYTHONUNBUFFERED=1
 function setup_ceph_ansible {
     # Prepare virtualenv for ceph-ansible deployment
     python3 -m venv --system-site-packages ~/ceph-venv
+    # NOTE(mgoddard): We need a recent pip to install the latest cryptography
+    # library. See https://github.com/pyca/cryptography/issues/5753
+    ~/ceph-venv/bin/pip install -I 'pip>=19.1.1'
     ~/ceph-venv/bin/pip install -Ir requirements.txt
     ~/ceph-venv/bin/pip install -IU selinux
 }

tests/post.yml

@@ -50,6 +50,8 @@
           - "--quiet"
 
 - hosts: primary
+  environment:
+    PATH: "{{ ansible_env.HOME + '/.local/bin:' + ansible_env.PATH }}"
   tasks:
     - name: check for existence of ara sqlite
       stat:

tests/run.yml

@@ -25,6 +25,8 @@
         dashboard_enabled: "{{ openstack_core_enabled }}"
         upper_constraints_file: "{{ ansible_env.HOME }}/src/opendev.org/openstack/requirements/upper-constraints.txt"
         docker_image_tag_suffix: "{{ '-aarch64' if ansible_architecture == 'aarch64' else '' }}"
+        pip_user_path_env:
+          PATH: "{{ ansible_env.HOME + '/.local/bin:' + ansible_env.PATH }}"
 
     - name: Install dig for Designate testing
       become: true
@@ -65,6 +67,7 @@
         # NOTE(yoctozepto): to avoid issues with IPv6 not enabled in the docker daemon
         # and since we don't need isolated networks here, use host networking
         network_mode: host
+  environment: "{{ pip_user_path_env }}"
   tasks:
     - name: detect whether need build images
       set_fact:
@@ -200,6 +203,14 @@
         state: directory
       become: true
 
+    # NOTE(mgoddard): We need a recent pip to install the latest cryptography
+    # library. See https://github.com/pyca/cryptography/issues/5753
+    - name: install pip 19.1.1+
+      pip:
+        name: "pip>=19.1.1"
+        executable: "pip3"
+        extra_args: "--user"
+
     - name: install kolla-ansible and dependencies
       vars:
         # Test latest ansible version on Ubuntu, minimum supported on others.
@@ -210,8 +221,7 @@
           - "ansible{{ ansible_version_constraint }}"
           - "ara<1.0.0"
         executable: "pip3"
-        extra_args: "-c {{ upper_constraints_file }}"
-      become: true
+        extra_args: "-c {{ upper_constraints_file }} --user"
 
     - name: get ARA callback plugin path
       command: "python3 -m ara.setup.callback_plugins"
@@ -486,6 +496,7 @@
 
 - hosts: primary
   any_errors_fatal: true
+  environment: "{{ pip_user_path_env }}"
   tasks:
     # Upgrade: update config.
     - block:
@@ -529,8 +540,7 @@
           pip:
             name: "{{ kolla_ansible_src_dir }}"
             executable: pip3
-            extra_args: "-c {{ upper_constraints_file }}"
-          become: true
+            extra_args: "-c {{ upper_constraints_file }} --user"
 
         # Update passwords.yml to include any new passwords added in this
         # release.
@@ -627,6 +637,7 @@
 
 - hosts: primary
   any_errors_fatal: true
+  environment: "{{ pip_user_path_env }}"
   tasks:
     - name: Run reconfigure.sh script
       script:

tox.ini

@@ -70,12 +70,6 @@ commands =
   rm -rf releasenotes/build
   sphinx-build -a -E -W -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html
 
-[testenv:lower-constraints]
-deps =
-  -c{toxinidir}/lower-constraints.txt
-  -r{toxinidir}/test-requirements.txt
-  -r{toxinidir}/requirements.txt
-
 [testenv:linters]
 # Env vars and deps need to be defined in top level tox env
 setenv =
@@ -131,7 +125,7 @@ setenv = {[testenv:linters]setenv}
 deps = {[testenv:linters]deps}
 commands =
   python {toxinidir}/tools/validate-all-file.py
-  ansible-lint -p --exclude {toxinidir}/tests --exclude {toxinidir}/roles
+  ansible-lint -p --exclude {toxinidir}/tests --exclude {toxinidir}/roles --exclude {toxinidir}/etc
 
 [testenv:yamllint]
 deps = {[testenv:linters]deps}

zuul.d/base.yaml

@@ -14,7 +14,6 @@
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*
-      - ^lower-constraints.txt$
       - ^releasenotes/.*$
       - ^deploy-guide/.*$
       - ^test-requirements.txt$

zuul.d/project.yaml

@@ -3,7 +3,6 @@
     templates:
       - check-requirements
       - openstack-cover-jobs
-      - openstack-lower-constraints-jobs
       - openstack-python3-victoria-jobs
       - ansible-role-jobs
       - publish-openstack-docs-pti