Fix multiple CI failures

1. Use stable/queens as version of kolla-ansible in ansible tests

Instead of always checking out the master branch, use the kolla-ansible
branch with which this Kayobe version is meant to be used.

This is based on change I074d4b9d444649ecf956d3cd92748862e8c89a5c,
though we cannot use {{ openstack_branch }} which was introduced with
Stein. We hardcode stable/queens instead, as we did in stable/rocky with
change I344b1e90d34e0c8e2fb30331038b296ac89449b1.

2. Install python-openstackclient using upper constraints

Detect current branch from .gitreview and use upper constraints to
install python-openstackclient, to guarantee compatibility with the
Python version in use. This is cherry picked from commit
ef33e6ecb7.

3. Update packages in virtualenvs

Kayobe uses a number of virtual environments on the remote hosts for
python dependencies such as shade, python-openstackclient, docker, etc.
By default these are stored in /opt/kayobe/venvs/. Typically we do not
provide version restrictions when installing these packages, so over the
course of time they may become stale and incompatible.

This change installs the latest version of packages allowed by OpenStack
upper constraints.

It also adds a new variable, 'pip_upper_constraints_file', to set the
upper constraints file. The existing variable
'kolla_upper_constraints_file' now defaults to the value of
'pip_upper_constraints_file'.

This is cherry picked from commit
ddfd6b6202.

4. Remove upgrade jobs from Zuul configuration

Upgrade jobs are failing because stable/pike has the same issues fixed
in kolla-ansible for stable/queens:

- https://review.opendev.org/#/c/717276/
- https://review.opendev.org/#/c/717302/

Fixing kayobe for stable/pike would involve fixing kolla-ansible for
stable/pike, which is not maintained anymore. Drop upgrade jobs instead.

Change-Id: Ie44508fe3d3b08190afa5a43748e43548a63ff82

ansible/baremetal-compute-inspect.yml

@@ -15,6 +15,8 @@
     - role: stackhpc.os-shade
       os_shade_venv: "{{ venv }}"
       os_shade_install_epel: "{{ yum_install_epel }}"
+      os_shade_state: latest
+      os_shade_upper_constraints_file: "{{ pip_upper_constraints_file }}"
 
     - role: activate-virtualenv
       activate_virtualenv_path: "{{ venv }}"

ansible/baremetal-compute-manage.yml

@@ -15,6 +15,8 @@
     - role: stackhpc.os-shade
       os_shade_venv: "{{ venv }}"
       os_shade_install_epel: "{{ yum_install_epel }}"
+      os_shade_state: latest
+      os_shade_upper_constraints_file: "{{ pip_upper_constraints_file }}"
 
     - role: activate-virtualenv
       activate_virtualenv_path: "{{ venv }}"

ansible/baremetal-compute-provide.yml

@@ -15,6 +15,8 @@
     - role: stackhpc.os-shade
       os_shade_venv: "{{ venv }}"
       os_shade_install_epel: "{{ yum_install_epel }}"
+      os_shade_state: latest
+      os_shade_upper_constraints_file: "{{ pip_upper_constraints_file }}"
 
     - role: activate-virtualenv
       activate_virtualenv_path: "{{ venv }}"

ansible/baremetal-compute-rename.yml

@@ -15,6 +15,8 @@
         name:
           - python-openstackclient
           - python-ironicclient
+        state: latest
+        extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}"
 
 - name: Rename baremetal compute nodes
   hosts: baremetal-compute

ansible/docker.yml

@@ -4,7 +4,7 @@
   tags:
     - docker
   vars:
-    - docker_upper_constraints_file: "{{ kolla_upper_constraints_file }}"
+    - docker_upper_constraints_file: "{{ pip_upper_constraints_file }}"
   roles:
     - role: docker
       docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}"

ansible/external-net.yml

@@ -5,6 +5,8 @@
   roles:
     - role: stackhpc.os-networks
       os_shade_install_epel: "{{ yum_install_epel }}"
+      os_shade_state: latest
+      os_shade_upper_constraints_file: "{{ pip_upper_constraints_file }}"
       os_networks_venv: "{{ virtualenv_path }}/shade"
       os_networks_openstack_auth_type: "{{ openstack_auth_type }}"
       os_networks_openstack_auth: "{{ openstack_auth }}"

ansible/group_vars/all/kolla

@@ -313,7 +313,7 @@ kolla_enable_tls_external: "no"
 kolla_openstack_logging_debug: "False"
 
 # Upper constraints file for installation of Kolla.
-kolla_upper_constraints_file: "https://opendev.org/openstack/requirements/raw/branch/stable/queens/upper-constraints.txt"
+kolla_upper_constraints_file: "{{ pip_upper_constraints_file }}"
 
 # User account to use for Kolla SSH access.
 kolla_ansible_user: kolla

ansible/group_vars/all/pip

@@ -0,0 +1,3 @@
+---
+# Upper constraints file for installation of python packages.
+pip_upper_constraints_file: "https://releases.openstack.org/constraints/upper/queens"

ansible/overcloud-docker-sdk-upgrade.yml

@@ -26,6 +26,6 @@
       pip:
         name: docker
         state: latest
-        extra_args: "{% if kolla_upper_constraints_file %}-c {{ kolla_upper_constraints_file }}{% endif %}"
+        extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}"
         virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}"
       become: "{{ virtualenv is not defined }}"

ansible/overcloud-grafana-configure.yml

@@ -19,6 +19,8 @@
     - role: stackhpc.os-openstackclient
       os_openstackclient_venv: "{{ venv }}"
       os_openstackclient_install_epel: "{{ yum_install_epel }}"
+      os_openstackclient_state: latest
+      os_openstackclient_upper_constraints_file: "{{ pip_upper_constraints_file }}"
       when: kolla_enable_monasca | bool
 
   tasks:

ansible/overcloud-introspection-rules-dell-lldp-workaround.yml

@@ -117,6 +117,8 @@
   roles:
     - role: ironic-inspector-rules
       os_shade_install_epel: "{{ yum_install_epel }}"
+      os_shade_state: latest
       ironic_inspector_venv: "{{ virtualenv_path }}/shade"
+      ironic_inspector_upper_constraints_file: "{{ pip_upper_constraints_file }}"
       ironic_inspector_auth_type: "{{ openstack_auth_type }}"
       ironic_inspector_auth: "{{ openstack_auth }}"

ansible/overcloud-introspection-rules.yml

@@ -36,6 +36,8 @@
       vars:
         os_openstackclient_venv: "{{ venv }}"
         os_openstackclient_install_epel: "{{ yum_install_epel }}"
+        os_openstackclient_state: latest
+        os_openstackclient_upper_constraints_file: "{{ pip_upper_constraints_file }}"
 
     - name: Retrieve the IPA kernel Glance image UUID
       shell: >
@@ -56,7 +58,9 @@
   roles:
     - role: ironic-inspector-rules
       os_shade_install_epel: "{{ yum_install_epel }}"
+      os_shade_state: latest
       ironic_inspector_venv: "{{ venv }}"
+      ironic_inspector_upper_constraints_file: "{{ pip_upper_constraints_file }}"
       ironic_inspector_auth_type: "{{ openstack_auth_type }}"
       ironic_inspector_auth: "{{ openstack_auth }}"
       ironic_inspector_rules: "{{ inspector_rules }}"

ansible/overcloud-ipa-build.yml

@@ -28,6 +28,8 @@
             name: stackhpc.os-images
           vars:
             os_images_venv: "{{ virtualenv_path }}/ipa-build-dib"
+            os_images_package_state: latest
+            os_images_upper_constraints_file: "{{ pip_upper_constraints_file }}"
             os_images_cache: "{{ image_cache_path }}"
             os_images_common: ""
             os_images_list:

ansible/overcloud-ipa-images.yml

@@ -100,7 +100,9 @@
   roles:
     - role: ipa-images
       os_shade_install_epel: "{{ yum_install_epel }}"
+      os_shade_state: latest
       ipa_images_venv: "{{ virtualenv_path }}/shade"
+      ipa_images_upper_constraints_file: "{{ pip_upper_constraints_file }}"
       ipa_images_openstack_auth_type: "{{ openstack_auth_type }}"
       ipa_images_openstack_auth: "{{ openstack_auth }}"
       ipa_images_openstack_auth_env: "{{ openstack_auth_env }}"

ansible/provision-net.yml

@@ -59,6 +59,9 @@
 
   roles:
     - role: stackhpc.os-networks
+      os_shade_install_epel: "{{ yum_install_epel }}"
+      os_shade_state: latest
+      os_shade_upper_constraints_file: "{{ pip_upper_constraints_file }}"
       os_networks_venv: "{{ virtualenv_path }}/shade"
       os_networks_auth_type: "{{ openstack_auth_type }}"
       os_networks_auth: "{{ openstack_auth }}"

ansible/roles/ipa-images/defaults/main.yml

@@ -2,6 +2,9 @@
 # Path to virtualenv in which to install shade and its dependencies.
 ipa_images_venv:
 
+# Upper constraints file for installation of python-ironicclient.
+ipa_images_upper_constraints_file:
+
 # Authentication type compatible with the 'os_image' Ansible module's
 # auth_type argument.
 ipa_images_openstack_auth_type:

ansible/roles/ipa-images/meta/main.yml

@@ -2,5 +2,7 @@
 dependencies:
   - role: stackhpc.os-shade
     os_shade_venv: "{{ ipa_images_venv }}"
+    os_shade_upper_constraints_file: "{{ ipa_images_upper_constraints_file }}"
   - role: stackhpc.os-openstackclient
     os_openstackclient_venv: "{{ ipa_images_venv }}"
+    os_openstackclient_upper_constraints_file: "{{ ipa_images_upper_constraints_file }}"

ansible/roles/ipa-images/tasks/main.yml

@@ -59,7 +59,9 @@
 - name: Ensure we have python-ironicclient installed
   pip:
     name: python-ironicclient
+    state: latest
     virtualenv: "{{ ipa_images_venv }}"
+    extra_args: "{% if ipa_images_upper_constraints_file %}-c {{ ipa_images_upper_constraints_file }}{% endif %}"
 
 # To support updating the IPA image, we check the MD5 sum of the cached image
 # files, and compare with the images in Glance (if there are any).

ansible/roles/ironic-inspector-rules/defaults/main.yml

@@ -2,6 +2,9 @@
 # Path to a directory in which to create a virtualenv.
 ironic_inspector_venv:
 
+# Upper constraints file for installation of python-ironic-inspector-client.
+ironic_inspector_upper_constraints_file:
+
 # Authentication type.
 ironic_inspector_auth_type:
 

ansible/roles/ironic-inspector-rules/meta/main.yml

@@ -2,3 +2,4 @@
 dependencies:
   - role: stackhpc.os-shade
     os_shade_venv: "{{ ironic_inspector_venv }}"
+    os_shade_upper_constraints_file: "{{ ironic_inspector_upper_constraints_file }}"

ansible/roles/ironic-inspector-rules/tasks/main.yml

@@ -3,8 +3,9 @@
   pip:
     name: "{{ item.name }}"
     version: "{{ item.version | default(omit) }}"
-    state: present
+    state: latest
     virtualenv: "{{ ironic_inspector_venv }}"
+    extra_args: "{% if ironic_inspector_upper_constraints_file %}-c {{ ironic_inspector_upper_constraints_file }}{% endif %}"
   with_items:
     - name: python-ironic-inspector-client
 

ansible/roles/kolla-ansible/tests/test-defaults.yml

@@ -15,7 +15,7 @@
             kolla_ansible_source_path: "{{ temp_path }}/src"
             kolla_ansible_ctl_install_type: "source"
             kolla_ansible_source_url: "http://github.com/openstack/kolla-ansible"
-            kolla_ansible_source_version: "master"
+            kolla_ansible_source_version: "stable/queens"
             kolla_ansible_venv: "{{ temp_path }}/venv"
             kolla_config_path: "{{ temp_path }}/etc/kolla"
             kolla_node_custom_config_path: "{{ temp_path }}/etc/kolla/config"

ansible/roles/kolla-ansible/tests/test-extras.yml

@@ -53,7 +53,7 @@
             kolla_ansible_source_path: "{{ temp_path }}/src"
             kolla_ansible_ctl_install_type: "source"
             kolla_ansible_source_url: "http://github.com/openstack/kolla-ansible"
-            kolla_ansible_source_version: "master"
+            kolla_ansible_source_version: "stable/queens"
             kolla_ansible_venv: "{{ temp_path }}/venv"
             kolla_ansible_vault_password: "fake-password"
             kolla_config_path: "{{ temp_path }}/etc/kolla"

ansible/roles/kolla-ansible/tests/test-requirements.yml

@@ -15,7 +15,7 @@
             kolla_ansible_source_path: "{{ temp_path }}/src"
             kolla_ansible_ctl_install_type: "source"
             kolla_ansible_source_url: "http://github.com/openstack/kolla-ansible"
-            kolla_ansible_source_version: "master"
+            kolla_ansible_source_version: "stable/queens"
             kolla_ansible_venv: "{{ temp_path }}/venv"
             kolla_ansible_venv_extra_requirements:
               - "hvac"

ansible/seed-introspection-rules.yml

@@ -6,7 +6,9 @@
   roles:
     - role: ironic-inspector-rules
       os_shade_install_epel: "{{ yum_install_epel }}"
+      os_shade_state: latest
       ironic_inspector_venv: "{{ virtualenv_path }}/shade"
+      ironic_inspector_upper_constraints_file: "{{ pip_upper_constraints_file }}"
       # No auth required for Bifrost.
       ironic_inspector_auth_type: None
       ironic_inspector_auth: {}

ansible/seed-ipa-build.yml

@@ -19,6 +19,8 @@
             name: stackhpc.os-images
           vars:
             os_images_venv: "{{ virtualenv_path }}/ipa-build-dib"
+            os_images_package_state: latest
+            os_images_upper_constraints_file: "{{ pip_upper_constraints_file }}"
             os_images_cache: "{{ image_cache_path }}"
             os_images_common: ""
             os_images_list:

dev/config.sh

@@ -23,3 +23,6 @@
 
 # Additional arguments to pass to kayobe commands.
 #export KAYOBE_EXTRA_ARGS=
+
+# Upper constraints to use when installing Python packages.
+#export UPPER_CONSTRAINTS_FILE=

dev/functions

@@ -40,6 +40,21 @@ function config_defaults {
 
     # Additional arguments to pass to kayobe commands.
     export KAYOBE_EXTRA_ARGS=${KAYOBE_EXTRA_ARGS:-}
+
+    # Use .gitreview as the key to determine the appropriate branch to clone
+    # for tests. Inspired by OSA code.
+    PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    if [ -f "${PARENT}/../.gitreview" ]; then
+      BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' "${PARENT}/../.gitreview")
+      if [[ "${BRANCH}" == "" ]]; then
+        SERIES="master"
+      else
+        SERIES="$(echo ${BRANCH} | sed 's|stable/||')"
+      fi
+
+      # Upper constraints to use when installing Python packages.
+      export UPPER_CONSTRAINTS_FILE="${UPPER_CONSTRAINTS_FILE:-https://releases.openstack.org/constraints/upper/${SERIES}}"
+    fi
 }
 
 function config_set {
@@ -339,7 +354,11 @@ function overcloud_test {
 
     environment_setup
 
-    pip install python-openstackclient
+    if [[ ! -z "$UPPER_CONSTRAINTS_FILE" ]]; then
+        pip install python-openstackclient -c "$UPPER_CONSTRAINTS_FILE"
+    else
+        pip install python-openstackclient
+    fi
 
     source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh"
     if ! openstack image show cirros >/dev/null 2>&1; then

etc/kayobe/kolla.yml

@@ -145,6 +145,7 @@
 #kolla_openstack_logging_debug:
 
 # Upper constraints file for installation of Kolla.
+# Default value is {{ pip_upper_constraints_file }}.
 #kolla_upper_constraints_file:
 
 # User account to use for Kolla SSH access.

etc/kayobe/pip.yml

@@ -1,5 +1,10 @@
 ---
 
+# Upper constraints file for installation of python packages.
+# Default value is
+# "https://releases.openstack.org/constraints/upper/queens"
+#pip_upper_constraints_file:
+
 # Use a local PyPi mirror for installing Pip packages
 #pip_local_mirror: false
 

playbooks/kayobe-overcloud-base/overrides.yml.j2

@@ -7,7 +7,7 @@ kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/opensta
 kolla_source_version: "{{ zuul.projects['opendev.org/openstack/kolla'].checkout }}"
 kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla-ansible'].src_dir }}"
 kolla_ansible_source_version: "{{ zuul.projects['opendev.org/openstack/kolla-ansible'].checkout }}"
-kolla_upper_constraints_file: "/tmp/upper-constraints.txt"
+pip_upper_constraints_file: "/tmp/upper-constraints.txt"
 
 # Use the CI infra's PyPI mirror.
 pip_local_mirror: true

playbooks/kayobe-overcloud-upgrade-base/globals.yml.j2

@@ -1,10 +0,0 @@
----
-# The Queens kayobe configuration uses the fernet token provider. To avoid
-# issues during upgrade, set the provider to fernet in the Pike deployment
-# also.
-keystone_token_provider: 'fernet'
-
-# Most development environments will use nested virtualisation, and we can't
-# guarantee that nested KVM support is available. Use QEMU as a lowest common
-# denominator.
-nova_compute_virt_type: qemu

playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2

@@ -1,16 +0,0 @@
----
-# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
-# Ansible is run directly on the controller.
-disable_selinux_do_reboot: false
-
-{% if not is_previous_release | default(false) %}
-kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}"
-kolla_source_version: "{{ zuul.projects['opendev.org/openstack/kolla'].checkout }}"
-kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla-ansible'].src_dir }}"
-kolla_ansible_source_version: "{{ zuul.projects['opendev.org/openstack/kolla-ansible'].checkout }}"
-kolla_upper_constraints_file: "/tmp/upper-constraints.txt"
-{% endif %}
-
-# Use the CI infra's PyPI mirror.
-pip_local_mirror: true
-pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"

playbooks/kayobe-overcloud-upgrade-base/post.yml

@@ -1,8 +0,0 @@
----
-- hosts: all
-  roles:
-    - role: kayobe-diagnostics
-      kayobe_diagnostics_phase: "post"
-      kayobe_diagnostics_log_dir: "/tmp/logs"
-      kayobe_diagnostics_config_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe-config-dev'].src_dir }}"
-      kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"

playbooks/kayobe-overcloud-upgrade-base/pre.yml

@@ -1,65 +0,0 @@
----
-- hosts: primary
-  vars:
-    logs_dir: "/tmp/logs"
-    kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe'].src_dir }}"
-    kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe-config-dev'].src_dir }}"
-    previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
-    previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
-  roles:
-    - role: kayobe-diagnostics
-      kayobe_diagnostics_phase: "pre"
-      kayobe_diagnostics_log_dir: "{{ logs_dir }}"
-
-    - role: kayobe-ci-prep
-  tasks:
-    # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
-    # the zuul and stack users.
-    - name: Ensure upper-contraints.txt exists
-      copy:
-        src: "{{ zuul.projects['opendev.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}"
-        dest: "/tmp"
-        mode: 0644
-        remote_src: true
-
-    - name: Ensure previous kayobe directory exists
-      file:
-        path: "{{ previous_kayobe_src_dir }}"
-        state: directory
-
-    - name: Ensure previous kayobe repository is cloned
-      command: >-
-        git clone {{ kayobe_src_dir }} {{ previous_kayobe_src_dir }} -b stable/{{ previous_release | lower }}
-
-    - name: Ensure previous kayobe-config directory exists
-      file:
-        path: "{{ previous_kayobe_config_src_dir }}"
-        state: directory
-
-    - name: Ensure kayobe-config repository is cloned
-      command: >-
-        git clone {{ kayobe_config_src_dir }} {{ previous_kayobe_config_src_dir }} -b stable/{{ previous_release | lower }}
-
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
-    - name: Ensure kayobe-config override config file exists
-      template:
-        src: overrides.yml.j2
-        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
-      vars:
-        is_previous_release: true
-
-    - name: Ensure kayobe-config globals.yml config file exists
-      template:
-        src: globals.yml.j2
-        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml"
-
-    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
-    # breth1 - to exist with an IP address of 192.168.33.3.
-    - name: Ensure all-in-one network bridge interface exists
-      command: "{{ item }}"
-      become: true
-      with_items:
-        - "ip l add breth1 type bridge"
-        - "ip l set breth1 up"
-        - "ip a add 192.168.33.3/24 dev breth1"

playbooks/kayobe-overcloud-upgrade-base/run.yml

@@ -1,83 +0,0 @@
----
-- hosts: primary
-  vars:
-    kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe'].src_dir }}"
-    kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe-config-dev'].src_dir }}"
-    previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
-    previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
-    logs_dir: "/tmp/logs"
-  tasks:
-
-    # Install the previous release of Kayobe, and use it to deploy a control
-    # plane.
-
-    - block:
-        - name: Ensure kayobe is installed
-          shell:
-            cmd: dev/install.sh > {{ logs_dir }}/ansible/install-pre-upgrade
-            chdir: "{{ previous_kayobe_src_dir }}"
-
-        - name: Ensure overcloud is deployed
-          shell:
-            cmd: dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy-pre-upgrade
-            chdir: "{{ previous_kayobe_src_dir }}"
-      environment:
-        KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}"
-
-    # Update the Kayobe configuration to the current release.
-
-    - name: Ensure kolla config directory exists
-      file:
-        path: "{{ kayobe_config_src_dir }}/etc/kolla"
-        state: directory
-
-    - name: Copy across relevant kayobe-config files
-      copy:
-        src: "{{ previous_kayobe_config_src_dir }}/{{ item }}"
-        dest: "{{ kayobe_config_src_dir }}/{{ item }}"
-        remote_src: true
-      with_items:
-        - etc/kayobe/kolla/passwords.yml
-        - etc/kolla/admin-openrc.sh
-        - etc/kolla/public-openrc.sh
-
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
-    - name: Ensure kayobe-config override config file exists
-      template:
-        src: overrides.yml.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
-
-    # Perform a smoke test against the previous release.
-
-    - block:
-        - name: Perform testing of the overcloud prior to upgrade
-          shell:
-            cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-pre-upgrade
-            # NOTE(mgoddard): Currently need to use the new kayobe repo for
-            # testing, since the overcloud-test.sh script is not available in Pike.
-            chdir: "{{ kayobe_src_dir }}"
-
-        # Upgrade Kayobe, and use it to perform an upgrade of the control plane.
-
-        - name: Ensure overcloud is upgraded
-          shell:
-            cmd: dev/overcloud-upgrade.sh > {{ logs_dir }}/ansible/overcloud-upgrade
-            chdir: "{{ kayobe_src_dir }}"
-
-        # FIXME(mgoddard): The nova-compute service does not seem to be correctly
-        # handling the SIGHUP after being upgraded, leading to "In shutdown, no new
-        # events can be scheduled" errors when booting an instance.
-        - name: Workaround for SIGHUP issue - restart nova-compute service
-          shell:
-            cmd: docker restart nova_compute
-          become: true
-
-        # Perform a smoke test against the upgraded current release.
-
-        - name: Perform testing of the upgraded overcloud
-          shell:
-            cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-post-upgrade
-            chdir: "{{ kayobe_src_dir }}"
-      environment:
-        KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"

playbooks/kayobe-seed-base/overrides.yml.j2

@@ -7,7 +7,7 @@ kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/opensta
 kolla_source_version: "{{ zuul.projects['opendev.org/openstack/kolla'].checkout }}"
 kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla-ansible'].src_dir }}"
 kolla_ansible_source_version: "{{ zuul.projects['opendev.org/openstack/kolla-ansible'].checkout }}"
-kolla_upper_constraints_file: "/tmp/upper-constraints.txt"
+pip_upper_constraints_file: "/tmp/upper-constraints.txt"
 
 # NOTE(mgoddard): We're using a cirros image, which doesn't require the
 # resolv.conf work around used for CentOS.

playbooks/kayobe-seed-upgrade-base/bifrost-overrides.yml.j2

@@ -1,8 +0,0 @@
----
-# Don't build an IPA deployment image, instead download upstream images.
-create_ipa_image: false
-download_ipa: true
-
-# Don't build a disk image. It takes time and can be unreliable.
-use_cirros: true
-cirros_deploy_image_upstream_url: https://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img

playbooks/kayobe-seed-upgrade-base/overrides.yml.j2

@@ -1,33 +0,0 @@
----
-# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
-# Ansible is run directly on the controller.
-disable_selinux_do_reboot: false
-
-{% if not is_previous_release | default(false) %}
-kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}"
-kolla_source_version: "{{ zuul.projects['opendev.org/openstack/kolla'].checkout }}"
-kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla-ansible'].src_dir }}"
-kolla_ansible_source_version: "{{ zuul.projects['opendev.org/openstack/kolla-ansible'].checkout }}"
-kolla_upper_constraints_file: "/tmp/upper-constraints.txt"
-{% endif %}
-
-# NOTE(mgoddard): We're using a cirros image, which doesn't require the
-# resolv.conf work around used for CentOS.
-overcloud_host_image_workaround_resolv_enabled: false
-
-# Use the CI infra's PyPI mirror.
-pip_local_mirror: true
-pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"
-
-{% if previous_release == "pike" %}
-# kayobe-config-dev in queens changed to use overlay by default. Specify
-# devicemapper explicitly to avoid changing.
-docker_storage_driver: devicemapper
-docker_storage_volume_thinpool_size: 45%VG
-
-# NOTE(mgoddard): Use a loopback-mounted LVM volume for docker storage since
-# the overlay driver doesn't work with the ansible template module until
-# ansible 2.4.0, and this is required by bifrost.
-seed_lvm_group_data_disks:
-  - /dev/loop0
-{% endif %}

playbooks/kayobe-seed-upgrade-base/post.yml

@@ -1,9 +0,0 @@
----
-- hosts: all
-  roles:
-    - role: kayobe-diagnostics
-      kayobe_diagnostics_phase: "post"
-      kayobe_diagnostics_log_dir: "/tmp/logs"
-      kayobe_diagnostics_config_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe-config-dev'].src_dir }}"
-      kayobe_diagnostics_previous_config_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
-      kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"

playbooks/kayobe-seed-upgrade-base/pre.yml

@@ -1,83 +0,0 @@
----
-- hosts: primary
-  vars:
-    logs_dir: "/tmp/logs"
-    kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe'].src_dir }}"
-    kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe-config-dev'].src_dir }}"
-    previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
-    previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
-  roles:
-    - role: kayobe-diagnostics
-      kayobe_diagnostics_phase: "pre"
-      kayobe_diagnostics_log_dir: "{{ logs_dir }}"
-
-    - role: kayobe-ci-prep
-  tasks:
-    # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
-    # the zuul and stack users.
-    - name: Ensure upper-contraints.txt exists
-      copy:
-        src: "{{ zuul.projects['opendev.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}"
-        dest: "/tmp"
-        mode: 0644
-        remote_src: true
-
-    - name: Ensure previous kayobe directory exists
-      file:
-        path: "{{ previous_kayobe_src_dir }}"
-        state: directory
-
-    - name: Ensure previous kayobe repository is cloned
-      command: >-
-        git clone {{ kayobe_src_dir }} {{ previous_kayobe_src_dir }} -b stable/{{ previous_release | lower }}
-
-    - name: Ensure previous kayobe-config directory exists
-      file:
-        path: "{{ previous_kayobe_config_src_dir }}"
-        state: directory
-
-    - name: Ensure kayobe-config repository is cloned
-      command: >-
-        git clone {{ kayobe_config_src_dir }} {{ previous_kayobe_config_src_dir }} -b stable/{{ previous_release | lower }}
-
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
-    - name: Ensure kayobe-config override config file exists
-      template:
-        src: overrides.yml.j2
-        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
-      vars:
-        is_previous_release: true
-
-    - name: Ensure bifrost overrides directory exists
-      file:
-        path: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost"
-        state: "directory"
-
-    - name: Ensure bifrost overrides file exists
-      template:
-        src: bifrost-overrides.yml.j2
-        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/bifrost.yml"
-
-    - block:
-        # NOTE(mgoddard): Create a loopback device backed by a file for docker
-        # storage.  We do this since the overlay driver doesn't work with the
-        # ansible template module until ansible 2.4.0, and this is required by
-        # bifrost.
-        - name: Ensure a docker storage backing file exists
-          command: truncate -s 20G /tmp/docker-storage
-
-        - name: Ensure the docker storage loopback device is created
-          command: losetup /dev/loop0 /tmp/docker-storage
-          become: true
-      when: previous_release == "pike"
-
-    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
-    # breth1 - to exist on the seed with an IP address of 192.168.33.5.
-    - name: Ensure all-in-one network bridge interface exists
-      command: "{{ item }}"
-      become: true
-      with_items:
-        - "ip l add breth1 type bridge"
-        - "ip l set breth1 up"
-        - "ip a add 192.168.33.5/24 dev breth1"

playbooks/kayobe-seed-upgrade-base/run.yml

@@ -1,64 +0,0 @@
----
-- hosts: primary
-  vars:
-    kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe'].src_dir }}"
-    kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kayobe-config-dev'].src_dir }}"
-    previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
-    previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
-    logs_dir: "/tmp/logs"
-  tasks:
-
-    # Install the previous release of Kayobe, and use it to deploy a control
-    # plane.
-
-    - block:
-        - name: Ensure kayobe is installed
-          shell:
-            cmd: dev/install.sh > {{ logs_dir }}/ansible/install-pre-upgrade
-            chdir: "{{ previous_kayobe_src_dir }}"
-
-        - name: Ensure seed is deployed
-          shell:
-            cmd: dev/seed-deploy.sh > {{ logs_dir }}/ansible/seed-deploy-pre-upgrade
-            chdir: "{{ previous_kayobe_src_dir }}"
-      environment:
-        KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}"
-        # Don't provision a seed VM - use the Zuul VM as the seed host.
-        KAYOBE_SEED_VM_PROVISION: 0
-
-    # Update the Kayobe configuration to the current release.
-
-    - name: Ensure bifrost overrides directory exists
-      file:
-        path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost"
-        state: "directory"
-
-    - name: Copy across relevant kayobe-config files
-      copy:
-        src: "{{ previous_kayobe_config_src_dir }}/{{ item }}"
-        dest: "{{ kayobe_config_src_dir }}/{{ item }}"
-        remote_src: true
-      with_items:
-        - etc/kayobe/kolla/passwords.yml
-        - etc/kayobe/kolla/config/bifrost/bifrost.yml
-
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
-    - name: Ensure kayobe-config override config file exists
-      template:
-        src: overrides.yml.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
-
-    # TODO(mgoddard): Perform a smoke test against the previous release.
-
-    - block:
-        # Upgrade Kayobe, and use it to perform an upgrade of the seed.
-
-        - name: Ensure seed is upgraded
-          shell:
-            cmd: dev/seed-upgrade.sh > {{ logs_dir }}/ansible/seed-upgrade
-            chdir: "{{ kayobe_src_dir }}"
-
-        # TODO(mgoddard): Perform a smoke test against the upgraded current release.
-      environment:
-        KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"

releasenotes/notes/stale-virtualenvs-125c513af3de9396.yaml

@@ -0,0 +1,18 @@
+---
+features:
+  - |
+    Adds a new configuration variable, ``pip_upper_constraints_file``, which is
+    used to configure the file or URL containing the python upper version
+    contraints. Its default value is
+    ``https://releases.openstack.org/constraints/upper/queens``.
+fixes:
+  - |
+    Fixes an issue with virtual environments on remote hosts, which may over
+    the course of time become stale and incompatible with Kayobe or other
+    software. This was fixed by installing the latest version of packages
+    allowed by OpenStack upper constraints. See `story 2005923
+    <https://storyboard.openstack.org/#!/story/2005923>`__ for details.
+upgrade:
+  - |
+    The default value of ``kolla_upper_constraints_file`` has been changed to
+    ``{{ pip_upper_constraints_file }}``.

requirements.yml

@@ -27,15 +27,15 @@
 - src: stackhpc.mellanox-switch
   version: v1.0.0
 - src: stackhpc.os-images
-  version: v1.0.0
+  version: v1.4.0
 - src: stackhpc.os-ironic-state
   version: v1.0.0
 - src: stackhpc.os-networks
   version: v1.0.0
 - src: stackhpc.os-openstackclient
-  version: v1.2.0
+  version: v1.3.0
 - src: stackhpc.os-shade
-  version: v1.2.0
+  version: v1.3.0
 # We are currently using a fork of this role since the upstream yatesr.timezone
 # role includes symlinks which do not play nicely with PBR. Once that issue has
 # been resolved in PBR, we can revert to using the upstream yatesr.timezone

zuul.d/jobs.yaml

@@ -58,44 +58,6 @@
     parent: kayobe-overcloud-base
     nodeset: kayobe-centos
 
-- job:
-    name: kayobe-overcloud-upgrade-base
-    description: |
-      Base job for testing overcloud upgrades.
-
-      Configures the primary VM as an overcloud controller using the previous
-      OpenStack release, and upgrades it to the current release.
-    pre-run: playbooks/kayobe-overcloud-upgrade-base/pre.yml
-    run: playbooks/kayobe-overcloud-upgrade-base/run.yml
-    post-run: playbooks/kayobe-overcloud-upgrade-base/post.yml
-    attempts: 1
-    timeout: 7200
-    required-projects:
-      # Include kayobe to ensure other projects can use this job.
-      - name: openstack/kayobe
-      - name: openstack/kayobe-config-dev
-      - name: openstack/kolla
-        override-checkout: stable/queens
-      - name: openstack/kolla-ansible
-        override-checkout: stable/queens
-      - name: openstack/requirements
-        override-checkout: stable/queens
-    vars:
-      # Name of the release to upgrade from.
-      previous_release: pike
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^doc/.*
-      - ^releasenotes/.*
-      - ^setup.cfg$
-      - ^tools/.*$
-      - ^tox.ini$
-
-- job:
-    name: kayobe-overcloud-upgrade-centos
-    parent: kayobe-overcloud-upgrade-base
-    nodeset: kayobe-centos
-
 - job:
     name: kayobe-seed-base
     description: |
@@ -125,41 +87,3 @@
     name: kayobe-seed-centos
     parent: kayobe-seed-base
     nodeset: kayobe-centos
-
-- job:
-    name: kayobe-seed-upgrade-base
-    description: |
-      Base job for testing seed upgrades.
-
-      Configures the primary VM as a seed using the previous OpenStack release,
-      and upgrades it to the current release.
-    pre-run: playbooks/kayobe-seed-upgrade-base/pre.yml
-    run: playbooks/kayobe-seed-upgrade-base/run.yml
-    post-run: playbooks/kayobe-seed-upgrade-base/post.yml
-    attempts: 1
-    timeout: 5400
-    required-projects:
-      # Include kayobe to ensure other projects can use this job.
-      - name: openstack/kayobe
-      - name: openstack/kayobe-config-dev
-      - name: openstack/kolla
-        override-checkout: stable/queens
-      - name: openstack/kolla-ansible
-        override-checkout: stable/queens
-      - name: openstack/requirements
-        override-checkout: stable/queens
-    vars:
-      # Name of the release to upgrade from.
-      previous_release: pike
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^doc/.*
-      - ^releasenotes/.*
-      - ^setup.cfg$
-      - ^tools/.*$
-      - ^tox.ini$
-
-- job:
-    name: kayobe-seed-upgrade-centos
-    parent: kayobe-seed-upgrade-base
-    nodeset: kayobe-centos

zuul.d/project.yaml

@@ -12,9 +12,7 @@
         - kayobe-tox-ansible
         - kayobe-tox-molecule
         - kayobe-overcloud-centos
-        - kayobe-overcloud-upgrade-centos
         - kayobe-seed-centos
-        - kayobe-seed-upgrade-centos
 
     gate:
       queue: kayobe
@@ -24,6 +22,4 @@
         - kayobe-tox-ansible
         - kayobe-tox-molecule
         - kayobe-overcloud-centos
-        - kayobe-overcloud-upgrade-centos
         - kayobe-seed-centos
-        - kayobe-seed-upgrade-centos