Fix gate, squashing multiple fixes

This commit includes multiple fixes to
make gate happy:

1. Use packaged uwsgi on Fedora and Ubuntu
- https://review.opendev.org/#/c/735615/7

2. Move process-stackviz role from Tempest to Devstack
- https://review.opendev.org/#/c/737078/4

3. Add ensure-pip to pre.yaml
- https://review.opendev.org/#/c/738495/2

4. Temporarily disable the grenade jobs

Change-Id: I82f539bfa533349293dd5a8ce309c9cc0ffb0393
(cherry picked from commit 2d903568ed)

.zuul.yaml

@@ -477,10 +477,12 @@
               - ^.*\.rst$
               - ^doc/.*$
         - neutron-grenade:
+            voting: false
             irrelevant-files:
               - ^.*\.rst$
               - ^doc/.*$
         - neutron-grenade-multinode:
+            voting: false
             irrelevant-files:
               - ^.*\.rst$
               - ^doc/.*$
@@ -498,18 +500,18 @@
         - devstack
         - devstack-unit-tests
         - openstack-tox-bashate
-        - neutron-grenade-multinode:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
+        #- neutron-grenade-multinode:
+        #    irrelevant-files:
+        #      - ^.*\.rst$
+        #      - ^doc/.*$
         - neutron-tempest-linuxbridge:
             irrelevant-files:
               - ^.*\.rst$
               - ^doc/.*$
-        - neutron-grenade:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
+        #- neutron-grenade:
+        #    irrelevant-files:
+        #      - ^.*\.rst$
+        #      - ^doc/.*$
     # Please add a note on each job and conditions for the job not
     # being experimental any more, so we can keep this list somewhat
     # pruned.

lib/apache

@@ -82,26 +82,52 @@ function install_apache_uwsgi {
         apxs="apxs"
     fi
 
-    # Ubuntu xenial is back level on uwsgi so the proxy doesn't
-    # actually work. Hence we have to build from source for now.
+    # This varies based on packaged/installed.  If we've
+    # pip_installed, then the pip setup will only build a "python"
+    # module that will be either python2 or python3 depending on what
+    # it was built with.
     #
-    # Centos 7 actually has the module in epel, but there was a big
-    # push to disable epel by default. As such, compile from source
-    # there as well.
+    # For package installs, the distro ships both plugins and you need
+    # to select the right one ... it will not be autodetected.
+    if python3_enabled; then
+        UWSGI_PYTHON_PLUGIN=python3
+    else
+        UWSGI_PYTHON_PLUGIN=python
+    fi
 
-    local dir
-    dir=$(mktemp -d)
-    pushd $dir
-    pip_install uwsgi
-    pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt
-    local uwsgi
-    uwsgi=$(ls uwsgi*)
-    tar xvf $uwsgi
-    cd uwsgi*/apache2
-    sudo $apxs -i -c mod_proxy_uwsgi.c
-    popd
-    # delete the temp directory
-    sudo rm -rf $dir
+    if is_ubuntu; then
+        install_package uwsgi \
+                        uwsgi-plugin-python \
+                        uwsgi-plugin-python3 \
+                        libapache2-mod-proxy-uwsgi
+    elif [[ $os_VENDOR == "Fedora" ]]; then
+        # Note httpd comes with mod_proxy_uwsgi and it is loaded by
+        # default; the mod_proxy_uwsgi package actually conflicts now.
+        # See:
+        #  https://bugzilla.redhat.com/show_bug.cgi?id=1574335
+        #
+        # Thus there is nothing else to do after this install
+        install_package uwsgi \
+                        uwsgi-plugin-python3
+    else
+        # Centos actually has the module in epel, but there was a big
+        # push to disable epel by default. As such, compile from source
+        # there.
+        local dir
+        dir=$(mktemp -d)
+        pushd $dir
+        pip_install uwsgi
+        pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt
+        local uwsgi
+        uwsgi=$(ls uwsgi*)
+        tar xvf $uwsgi
+        cd uwsgi*/apache2
+        sudo $apxs -i -c mod_proxy_uwsgi.c
+        popd
+        # delete the temp directory
+        sudo rm -rf $dir
+        UWSGI_PYTHON_PLUGIN=python
+    fi
 
     if is_ubuntu || is_suse ; then
         # we've got to enable proxy and proxy_uwsgi for this to work
@@ -265,7 +291,7 @@ function write_uwsgi_config {
     # configured after graceful shutdown
     iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
     iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins python
+    iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN}
     # uwsgi recommends this to prevent thundering herd on accept.
     iniset "$file" uwsgi thunder-lock true
     # Set hook to trigger graceful shutdown on SIGTERM
@@ -318,7 +344,7 @@ function write_local_uwsgi_http_config {
     iniset "$file" uwsgi die-on-term true
     iniset "$file" uwsgi exit-on-reload false
     iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins python
+    iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN}
     # uwsgi recommends this to prevent thundering herd on accept.
     iniset "$file" uwsgi thunder-lock true
     # Set hook to trigger graceful shutdown on SIGTERM

playbooks/pre.yaml

@@ -28,3 +28,4 @@
     - setup-devstack-cache
     - start-fresh-logging
     - write-devstack-local-conf
+    - ensure-pip

roles/process-stackviz/README.rst

@@ -0,0 +1,22 @@
+Generate stackviz report.
+
+Generate stackviz report using subunit and dstat data, using
+the stackviz archive embedded in test images.
+
+**Role Variables**
+
+.. zuul:rolevar:: devstack_base_dir
+   :default: /opt/stack
+
+   The devstack base directory.
+
+.. zuul:rolevar:: stage_dir
+   :default: "{{ ansible_user_dir }}"
+
+   The stage directory where the input data can be found and
+   the output will be produced.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ devstack_base_dir }}/tempest
+
+   Directory to work in. It has to be a fully qualified path.

roles/process-stackviz/defaults/main.yaml

@@ -0,0 +1,3 @@
+devstack_base_dir: /opt/stack
+stage_dir: "{{ ansible_user_dir }}"
+zuul_work_dir: "{{ devstack_base_dir }}/tempest"

roles/process-stackviz/tasks/main.yaml

@@ -0,0 +1,70 @@
+- name: Devstack checks if stackviz archive exists
+  stat:
+    path: "/opt/cache/files/stackviz-latest.tar.gz"
+  register: stackviz_archive
+
+- debug:
+    msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz"
+  when: not stackviz_archive.stat.exists
+
+- name: Check if subunit data exists
+  stat:
+    path: "{{ zuul_work_dir }}/testrepository.subunit"
+  register: subunit_input
+
+- debug:
+    msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit"
+  when: not subunit_input.stat.exists
+
+- name: Install stackviz
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+  block:
+    - include_role:
+        name: ensure-pip
+
+    - pip:
+        name: "file://{{ stackviz_archive.stat.path }}"
+        virtualenv: /tmp/stackviz
+        virtualenv_command: '{{ ensure_pip_virtualenv_command }}'
+        extra_args: -U
+
+- name: Deploy stackviz static html+js
+  command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+
+- name: Check if dstat data exists
+  stat:
+    path: "{{ devstack_base_dir }}/logs/dstat-csv.log"
+  register: dstat_input
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+
+- name: Run stackviz with dstat
+  shell: |
+    cat {{ subunit_input.stat.path }} | \
+      /tmp/stackviz/bin/stackviz-export \
+        --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \
+        --env --stdin \
+        {{ stage_dir }}/stackviz/data
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+    - dstat_input.stat.exists
+  failed_when: False
+
+- name: Run stackviz without dstat
+  shell: |
+    cat {{ subunit_input.stat.path }} | \
+      /tmp/stackviz/bin/stackviz-export \
+        --env --stdin \
+        {{ stage_dir }}/stackviz/data
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+    - not dstat_input.stat.exists
+  failed_when: False