Fix gate, squashing multiple fixes

This commit includes multiple fixes to make gate happy: 1. Use packaged uwsgi on Fedora and Ubuntu - https://review.opendev.org/#/c/735615/7 2. Move process-stackviz role from Tempest to Devstack - https://review.opendev.org/#/c/737078/4 3. Add ensure-pip to pre.yaml - https://review.opendev.org/#/c/738495/2 4. Use uwsgi binary from path - https://review.opendev.org/#/c/739072/ 5. Temporarily disable the grenade jobs Conflicts: .zuul.yaml lib/neutron lib/neutron-legacy Change-Id: I82f539bfa533349293dd5a8ce309c9cc0ffb0393 (cherry picked from commit 2d903568ed) (cherry picked from commit ad6cc52c74)
2020-06-30 17:28:15 -05:00 · 2020-06-30 17:28:15 -05:00 · f4b063b207
parent 12634ba6e2
commit f4b063b207
11 changed files with 158 additions and 34 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -343,10 +343,12 @@
              - ^.*\.rst$
              - ^doc/.*$
        - neutron-grenade:
+            voting: false
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
        - neutron-grenade-multinode:
+            voting: false
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
@ -364,18 +366,18 @@
        - devstack
        - devstack-unit-tests
        - openstack-tox-bashate
-        - neutron-grenade-multinode:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
+        #- neutron-grenade-multinode:
+        #    irrelevant-files:
+        #      - ^.*\.rst$
+        #      - ^doc/.*$
        - neutron-tempest-linuxbridge:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-grenade:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
+        #- neutron-grenade:
+        #    irrelevant-files:
+        #      - ^.*\.rst$
+        #      - ^doc/.*$
    experimental:
      jobs:
        - devstack-plugin-ceph-tempest:
--- a/lib/apache
+++ b/lib/apache
@ -82,26 +82,52 @@ function install_apache_uwsgi {
        apxs="apxs"
    fi

-    # Ubuntu xenial is back level on uwsgi so the proxy doesn't
-    # actually work. Hence we have to build from source for now.
+    # This varies based on packaged/installed.  If we've
+    # pip_installed, then the pip setup will only build a "python"
+    # module that will be either python2 or python3 depending on what
+    # it was built with.
    #
-    # Centos 7 actually has the module in epel, but there was a big
-    # push to disable epel by default. As such, compile from source
-    # there as well.
+    # For package installs, the distro ships both plugins and you need
+    # to select the right one ... it will not be autodetected.
+    if python3_enabled; then
+        UWSGI_PYTHON_PLUGIN=python3
+    else
+        UWSGI_PYTHON_PLUGIN=python
+    fi

-    local dir
-    dir=$(mktemp -d)
-    pushd $dir
-    pip_install uwsgi
-    pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt
-    local uwsgi
-    uwsgi=$(ls uwsgi*)
-    tar xvf $uwsgi
-    cd uwsgi*/apache2
-    sudo $apxs -i -c mod_proxy_uwsgi.c
-    popd
-    # delete the temp directory
-    sudo rm -rf $dir
+    if is_ubuntu; then
+        install_package uwsgi \
+                        uwsgi-plugin-python \
+                        uwsgi-plugin-python3 \
+                        libapache2-mod-proxy-uwsgi
+    elif [[ $os_VENDOR == "Fedora" ]]; then
+        # Note httpd comes with mod_proxy_uwsgi and it is loaded by
+        # default; the mod_proxy_uwsgi package actually conflicts now.
+        # See:
+        #  https://bugzilla.redhat.com/show_bug.cgi?id=1574335
+        #
+        # Thus there is nothing else to do after this install
+        install_package uwsgi \
+                        uwsgi-plugin-python3
+    else
+        # Centos actually has the module in epel, but there was a big
+        # push to disable epel by default. As such, compile from source
+        # there.
+        local dir
+        dir=$(mktemp -d)
+        pushd $dir
+        pip_install uwsgi
+        pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt
+        local uwsgi
+        uwsgi=$(ls uwsgi*)
+        tar xvf $uwsgi
+        cd uwsgi*/apache2
+        sudo $apxs -i -c mod_proxy_uwsgi.c
+        popd
+        # delete the temp directory
+        sudo rm -rf $dir
+        UWSGI_PYTHON_PLUGIN=python
+    fi

    if is_ubuntu || is_suse ; then
        # we've got to enable proxy and proxy_uwsgi for this to work
@ -265,7 +291,7 @@ function write_uwsgi_config {
    # configured after graceful shutdown
    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins python
+    iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN}
    # uwsgi recommends this to prevent thundering herd on accept.
    iniset "$file" uwsgi thunder-lock true
    # Set hook to trigger graceful shutdown on SIGTERM
@ -318,7 +344,7 @@ function write_local_uwsgi_http_config {
    iniset "$file" uwsgi die-on-term true
    iniset "$file" uwsgi exit-on-reload false
    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins python
+    iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN}
    # uwsgi recommends this to prevent thundering herd on accept.
    iniset "$file" uwsgi thunder-lock true
    # Set hook to trigger graceful shutdown on SIGTERM
--- a/lib/cinder
+++ b/lib/cinder
@ -507,7 +507,7 @@ function start_cinder {
                start_tls_proxy cinder '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_PORT_INT
            fi
        else
-            run_process "c-api" "$CINDER_BIN_DIR/uwsgi --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF"
+            run_process "c-api" "$(which uwsgi) --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF"
            cinder_url=$service_protocol://$SERVICE_HOST/volume/v3
        fi
    fi
--- a/lib/glance
+++ b/lib/glance
@ -354,7 +354,7 @@ function start_glance {

    run_process g-reg "$GLANCE_BIN_DIR/glance-registry --config-file=$GLANCE_CONF_DIR/glance-registry.conf"
    if [[ "$WSGI_MODE" == "uwsgi" ]]; then
-        run_process g-api "$GLANCE_BIN_DIR/uwsgi --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF"
+        run_process g-api "$(which uwsgi) --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF"
    else
        run_process g-api "$GLANCE_BIN_DIR/glance-api --config-file=$GLANCE_CONF_DIR/glance-api.conf"
    fi
--- a/lib/keystone
+++ b/lib/keystone
@ -550,7 +550,7 @@ function start_keystone {
        enable_apache_site keystone
        restart_apache_server
    else # uwsgi
-        run_process keystone "$KEYSTONE_BIN_DIR/uwsgi --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" ""
+        run_process keystone "$(which uwsgi) --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" ""
    fi

    echo "Waiting for keystone to start..."
--- a/lib/nova
+++ b/lib/nova
@ -833,7 +833,7 @@ function start_nova_api {
            start_tls_proxy nova '*' $NOVA_SERVICE_PORT $NOVA_SERVICE_HOST $NOVA_SERVICE_PORT_INT
        fi
    else
-        run_process "n-api" "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api --ini $NOVA_UWSGI_CONF"
+        run_process "n-api" "$(which uwsgi) --procname-prefix nova-api --ini $NOVA_UWSGI_CONF"
        nova_url=$service_protocol://$SERVICE_HOST/compute/v2.1/
    fi

@ -944,7 +944,7 @@ function start_nova_rest {
    if [ "$NOVA_USE_MOD_WSGI" == "False" ]; then
        run_process n-api-meta "$NOVA_BIN_DIR/nova-api-metadata --config-file $compute_cell_conf"
    else
-        run_process n-api-meta "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF"
+        run_process n-api-meta "$(which uwsgi) --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF"
    fi

    run_process n-novnc "$NOVA_BIN_DIR/nova-novncproxy --config-file $api_cell_conf --web $NOVNC_WEB_DIR"
--- a/lib/placement
+++ b/lib/placement
@ -170,7 +170,7 @@ function install_placement {
 # start_placement_api() - Start the API processes ahead of other things
 function start_placement_api {
    if [[ "$WSGI_MODE" == "uwsgi" ]]; then
-        run_process "placement-api" "$PLACEMENT_BIN_DIR/uwsgi --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF"
+        run_process "placement-api" "$(which uwsgi) --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF"
    else
        enable_apache_site placement-api
        restart_apache_server
--- a/playbooks/pre.yaml
+++ b/playbooks/pre.yaml
@ -28,3 +28,4 @@
    - setup-devstack-cache
    - start-fresh-logging
    - write-devstack-local-conf
+    - ensure-pip
--- a/roles/process-stackviz/README.rst
+++ b/roles/process-stackviz/README.rst
@ -0,0 +1,22 @@
+Generate stackviz report.
+
+Generate stackviz report using subunit and dstat data, using
+the stackviz archive embedded in test images.
+
+**Role Variables**
+
+.. zuul:rolevar:: devstack_base_dir
+   :default: /opt/stack
+
+   The devstack base directory.
+
+.. zuul:rolevar:: stage_dir
+   :default: "{{ ansible_user_dir }}"
+
+   The stage directory where the input data can be found and
+   the output will be produced.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ devstack_base_dir }}/tempest
+
+   Directory to work in. It has to be a fully qualified path.
--- a/roles/process-stackviz/defaults/main.yaml
+++ b/roles/process-stackviz/defaults/main.yaml
@ -0,0 +1,3 @@
+devstack_base_dir: /opt/stack
+stage_dir: "{{ ansible_user_dir }}"
+zuul_work_dir: "{{ devstack_base_dir }}/tempest"
--- a/roles/process-stackviz/tasks/main.yaml
+++ b/roles/process-stackviz/tasks/main.yaml
@ -0,0 +1,70 @@
+- name: Devstack checks if stackviz archive exists
+  stat:
+    path: "/opt/cache/files/stackviz-latest.tar.gz"
+  register: stackviz_archive
+
+- debug:
+    msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz"
+  when: not stackviz_archive.stat.exists
+
+- name: Check if subunit data exists
+  stat:
+    path: "{{ zuul_work_dir }}/testrepository.subunit"
+  register: subunit_input
+
+- debug:
+    msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit"
+  when: not subunit_input.stat.exists
+
+- name: Install stackviz
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+  block:
+    - include_role:
+        name: ensure-pip
+
+    - pip:
+        name: "file://{{ stackviz_archive.stat.path }}"
+        virtualenv: /tmp/stackviz
+        virtualenv_command: '{{ ensure_pip_virtualenv_command }}'
+        extra_args: -U
+
+- name: Deploy stackviz static html+js
+  command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+
+- name: Check if dstat data exists
+  stat:
+    path: "{{ devstack_base_dir }}/logs/dstat-csv.log"
+  register: dstat_input
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+
+- name: Run stackviz with dstat
+  shell: |
+    cat {{ subunit_input.stat.path }} | \
+      /tmp/stackviz/bin/stackviz-export \
+        --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \
+        --env --stdin \
+        {{ stage_dir }}/stackviz/data
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+    - dstat_input.stat.exists
+  failed_when: False
+
+- name: Run stackviz without dstat
+  shell: |
+    cat {{ subunit_input.stat.path }} | \
+      /tmp/stackviz/bin/stackviz-export \
+        --env --stdin \
+        {{ stage_dir }}/stackviz/data
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+    - not dstat_input.stat.exists
+  failed_when: False