From ad6cc52c746f1fbdb1d89b14c09dc4aa7907bfe4 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Tue, 30 Jun 2020 17:28:15 -0500 Subject: [PATCH] Fix gate, squashing multiple fixes This commit includes multiple fixes to make gate happy: 1. Use packaged uwsgi on Fedora and Ubuntu - https://review.opendev.org/#/c/735615/7 2. Move process-stackviz role from Tempest to Devstack - https://review.opendev.org/#/c/737078/4 3. Add ensure-pip to pre.yaml - https://review.opendev.org/#/c/738495/2 4. Temporarily disable the grenade jobs Change-Id: I82f539bfa533349293dd5a8ce309c9cc0ffb0393 (cherry picked from commit 2d903568ed4158aa668bbda6986307a8780c71a4) --- .zuul.yaml | 18 +++--- lib/apache | 66 ++++++++++++++------- playbooks/pre.yaml | 1 + roles/process-stackviz/README.rst | 22 +++++++ roles/process-stackviz/defaults/main.yaml | 3 + roles/process-stackviz/tasks/main.yaml | 70 +++++++++++++++++++++++ 6 files changed, 152 insertions(+), 28 deletions(-) create mode 100644 roles/process-stackviz/README.rst create mode 100644 roles/process-stackviz/defaults/main.yaml create mode 100644 roles/process-stackviz/tasks/main.yaml diff --git a/.zuul.yaml b/.zuul.yaml index 01420702e1..a61dd84a77 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -477,10 +477,12 @@ - ^.*\.rst$ - ^doc/.*$ - neutron-grenade: + voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - neutron-grenade-multinode: + voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ @@ -498,18 +500,18 @@ - devstack - devstack-unit-tests - openstack-tox-bashate - - neutron-grenade-multinode: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + #- neutron-grenade-multinode: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ - neutron-tempest-linuxbridge: irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - - neutron-grenade: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + #- neutron-grenade: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ # Please add a note on each job and conditions for the job not # being experimental any more, so we can keep this list somewhat # pruned. diff --git a/lib/apache b/lib/apache index 84cec73234..a31188bebb 100644 --- a/lib/apache +++ b/lib/apache @@ -82,26 +82,52 @@ function install_apache_uwsgi { apxs="apxs" fi - # Ubuntu xenial is back level on uwsgi so the proxy doesn't - # actually work. Hence we have to build from source for now. + # This varies based on packaged/installed. If we've + # pip_installed, then the pip setup will only build a "python" + # module that will be either python2 or python3 depending on what + # it was built with. # - # Centos 7 actually has the module in epel, but there was a big - # push to disable epel by default. As such, compile from source - # there as well. + # For package installs, the distro ships both plugins and you need + # to select the right one ... it will not be autodetected. + if python3_enabled; then + UWSGI_PYTHON_PLUGIN=python3 + else + UWSGI_PYTHON_PLUGIN=python + fi - local dir - dir=$(mktemp -d) - pushd $dir - pip_install uwsgi - pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt - local uwsgi - uwsgi=$(ls uwsgi*) - tar xvf $uwsgi - cd uwsgi*/apache2 - sudo $apxs -i -c mod_proxy_uwsgi.c - popd - # delete the temp directory - sudo rm -rf $dir + if is_ubuntu; then + install_package uwsgi \ + uwsgi-plugin-python \ + uwsgi-plugin-python3 \ + libapache2-mod-proxy-uwsgi + elif [[ $os_VENDOR == "Fedora" ]]; then + # Note httpd comes with mod_proxy_uwsgi and it is loaded by + # default; the mod_proxy_uwsgi package actually conflicts now. + # See: + # https://bugzilla.redhat.com/show_bug.cgi?id=1574335 + # + # Thus there is nothing else to do after this install + install_package uwsgi \ + uwsgi-plugin-python3 + else + # Centos actually has the module in epel, but there was a big + # push to disable epel by default. As such, compile from source + # there. + local dir + dir=$(mktemp -d) + pushd $dir + pip_install uwsgi + pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt + local uwsgi + uwsgi=$(ls uwsgi*) + tar xvf $uwsgi + cd uwsgi*/apache2 + sudo $apxs -i -c mod_proxy_uwsgi.c + popd + # delete the temp directory + sudo rm -rf $dir + UWSGI_PYTHON_PLUGIN=python + fi if is_ubuntu || is_suse ; then # we've got to enable proxy and proxy_uwsgi for this to work @@ -265,7 +291,7 @@ function write_uwsgi_config { # configured after graceful shutdown iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT iniset "$file" uwsgi enable-threads true - iniset "$file" uwsgi plugins python + iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN} # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true # Set hook to trigger graceful shutdown on SIGTERM @@ -318,7 +344,7 @@ function write_local_uwsgi_http_config { iniset "$file" uwsgi die-on-term true iniset "$file" uwsgi exit-on-reload false iniset "$file" uwsgi enable-threads true - iniset "$file" uwsgi plugins python + iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN} # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true # Set hook to trigger graceful shutdown on SIGTERM diff --git a/playbooks/pre.yaml b/playbooks/pre.yaml index 4689a6354f..d3ed29c7e1 100644 --- a/playbooks/pre.yaml +++ b/playbooks/pre.yaml @@ -28,3 +28,4 @@ - setup-devstack-cache - start-fresh-logging - write-devstack-local-conf + - ensure-pip diff --git a/roles/process-stackviz/README.rst b/roles/process-stackviz/README.rst new file mode 100644 index 0000000000..a8447d2355 --- /dev/null +++ b/roles/process-stackviz/README.rst @@ -0,0 +1,22 @@ +Generate stackviz report. + +Generate stackviz report using subunit and dstat data, using +the stackviz archive embedded in test images. + +**Role Variables** + +.. zuul:rolevar:: devstack_base_dir + :default: /opt/stack + + The devstack base directory. + +.. zuul:rolevar:: stage_dir + :default: "{{ ansible_user_dir }}" + + The stage directory where the input data can be found and + the output will be produced. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ devstack_base_dir }}/tempest + + Directory to work in. It has to be a fully qualified path. diff --git a/roles/process-stackviz/defaults/main.yaml b/roles/process-stackviz/defaults/main.yaml new file mode 100644 index 0000000000..f3bc32b149 --- /dev/null +++ b/roles/process-stackviz/defaults/main.yaml @@ -0,0 +1,3 @@ +devstack_base_dir: /opt/stack +stage_dir: "{{ ansible_user_dir }}" +zuul_work_dir: "{{ devstack_base_dir }}/tempest" diff --git a/roles/process-stackviz/tasks/main.yaml b/roles/process-stackviz/tasks/main.yaml new file mode 100644 index 0000000000..c51c66cdb3 --- /dev/null +++ b/roles/process-stackviz/tasks/main.yaml @@ -0,0 +1,70 @@ +- name: Devstack checks if stackviz archive exists + stat: + path: "/opt/cache/files/stackviz-latest.tar.gz" + register: stackviz_archive + +- debug: + msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz" + when: not stackviz_archive.stat.exists + +- name: Check if subunit data exists + stat: + path: "{{ zuul_work_dir }}/testrepository.subunit" + register: subunit_input + +- debug: + msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit" + when: not subunit_input.stat.exists + +- name: Install stackviz + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + block: + - include_role: + name: ensure-pip + + - pip: + name: "file://{{ stackviz_archive.stat.path }}" + virtualenv: /tmp/stackviz + virtualenv_command: '{{ ensure_pip_virtualenv_command }}' + extra_args: -U + +- name: Deploy stackviz static html+js + command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + +- name: Check if dstat data exists + stat: + path: "{{ devstack_base_dir }}/logs/dstat-csv.log" + register: dstat_input + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + +- name: Run stackviz with dstat + shell: | + cat {{ subunit_input.stat.path }} | \ + /tmp/stackviz/bin/stackviz-export \ + --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \ + --env --stdin \ + {{ stage_dir }}/stackviz/data + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + - dstat_input.stat.exists + failed_when: False + +- name: Run stackviz without dstat + shell: | + cat {{ subunit_input.stat.path }} | \ + /tmp/stackviz/bin/stackviz-export \ + --env --stdin \ + {{ stage_dir }}/stackviz/data + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + - not dstat_input.stat.exists + failed_when: False