From e963dc2fd4add7c438a15407b5e047957f690c4a Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Tue, 30 Jun 2020 17:28:15 -0500 Subject: [PATCH] Fix gate, squashing multiple fixes This commit includes multiple fixes to make gate happy: 1. Use packaged uwsgi on Fedora and Ubuntu - https://review.opendev.org/#/c/735615/7 2. Move process-stackviz role from Tempest to Devstack - https://review.opendev.org/#/c/737078/4 3. Add ensure-pip to pre.yaml - https://review.opendev.org/#/c/738495/2 4. Use uwsgi binary from path - https://review.opendev.org/#/c/739072/ 5. Temporarily disable the grenade jobs Conflicts: .zuul.yaml lib/neutron lib/neutron-legacy Change-Id: I82f539bfa533349293dd5a8ce309c9cc0ffb0393 (cherry picked from commit 2d903568ed4158aa668bbda6986307a8780c71a4) (cherry picked from commit ad6cc52c746f1fbdb1d89b14c09dc4aa7907bfe4) --- .zuul.yaml | 18 +++--- lib/apache | 66 ++++++++++++++------- lib/cinder | 2 +- lib/glance | 2 +- lib/keystone | 2 +- lib/nova | 4 +- lib/placement | 2 +- playbooks/pre.yaml | 1 + roles/process-stackviz/README.rst | 22 +++++++ roles/process-stackviz/defaults/main.yaml | 3 + roles/process-stackviz/tasks/main.yaml | 70 +++++++++++++++++++++++ 11 files changed, 158 insertions(+), 34 deletions(-) create mode 100644 roles/process-stackviz/README.rst create mode 100644 roles/process-stackviz/defaults/main.yaml create mode 100644 roles/process-stackviz/tasks/main.yaml diff --git a/.zuul.yaml b/.zuul.yaml index 84218894d4..7366182d70 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -398,10 +398,12 @@ - ^.*\.rst$ - ^doc/.*$ - neutron-grenade: + voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - neutron-grenade-multinode: + voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ @@ -419,18 +421,18 @@ - devstack - devstack-unit-tests - openstack-tox-bashate - - neutron-grenade-multinode: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + #- neutron-grenade-multinode: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ - neutron-tempest-linuxbridge: irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - - neutron-grenade: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + #- neutron-grenade: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ experimental: jobs: - devstack-plugin-ceph-tempest: diff --git a/lib/apache b/lib/apache index dfca25a764..2a5f046107 100644 --- a/lib/apache +++ b/lib/apache @@ -82,26 +82,52 @@ function install_apache_uwsgi { apxs="apxs" fi - # Ubuntu xenial is back level on uwsgi so the proxy doesn't - # actually work. Hence we have to build from source for now. + # This varies based on packaged/installed. If we've + # pip_installed, then the pip setup will only build a "python" + # module that will be either python2 or python3 depending on what + # it was built with. # - # Centos 7 actually has the module in epel, but there was a big - # push to disable epel by default. As such, compile from source - # there as well. + # For package installs, the distro ships both plugins and you need + # to select the right one ... it will not be autodetected. + if python3_enabled; then + UWSGI_PYTHON_PLUGIN=python3 + else + UWSGI_PYTHON_PLUGIN=python + fi - local dir - dir=$(mktemp -d) - pushd $dir - pip_install uwsgi - pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt - local uwsgi - uwsgi=$(ls uwsgi*) - tar xvf $uwsgi - cd uwsgi*/apache2 - sudo $apxs -i -c mod_proxy_uwsgi.c - popd - # delete the temp directory - sudo rm -rf $dir + if is_ubuntu; then + install_package uwsgi \ + uwsgi-plugin-python \ + uwsgi-plugin-python3 \ + libapache2-mod-proxy-uwsgi + elif [[ $os_VENDOR == "Fedora" ]]; then + # Note httpd comes with mod_proxy_uwsgi and it is loaded by + # default; the mod_proxy_uwsgi package actually conflicts now. + # See: + # https://bugzilla.redhat.com/show_bug.cgi?id=1574335 + # + # Thus there is nothing else to do after this install + install_package uwsgi \ + uwsgi-plugin-python3 + else + # Centos actually has the module in epel, but there was a big + # push to disable epel by default. As such, compile from source + # there. + local dir + dir=$(mktemp -d) + pushd $dir + pip_install uwsgi + pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt + local uwsgi + uwsgi=$(ls uwsgi*) + tar xvf $uwsgi + cd uwsgi*/apache2 + sudo $apxs -i -c mod_proxy_uwsgi.c + popd + # delete the temp directory + sudo rm -rf $dir + UWSGI_PYTHON_PLUGIN=python + fi if is_ubuntu || is_suse ; then # we've got to enable proxy and proxy_uwsgi for this to work @@ -261,7 +287,7 @@ function write_uwsgi_config { iniset "$file" uwsgi die-on-term true iniset "$file" uwsgi exit-on-reload true iniset "$file" uwsgi enable-threads true - iniset "$file" uwsgi plugins python + iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN} # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true # Override the default size for headers from the 4k default. @@ -313,7 +339,7 @@ function write_local_uwsgi_http_config { iniset "$file" uwsgi die-on-term true iniset "$file" uwsgi exit-on-reload true iniset "$file" uwsgi enable-threads true - iniset "$file" uwsgi plugins python + iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN} # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true # Override the default size for headers from the 4k default. diff --git a/lib/cinder b/lib/cinder index 22c5168089..4f48ba5dd1 100644 --- a/lib/cinder +++ b/lib/cinder @@ -511,7 +511,7 @@ function start_cinder { start_tls_proxy cinder '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_POR_INT fi else - run_process "c-api" "$CINDER_BIN_DIR/uwsgi --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF" + run_process "c-api" "$(which uwsgi) --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF" cinder_url=$service_protocol://$SERVICE_HOST/volume/v3 fi fi diff --git a/lib/glance b/lib/glance index 7b42488b59..6bfc3f6ce8 100644 --- a/lib/glance +++ b/lib/glance @@ -345,7 +345,7 @@ function start_glance { run_process g-reg "$GLANCE_BIN_DIR/glance-registry --config-file=$GLANCE_CONF_DIR/glance-registry.conf" if [[ "$WSGI_MODE" == "uwsgi" ]]; then - run_process g-api "$GLANCE_BIN_DIR/uwsgi --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF" + run_process g-api "$(which uwsgi) --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF" else run_process g-api "$GLANCE_BIN_DIR/glance-api --config-file=$GLANCE_CONF_DIR/glance-api.conf" fi diff --git a/lib/keystone b/lib/keystone index 7c2da27b17..3cce1da30d 100644 --- a/lib/keystone +++ b/lib/keystone @@ -552,7 +552,7 @@ function start_keystone { tail_log key /var/log/$APACHE_NAME/keystone.log tail_log key-access /var/log/$APACHE_NAME/keystone_access.log else # uwsgi - run_process keystone "$KEYSTONE_BIN_DIR/uwsgi --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" "" + run_process keystone "$(which uwsgi) --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" "" fi echo "Waiting for keystone to start..." diff --git a/lib/nova b/lib/nova index bc8b1432c7..06c68519be 100644 --- a/lib/nova +++ b/lib/nova @@ -799,7 +799,7 @@ function start_nova_api { start_tls_proxy nova '*' $NOVA_SERVICE_PORT $NOVA_SERVICE_HOST $NOVA_SERVICE_PORT_INT fi else - run_process "n-api" "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api --ini $NOVA_UWSGI_CONF" + run_process "n-api" "$(which uwsgi) --procname-prefix nova-api --ini $NOVA_UWSGI_CONF" nova_url=$service_protocol://$SERVICE_HOST/compute/v2.1/ fi @@ -903,7 +903,7 @@ function start_nova_rest { if [ "$NOVA_USE_MOD_WSGI" == "False" ]; then run_process n-api-meta "$NOVA_BIN_DIR/nova-api-metadata --config-file $compute_cell_conf" else - run_process n-api-meta "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF" + run_process n-api-meta "$(which uwsgi) --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF" fi run_process n-novnc "$NOVA_BIN_DIR/nova-novncproxy --config-file $api_cell_conf --web $NOVNC_WEB_DIR" diff --git a/lib/placement b/lib/placement index eda345dc50..a0c36cd611 100644 --- a/lib/placement +++ b/lib/placement @@ -165,7 +165,7 @@ function install_placement { # start_placement_api() - Start the API processes ahead of other things function start_placement_api { if [[ "$WSGI_MODE" == "uwsgi" ]]; then - run_process "placement-api" "$PLACEMENT_BIN_DIR/uwsgi --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF" + run_process "placement-api" "$(which uwsgi) --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF" else enable_apache_site placement-api restart_apache_server diff --git a/playbooks/pre.yaml b/playbooks/pre.yaml index 4689a6354f..d3ed29c7e1 100644 --- a/playbooks/pre.yaml +++ b/playbooks/pre.yaml @@ -28,3 +28,4 @@ - setup-devstack-cache - start-fresh-logging - write-devstack-local-conf + - ensure-pip diff --git a/roles/process-stackviz/README.rst b/roles/process-stackviz/README.rst new file mode 100644 index 0000000000..a8447d2355 --- /dev/null +++ b/roles/process-stackviz/README.rst @@ -0,0 +1,22 @@ +Generate stackviz report. + +Generate stackviz report using subunit and dstat data, using +the stackviz archive embedded in test images. + +**Role Variables** + +.. zuul:rolevar:: devstack_base_dir + :default: /opt/stack + + The devstack base directory. + +.. zuul:rolevar:: stage_dir + :default: "{{ ansible_user_dir }}" + + The stage directory where the input data can be found and + the output will be produced. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ devstack_base_dir }}/tempest + + Directory to work in. It has to be a fully qualified path. diff --git a/roles/process-stackviz/defaults/main.yaml b/roles/process-stackviz/defaults/main.yaml new file mode 100644 index 0000000000..f3bc32b149 --- /dev/null +++ b/roles/process-stackviz/defaults/main.yaml @@ -0,0 +1,3 @@ +devstack_base_dir: /opt/stack +stage_dir: "{{ ansible_user_dir }}" +zuul_work_dir: "{{ devstack_base_dir }}/tempest" diff --git a/roles/process-stackviz/tasks/main.yaml b/roles/process-stackviz/tasks/main.yaml new file mode 100644 index 0000000000..c51c66cdb3 --- /dev/null +++ b/roles/process-stackviz/tasks/main.yaml @@ -0,0 +1,70 @@ +- name: Devstack checks if stackviz archive exists + stat: + path: "/opt/cache/files/stackviz-latest.tar.gz" + register: stackviz_archive + +- debug: + msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz" + when: not stackviz_archive.stat.exists + +- name: Check if subunit data exists + stat: + path: "{{ zuul_work_dir }}/testrepository.subunit" + register: subunit_input + +- debug: + msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit" + when: not subunit_input.stat.exists + +- name: Install stackviz + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + block: + - include_role: + name: ensure-pip + + - pip: + name: "file://{{ stackviz_archive.stat.path }}" + virtualenv: /tmp/stackviz + virtualenv_command: '{{ ensure_pip_virtualenv_command }}' + extra_args: -U + +- name: Deploy stackviz static html+js + command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + +- name: Check if dstat data exists + stat: + path: "{{ devstack_base_dir }}/logs/dstat-csv.log" + register: dstat_input + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + +- name: Run stackviz with dstat + shell: | + cat {{ subunit_input.stat.path }} | \ + /tmp/stackviz/bin/stackviz-export \ + --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \ + --env --stdin \ + {{ stage_dir }}/stackviz/data + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + - dstat_input.stat.exists + failed_when: False + +- name: Run stackviz without dstat + shell: | + cat {{ subunit_input.stat.path }} | \ + /tmp/stackviz/bin/stackviz-export \ + --env --stdin \ + {{ stage_dir }}/stackviz/data + when: + - stackviz_archive.stat.exists + - subunit_input.stat.exists + - not dstat_input.stat.exists + failed_when: False