From 57b248f9fe7f4701975a71bfe8eab982086b1664 Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Tue, 29 Apr 2025 14:29:11 +0100 Subject: [PATCH] Add support for pyproject.toml and wsgi module paths pip 23.1 removed the "setup.py install" fallback for projects that do not have pyproject.toml and now uses a pyproject.toml which is vendored in pip [1][2]. pip 24.2 has now deprecated a similar fallback to "setup.py develop" and plans to fully remove this in pip 25.0 [3][4][5]. pbr supports editable installs since 6.0.0 pip 25.1 has now been released and the removal is complete. by adding our own minimal pyproject.toml to ensure we are using the correct build system. This change also requires that we adapt how we generate our wsgi entry point. when pyproject.toml is used the wsgi console script is not generated in an editbale install such as is used in devstck To adress this we need to refactor our usage of our wsgi applciation to use a module path instead. This change does not remove the declaration of our wsgi_scrtip entry point but it shoudl be considered deprecated and it will be removed in the future. To unblock the gate the devstack plugin is modifed to to deploy using the wsgi module instead of the console script. Finally supprot for the mod_wsgi wsgi mode is removed. that was deprecated in devstack a few cycle ago and support was removed in I8823e98809ed6b66c27dbcf21a00eea68ef403e8 [1] https://pip.pypa.io/en/stable/news/#v23-1 [2] https://github.com/pypa/pip/issues/8368 [3] https://pip.pypa.io/en/stable/news/#v24-2 [4] https://github.com/pypa/pip/issues/11457 [5] https://ichard26.github.io/blog/2024/08/whats-new-in-pip-24.2/ Closes-Bug: #2109608 Depends-on: https://review.opendev.org/c/openstack/watcher/+/948502 Change-Id: Iad77939ab0403c5720c549f96edfc77d2b7d90ee --- devstack/lib/watcher | 83 +++---------------- pyproject.toml | 3 + ...-wsgi-module-support-597f479e31979270.yaml | 30 +++++++ ...with-multiple-target-0e65d20711d1abe2.yaml | 7 +- setup.cfg | 2 +- tox.ini | 11 +-- watcher/wsgi/__init__.py | 0 watcher/wsgi/api.py | 18 ++++ 8 files changed, 69 insertions(+), 85 deletions(-) create mode 100644 pyproject.toml create mode 100644 releasenotes/notes/add-wsgi-module-support-597f479e31979270.yaml create mode 100644 watcher/wsgi/__init__.py create mode 100644 watcher/wsgi/api.py diff --git a/devstack/lib/watcher b/devstack/lib/watcher index bad94ff19..2baf4f561 100644 --- a/devstack/lib/watcher +++ b/devstack/lib/watcher @@ -55,11 +55,7 @@ else WATCHER_BIN_DIR=$(get_python_exec_prefix) fi -# There are 2 modes, which is "uwsgi" which runs with an apache -# proxy uwsgi in front of it, or "mod_wsgi", which runs in -# apache. mod_wsgi is deprecated, don't use it. -WATCHER_USE_WSGI_MODE=${WATCHER_USE_WSGI_MODE:-$WSGI_MODE} -WATCHER_UWSGI=$WATCHER_BIN_DIR/watcher-api-wsgi +WATCHER_UWSGI=watcher.wsgi.api:application WATCHER_UWSGI_CONF=$WATCHER_CONF_DIR/watcher-uwsgi.ini if is_suse; then @@ -73,11 +69,7 @@ WATCHER_SERVICE_PORT=${WATCHER_SERVICE_PORT:-9322} WATCHER_SERVICE_PORT_INT=${WATCHER_SERVICE_PORT_INT:-19322} WATCHER_SERVICE_PROTOCOL=${WATCHER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL} -if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then - WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST/infra-optim" -else - WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT" -fi +WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST/infra-optim" # Entry Points # ------------ @@ -101,11 +93,7 @@ function _cleanup_watcher_apache_wsgi { # runs that a clean run would need to clean up function cleanup_watcher { sudo rm -rf $WATCHER_STATE_PATH - if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then - remove_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" - else - _cleanup_watcher_apache_wsgi - fi + remove_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" } # configure_watcher() - Set config files, create data dirs, etc @@ -154,31 +142,6 @@ function create_watcher_accounts { "$WATCHER_API_URL" } -# _config_watcher_apache_wsgi() - Set WSGI config files of watcher -function _config_watcher_apache_wsgi { - local watcher_apache_conf - if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then - local service_port=$WATCHER_SERVICE_PORT - if is_service_enabled tls-proxy; then - service_port=$WATCHER_SERVICE_PORT_INT - service_protocol="http" - fi - sudo mkdir -p $WATCHER_WSGI_DIR - sudo cp $WATCHER_DIR/watcher/api/app.wsgi $WATCHER_WSGI_DIR/app.wsgi - watcher_apache_conf=$(apache_site_config_for watcher-api) - sudo cp $WATCHER_DEVSTACK_FILES_DIR/apache-watcher-api.template $watcher_apache_conf - sudo sed -e " - s|%WATCHER_SERVICE_PORT%|$service_port|g; - s|%WATCHER_WSGI_DIR%|$WATCHER_WSGI_DIR|g; - s|%USER%|$STACK_USER|g; - s|%APIWORKERS%|$API_WORKERS|g; - s|%APACHE_NAME%|$APACHE_NAME|g; - " -i $watcher_apache_conf - enable_apache_site watcher-api - fi - -} - # create_watcher_conf() - Create a new watcher.conf file function create_watcher_conf { # (Re)create ``watcher.conf`` @@ -196,11 +159,6 @@ function create_watcher_conf { iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)" iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT_INT" # iniset $WATCHER_CONF api enable_ssl_api "True" - else - if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then - iniset $WATCHER_CONF api host "$(ipv6_unquote $WATCHER_SERVICE_HOST)" - iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT" - fi fi iniset $WATCHER_CONF oslo_policy policy_file $WATCHER_POLICY_YAML @@ -228,12 +186,8 @@ function create_watcher_conf { # Format logging setup_logging $WATCHER_CONF - #config apache files - if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then - write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim" - else - _config_watcher_apache_wsgi - fi + write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim" "" "watcher-api" + # Register SSL certificates if provided if is_ssl_enabled_service watcher; then ensure_certificates WATCHER @@ -273,9 +227,6 @@ function install_watcherclient { function install_watcher { git_clone $WATCHER_REPO $WATCHER_DIR $WATCHER_BRANCH setup_develop $WATCHER_DIR - if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then - install_apache_wsgi - fi } # start_watcher_api() - Start the API process ahead of other things @@ -289,19 +240,10 @@ function start_watcher_api { service_port=$WATCHER_SERVICE_PORT_INT service_protocol="http" fi - if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then - run_process "watcher-api" "$(which uwsgi) --procname-prefix watcher-api --ini $WATCHER_UWSGI_CONF" - watcher_url=$service_protocol://$SERVICE_HOST/infra-optim - else - watcher_url=$service_protocol://$SERVICE_HOST:$service_port - enable_apache_site watcher-api - restart_apache_server - # Start proxies if enabled - if is_service_enabled tls-proxy; then - start_tls_proxy watcher '*' $WATCHER_SERVICE_PORT $WATCHER_SERVICE_HOST $WATCHER_SERVICE_PORT_INT - fi - fi - + run_process "watcher-api" "$(which uwsgi) --procname-prefix watcher-api --ini $WATCHER_UWSGI_CONF" + watcher_url=$service_protocol://$SERVICE_HOST/infra-optim + # TODO(sean-k-mooney): we should probably check that we can hit + # the microversion endpoint and get a valid response. echo "Waiting for watcher-api to start..." if ! wait_for_service $SERVICE_TIMEOUT $watcher_url; then die $LINENO "watcher-api did not start" @@ -319,12 +261,7 @@ function start_watcher { # stop_watcher() - Stop running processes (non-screen) function stop_watcher { - if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then - stop_process watcher-api - else - disable_apache_site watcher-api - restart_apache_server - fi + stop_process watcher-api for serv in watcher-decision-engine watcher-applier; do stop_process $serv done diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..5e862a959 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["pbr>=6.0.0", "setuptools>=64.0.0"] +build-backend = "pbr.build" diff --git a/releasenotes/notes/add-wsgi-module-support-597f479e31979270.yaml b/releasenotes/notes/add-wsgi-module-support-597f479e31979270.yaml new file mode 100644 index 000000000..a335da9ce --- /dev/null +++ b/releasenotes/notes/add-wsgi-module-support-597f479e31979270.yaml @@ -0,0 +1,30 @@ +--- +features: + - | + A new module, ``watcher.wsgi``, has been added as a place to gather WSGI + ``application`` objects. This is intended to ease deployment by providing + a consistent location for these objects. For example, if using uWSGI then + instead of: + + .. code-block:: ini + + [uwsgi] + wsgi-file = /bin/watcher-api-wsgi + + You can now use: + + .. code-block:: ini + + [uwsgi] + module = watcher.wsgi.api:application + + This also simplifies deployment with other WSGI servers that expect module + paths such as gunicorn. +deprecations: + - | + The watcher-api-wsgi console script is deprecated for removal + in a future release. This artifact is generated using a setup-tools + extension that is provide by PBR which is also deprecated. + due to the changes in python packaging this custom extensions + is planned to be removed form all OpenStack projects in a future + PBR release in favor of module based wsgi applications entry points. diff --git a/releasenotes/notes/bug-2103451-fixes-prometheus-queries-with-multiple-target-0e65d20711d1abe2.yaml b/releasenotes/notes/bug-2103451-fixes-prometheus-queries-with-multiple-target-0e65d20711d1abe2.yaml index c96ce6718..36a3b8568 100644 --- a/releasenotes/notes/bug-2103451-fixes-prometheus-queries-with-multiple-target-0e65d20711d1abe2.yaml +++ b/releasenotes/notes/bug-2103451-fixes-prometheus-queries-with-multiple-target-0e65d20711d1abe2.yaml @@ -2,8 +2,7 @@ fixes: - | When using prometheus datasource and more that one target has the same value - for the `fqdn_label`, the driver used the wrong instance label to query for host - metrics. The `instance` label is no longer used in the queries but the `fqdn_label` + for the ``fqdn_label``, the driver used the wrong instance label to query for host + metrics. The ``instance`` label is no longer used in the queries but the ``fqdn_label`` which identifies all the metrics for a specific compute node. - - .. _Bug 2103451: https://bugs.launchpad.net/watcher/+bug/2103451 + see Bug 2103451: https://bugs.launchpad.net/watcher/+bug/2103451 for more info. diff --git a/setup.cfg b/setup.cfg index 0ca8e5fbe..c3ee76534 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,6 +6,7 @@ description_file = author = OpenStack author_email = openstack-discuss@lists.openstack.org home_page = https://docs.openstack.org/watcher/latest/ +# TODO(sean-k-mooney): bump to >= 3.10 before m3. python_requires = >=3.9 classifier = Environment :: OpenStack @@ -17,7 +18,6 @@ classifier = Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3 - Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 diff --git a/tox.ini b/tox.ini index d41666428..349e5300c 100644 --- a/tox.ini +++ b/tox.ini @@ -106,8 +106,10 @@ commands = make -C doc/build/pdf [testenv:releasenotes] -deps = -r{toxinidir}/doc/requirements.txt -commands = sphinx-build -a -W -E -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html +deps = {[testenv:docs]deps} +commands = + rm -rf releasenotes/build + sphinx-build -W --keep-going -b html -j auto releasenotes/source releasenotes/build/html [testenv:bandit] skip_install = true @@ -146,8 +148,3 @@ extension = N342 = checks:no_redundant_import_alias N366 = checks:import_stock_mock paths = ./watcher/hacking - -[doc8] -extension=.rst -# todo: stop ignoring doc/source/man when https://bugs.launchpad.net/doc8/+bug/1502391 is fixed -ignore-path=doc/source/image_src,doc/source/man,doc/source/api diff --git a/watcher/wsgi/__init__.py b/watcher/wsgi/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/watcher/wsgi/api.py b/watcher/wsgi/api.py new file mode 100644 index 000000000..695ca7924 --- /dev/null +++ b/watcher/wsgi/api.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""WSGI application entry-point for Watcher API.""" +import threading +from watcher.api import wsgi +application = None +with threading.Lock(): + if application is None: + application = wsgi.initialize_wsgi_app()