Add uWSGI support

This patch implements uWSGI support for Watcher API service.
Because mod_wsgi is deprecated, using uwsgi to replace of mod_wsgi.
Most of Openstack projects have finished it.

Closes-Bug: #1834392
Change-Id: I3fad8d30a15aba493fb91da9337c2515ddea5167

devstack/lib/watcher

@@ -51,7 +51,19 @@ if is_ssl_enabled_service "watcher" || is_service_enabled tls-proxy; then
     WATCHER_SERVICE_PROTOCOL="https"
 fi
 
-WATCHER_USE_MOD_WSGI=$(trueorfalse True WATCHER_USE_MOD_WSGI)
+# Support entry points installation of console scripts
+if [[ -d $WATCHER_DIR/bin ]]; then
+    WATCHER_BIN_DIR=$WATCHER_DIR/bin
+else
+    WATCHER_BIN_DIR=$(get_python_exec_prefix)
+fi
+
+# There are 2 modes, which is "uwsgi" which runs with an apache
+# proxy uwsgi in front of it, or "mod_wsgi", which runs in
+# apache. mod_wsgi is deprecated, don't use it.
+WATCHER_USE_WSGI_MODE=${WATCHER_USE_WSGI_MODE:-$WSGI_MODE}
+WATCHER_UWSGI=$WATCHER_BIN_DIR/watcher-api-wsgi
+WATCHER_UWSGI_CONF=$WATCHER_CONF_DIR/watcher-uwsgi.ini
 
 if is_suse; then
     WATCHER_WSGI_DIR=${WATCHER_WSGI_DIR:-/srv/www/htdocs/watcher}
@@ -64,11 +76,10 @@ WATCHER_SERVICE_PORT=${WATCHER_SERVICE_PORT:-9322}
 WATCHER_SERVICE_PORT_INT=${WATCHER_SERVICE_PORT_INT:-19322}
 WATCHER_SERVICE_PROTOCOL=${WATCHER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 
-# Support entry points installation of console scripts
-if [[ -d $WATCHER_DIR/bin ]]; then
-    WATCHER_BIN_DIR=$WATCHER_DIR/bin
+if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+    WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST/infra-optim"
 else
-    WATCHER_BIN_DIR=$(get_python_exec_prefix)
+    WATCHER_API_URL="$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT"
 fi
 
 # Entry Points
@@ -93,7 +104,9 @@ function _cleanup_watcher_apache_wsgi {
 # runs that a clean run would need to clean up
 function cleanup_watcher {
     sudo rm -rf $WATCHER_STATE_PATH $WATCHER_AUTH_CACHE_DIR
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+        remove_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI"
+    else
         _cleanup_watcher_apache_wsgi
     fi
 }
@@ -139,15 +152,15 @@ function create_watcher_accounts {
         "infra-optim" "Watcher Infrastructure Optimization Service")
     get_or_create_endpoint $watcher_service \
         "$REGION_NAME" \
-        "$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT" \
-        "$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT" \
-        "$WATCHER_SERVICE_PROTOCOL://$WATCHER_SERVICE_HOST:$WATCHER_SERVICE_PORT"
+        "$WATCHER_API_URL"\
+        "$WATCHER_API_URL"\
+        "$WATCHER_API_URL"
 }
 
 # _config_watcher_apache_wsgi() - Set WSGI config files of watcher
 function _config_watcher_apache_wsgi {
     local watcher_apache_conf
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
         local service_port=$WATCHER_SERVICE_PORT
         if is_service_enabled tls-proxy; then
             service_port=$WATCHER_SERVICE_PORT_INT
@@ -165,8 +178,6 @@ function _config_watcher_apache_wsgi {
             s|%APACHE_NAME%|$APACHE_NAME|g;
         " -i $watcher_apache_conf
         enable_apache_site watcher-api
-        tail_log watcher-access /var/log/$APACHE_NAME/watcher-api-access.log
-        tail_log watcher-api /var/log/$APACHE_NAME/watcher-api.log
    fi
 
 }
@@ -185,10 +196,14 @@ function create_watcher_conf {
     iniset $WATCHER_CONF api host "$WATCHER_SERVICE_HOST"
 
     if is_service_enabled tls-proxy; then
+        iniset $WATCHER_CONF api host "$WATCHER_SERVICE_HOST"
         iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT_INT"
         # iniset $WATCHER_CONF api enable_ssl_api "True"
     else
-        iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT"
+        if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
+            iniset $WATCHER_CONF api host "$WATCHER_SERVICE_HOST"
+            iniset $WATCHER_CONF api port "$WATCHER_SERVICE_PORT"
+        fi
     fi
 
     iniset $WATCHER_CONF oslo_policy policy_file $WATCHER_POLICY_YAML
@@ -217,7 +232,9 @@ function create_watcher_conf {
     setup_logging $WATCHER_CONF
 
     #config apache files
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+        write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim"
+    else
         _config_watcher_apache_wsgi
     fi
     # Register SSL certificates if provided
@@ -268,7 +285,7 @@ function install_watcherclient {
 function install_watcher {
     git_clone $WATCHER_REPO $WATCHER_DIR $WATCHER_BRANCH
     setup_develop $WATCHER_DIR
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "mod_wsgi" ]]; then
         install_apache_wsgi
     fi
 }
@@ -279,24 +296,26 @@ function start_watcher_api {
 
     local service_port=$WATCHER_SERVICE_PORT
     local service_protocol=$WATCHER_SERVICE_PROTOCOL
+    local watcher_url
     if is_service_enabled tls-proxy; then
         service_port=$WATCHER_SERVICE_PORT_INT
         service_protocol="http"
     fi
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+        run_process "watcher-api" "$WATCHER_BIN_DIR/uwsgi --ini $WATCHER_UWSGI_CONF"
+        watcher_url=$service_protocol://$SERVICE_HOST/infra-optim
+    else
+        watcher_url=$service_protocol://$SERVICE_HOST:$service_port
         enable_apache_site watcher-api
         restart_apache_server
-    else
-        run_process watcher-api "$WATCHER_BIN_DIR/watcher-api --config-file $WATCHER_CONF"
-    fi
-
-    # Start proxies if enabled
-    if is_service_enabled tls-proxy; then
-        start_tls_proxy watcher '*' $WATCHER_SERVICE_PORT $WATCHER_SERVICE_HOST $WATCHER_SERVICE_PORT_INT
+        # Start proxies if enabled
+        if is_service_enabled tls-proxy; then
+            start_tls_proxy watcher '*' $WATCHER_SERVICE_PORT $WATCHER_SERVICE_HOST $WATCHER_SERVICE_PORT_INT
+        fi
     fi
 
     echo "Waiting for watcher-api to start..."
-    if ! wait_for_service $SERVICE_TIMEOUT $service_protocol://$WATCHER_SERVICE_HOST:$service_port; then
+    if ! wait_for_service $SERVICE_TIMEOUT $watcher_url; then
         die $LINENO "watcher-api did not start"
     fi
 
@@ -312,11 +331,11 @@ function start_watcher {
 
 # stop_watcher() - Stop running processes (non-screen)
 function stop_watcher {
-    if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+    if [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]; then
+        stop_process watcher-api
+    else
         disable_apache_site watcher-api
         restart_apache_server
-    else
-        stop_process watcher-api
     fi
     for serv in watcher-decision-engine watcher-applier; do
         stop_process $serv

devstack/upgrade/upgrade.sh

@@ -40,6 +40,10 @@ set -o errexit
 source $TARGET_DEVSTACK_DIR/stackrc
 source $TARGET_DEVSTACK_DIR/lib/apache
 source $TARGET_DEVSTACK_DIR/lib/tls
+source $TARGET_DEVSTACK_DIR/lib/keystone
+
+source $TOP_DIR/openrc admin admin
+
 source $(dirname $(dirname $BASH_SOURCE))/settings
 source $(dirname $(dirname $BASH_SOURCE))/plugin.sh
 
@@ -56,6 +60,15 @@ install_watcher
 # calls upgrade-watcher for specific release
 upgrade_project watcher $RUN_DIR $BASE_DEVSTACK_BRANCH $TARGET_DEVSTACK_BRANCH
 
+if [[ ! -f "$WATCHER_UWSGI_CONF" ]] && [[ "$WATCHER_USE_WSGI_MODE" == "uwsgi" ]]
+then write_uwsgi_config "$WATCHER_UWSGI_CONF" "$WATCHER_UWSGI" "/infra-optim"
+     endpoints=$(openstack endpoint list --service watcher -c ID -f value)
+     for id in $endpoints; do
+         openstack endpoint delete $id
+     done
+     create_watcher_accounts
+fi
+
 # Migrate the database
 watcher-db-manage upgrade || die $LINO "DB migration error"
 

etc/apache2/watcher

@@ -16,7 +16,7 @@ Listen 9322
 
 <VirtualHost *:9322>
     WSGIDaemonProcess watcher-api user=stack group=stack processes=2 threads=2 display-name=%{GROUP}
-    WSGIScriptAlias / /opt/stack/watcher/watcher/api/app.wsgi
+    WSGIScriptAlias / /usr/local/bin/watcher-api-wsgi
     WSGIProcessGroup watcher-api
 
     ErrorLog /var/log/httpd/watcher_error.log

releasenotes/notes/uwsgi-support-8dcea6961e56dad0.yaml

@@ -0,0 +1,12 @@
+---
+upgrade:
+  - |
+    An Watcher API WSGI application script ``watcher-api-wsgi`` is now
+    available. It is auto-generated by ``pbr`` and allows to run the API
+    service using WSGI server (for example Nginx and uWSGI).
+deprecations:
+  - |
+    Using ``watcher/api/app.wsgi`` script is deprecated and it will be removed
+    in U release.
+    Please switch to automatically generated ``watcher-api-wsgi`` script
+    instead.

setup.cfg

@@ -46,6 +46,9 @@ console_scripts =
     watcher-sync = watcher.cmd.sync:main
     watcher-status = watcher.cmd.status:main
 
+wsgi_scripts =
+    watcher-api-wsgi = watcher.api.wsgi:initialize_wsgi_app
+
 watcher.database.migration_backend =
     sqlalchemy = watcher.db.sqlalchemy.migration
 

watcher/api/app.wsgi

@@ -16,24 +16,9 @@
 Use this file for deploying the API service under Apache2 mod_wsgi.
 """
 
-import sys
 
-from oslo_config import cfg
-import oslo_i18n as i18n
-from oslo_log import log
+# This script is deprecated and it will be removed in U release.
+# Please switch to automatically generated watcher-api-wsgi script instead.
+from watcher.api import wsgi
 
-from watcher.api import app
-from watcher.common import service
-
-
-CONF = cfg.CONF
-
-i18n.install('watcher')
-
-service.prepare_service(sys.argv)
-
-LOG = log.getLogger(__name__)
-LOG.debug("Configuration:")
-CONF.log_opt_values(LOG, log.DEBUG)
-
-application = app.VersionSelectorApplication()
+application = wsgi.initialize_wsgi_app(show_deprecated=True)

watcher/api/wsgi.py

@@ -0,0 +1,41 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""WSGI script for Watcher API, installed by pbr."""
+
+import sys
+
+from oslo_config import cfg
+import oslo_i18n as i18n
+from oslo_log import log
+
+from watcher.api import app
+from watcher.common import service
+
+
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
+
+
+def initialize_wsgi_app(show_deprecated=False):
+    i18n.install('watcher')
+
+    service.prepare_service(sys.argv)
+
+    LOG.debug("Configuration:")
+    CONF.log_opt_values(LOG, log.DEBUG)
+
+    if show_deprecated:
+        LOG.warning("Using watcher/api/app.wsgi is deprecated and it will "
+                    "be removed in U release. Please use automatically "
+                    "generated watcher-api-wsgi instead.")
+
+    return app.VersionSelectorApplication()