Browse Source

Change process name of neutron-server to match worker role

We spawn a lot of neutron-servers, on all but the smallest systems.
It's often hard to tell which are busy/overloaded or spinning.
Add an option to set the process names to their role.

This has a small chance of breaking existing scripting, depending
how they're parsing ps output.

Sample output:
$ ps xw | grep neutron-server
 1126 pts/2    S+     0:00 grep --color=auto neutron-server
25355 ?        Ss     0:26 /usr/bin/python /usr/local/bin/neutron-server \
  --config-file /etc/neutron/neutron.conf \
  --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
25368 ?        S      0:00 neutron-server: api worker
25369 ?        S      0:00 neutron-server: api worker
25370 ?        S      0:00 neutron-server: api worker
25371 ?        S      0:00 neutron-server: api worker
25372 ?        S      0:02 neutron-server: rpc worker
25373 ?        S      0:02 neutron-server: rpc worker
25374 ?        S      0:02 neutron-server: services worker

The "normal" looking ps output is the main parent.

Partial-Bug: #1816485
Depends-On: https://review.openstack.org/637119
Change-Id: I0e664a5f8e792d85b8f5483fb8c6f1cd59a677cd
changes/19/637019/16
Doug Wiegley 2 years ago
committed by Brian Haley
parent
commit
61b231a999
  1. 7
      neutron/conf/common.py
  2. 12
      neutron/service.py
  3. 3
      neutron/tests/unit/test_service.py
  4. 4
      neutron/tests/unit/test_wsgi.py
  5. 18
      neutron/worker.py
  6. 23
      neutron/wsgi.py
  7. 20
      releasenotes/notes/setproctitle_workers-bc27a8baa5ef2279.yaml

7
neutron/conf/common.py

@ -110,6 +110,13 @@ core_opts = [
cfg.IntOpt('send_events_interval', default=2,
help=_('Number of seconds between sending events to nova if '
'there are any events to send.')),
cfg.StrOpt('setproctitle', default='on',
help=_("Set process name to match child worker role. "
"Available options are: 'off' - retains the previous "
"behavior; 'on' - renames processes to "
"'neutron-server: role (original string)'; "
"'brief' - renames the same as 'on', but without the "
"original string, such as 'neutron-server: role'.")),
cfg.StrOpt('ipam_driver', default='internal',
help=_("Neutron IPAM (IP address management) driver to use. "
"By default, the reference implementation of the "

12
neutron/service.py

@ -24,7 +24,6 @@ from neutron_lib import context
from neutron_lib.db import api as session
from neutron_lib.plugins import directory
from neutron_lib import rpc as n_rpc
from neutron_lib import worker as neutron_worker
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log as logging
@ -38,6 +37,7 @@ import psutil
from neutron.common import config
from neutron.common import profiler
from neutron.conf import service
from neutron import worker as neutron_worker
from neutron import wsgi
@ -94,7 +94,7 @@ def serve_wsgi(cls):
return service
class RpcWorker(neutron_worker.BaseWorker):
class RpcWorker(neutron_worker.NeutronBaseWorker):
"""Wraps a worker to be handled by ProcessLauncher"""
start_listeners_method = 'start_rpc_listeners'
@ -107,7 +107,7 @@ class RpcWorker(neutron_worker.BaseWorker):
self._servers = []
def start(self):
super(RpcWorker, self).start()
super(RpcWorker, self).start(desc="rpc worker")
for plugin in self._plugins:
if hasattr(plugin, self.start_listeners_method):
try:
@ -220,7 +220,7 @@ def _get_plugins_workers():
]
class AllServicesNeutronWorker(neutron_worker.BaseWorker):
class AllServicesNeutronWorker(neutron_worker.NeutronBaseWorker):
def __init__(self, services, worker_process_count=1):
super(AllServicesNeutronWorker, self).__init__(worker_process_count)
self._services = services
@ -230,7 +230,7 @@ class AllServicesNeutronWorker(neutron_worker.BaseWorker):
def start(self):
for srv in self._services:
self._launcher.launch_service(srv)
super(AllServicesNeutronWorker, self).start()
super(AllServicesNeutronWorker, self).start(desc="services worker")
def stop(self):
self._launcher.stop()
@ -322,7 +322,7 @@ def _run_wsgi(app_name):
def run_wsgi_app(app):
server = wsgi.Server("Neutron")
server.start(app, cfg.CONF.bind_port, cfg.CONF.bind_host,
workers=_get_api_workers())
workers=_get_api_workers(), desc="api worker")
LOG.info("Neutron service started, listening on %(host)s:%(port)s",
{'host': cfg.CONF.bind_host, 'port': cfg.CONF.bind_port})
return server

3
neutron/tests/unit/test_service.py

@ -80,7 +80,8 @@ class TestRunWsgiApp(base.BaseTestCase):
service.run_wsgi_app(mock.sentinel.app)
start_call = mock_server.return_value.start.call_args
expected_call = mock.call(
mock.ANY, mock.ANY, mock.ANY, workers=expected_passed_value)
mock.ANY, mock.ANY, mock.ANY, desc='api worker',
workers=expected_passed_value)
self.assertEqual(expected_call, start_call)
def test_api_workers_zero(self):

4
neutron/tests/unit/test_wsgi.py

@ -74,7 +74,7 @@ class TestWorkerService(TestServiceBase):
_service.pool.spawn.return_value = None
_app = mock.Mock()
workerservice = wsgi.WorkerService(_service, _app)
workerservice = wsgi.WorkerService(_service, _app, "on")
workerservice.start()
self.assertFalse(apimock.called)
@ -82,7 +82,7 @@ class TestWorkerService(TestServiceBase):
_service = mock.Mock()
_app = mock.Mock()
worker_service = wsgi.WorkerService(_service, _app)
worker_service = wsgi.WorkerService(_service, _app, "on")
self._test_reset(worker_service)

18
neutron/worker.py

@ -11,10 +11,24 @@
# under the License.
from neutron_lib import worker
from oslo_config import cfg
from oslo_service import loopingcall
class PeriodicWorker(worker.BaseWorker):
class NeutronBaseWorker(worker.BaseWorker):
def __init__(self, worker_process_count=1, set_proctitle=None):
set_proctitle = set_proctitle or cfg.CONF.setproctitle
super(NeutronBaseWorker, self).__init__(
worker_process_count=worker_process_count,
set_proctitle=set_proctitle
)
def start(self, name="neutron-server", desc=None):
super(NeutronBaseWorker, self).start(name=name, desc=desc)
class PeriodicWorker(NeutronBaseWorker):
"""A worker that runs a function at a fixed interval."""
def __init__(self, check_func, interval, initial_delay):
@ -26,7 +40,7 @@ class PeriodicWorker(worker.BaseWorker):
self._initial_delay = initial_delay
def start(self):
super(PeriodicWorker, self).start()
super(PeriodicWorker, self).start(desc="periodic worker")
if self._loop is None:
self._loop = loopingcall.FixedIntervalLoopingCall(self._check_func)
self._loop.start(interval=self._interval,

23
neutron/wsgi.py

@ -25,7 +25,6 @@ import eventlet.wsgi
from neutron_lib import context
from neutron_lib.db import api as db_api
from neutron_lib import exceptions as exception
from neutron_lib import worker as neutron_worker
from oslo_config import cfg
import oslo_i18n
from oslo_log import log as logging
@ -43,6 +42,7 @@ import webob.exc
from neutron._i18n import _
from neutron.common import config
from neutron.conf import wsgi as wsgi_config
from neutron import worker as neutron_worker
CONF = cfg.CONF
wsgi_config.register_socket_opts()
@ -58,19 +58,20 @@ def encode_body(body):
return encodeutils.to_utf8(body)
class WorkerService(neutron_worker.BaseWorker):
class WorkerService(neutron_worker.NeutronBaseWorker):
"""Wraps a worker to be handled by ProcessLauncher"""
def __init__(self, service, application, disable_ssl=False,
def __init__(self, service, application, set_proctitle, disable_ssl=False,
worker_process_count=0):
super(WorkerService, self).__init__(worker_process_count)
super(WorkerService, self).__init__(worker_process_count,
set_proctitle)
self._service = service
self._application = application
self._disable_ssl = disable_ssl
self._server = None
def start(self):
super(WorkerService, self).start()
def start(self, desc=None):
super(WorkerService, self).start(desc=desc)
# When api worker is stopped it kills the eventlet wsgi server which
# internally closes the wsgi server socket object. This server socket
# object becomes not usable which leads to "Bad file descriptor"
@ -162,7 +163,7 @@ class Server(object):
return sock
def start(self, application, port, host='0.0.0.0', workers=0):
def start(self, application, port, host='0.0.0.0', workers=0, desc=None):
"""Run a WSGI server with the given application."""
self._host = host
self._port = port
@ -174,14 +175,16 @@ class Server(object):
self._launch(application, workers)
def _launch(self, application, workers=0):
service = WorkerService(self, application, self.disable_ssl, workers)
def _launch(self, application, workers=0, desc=None):
set_proctitle = "off" if desc is None else CONF.setproctitle
service = WorkerService(self, application, set_proctitle,
self.disable_ssl, workers)
if workers < 1:
# The API service should run in the current process.
self._server = service
# Dump the initial option values
cfg.CONF.log_opt_values(LOG, logging.DEBUG)
service.start()
service.start(desc=desc)
systemd.notify_once()
else:
# dispose the whole pool before os.fork, otherwise there will

20
releasenotes/notes/setproctitle_workers-bc27a8baa5ef2279.yaml

@ -0,0 +1,20 @@
features:
- Neutron child processes now set their process titles
to match their roles ('api worker', 'rpc worker',
'periodic worker', 'services worker', or any other defined
by workers from out-of-tree plugins.) This behavior can be
disabled by setting the ``setproctitle`` config option in the
``[default]`` section in neutron.conf to ``off``. The original
process string is also appended to the end, to help with
scripting that is looking for the old strings. There is also an
option called ``brief``, which results in much shorter and easier
to read process names. The default setting for this
option is ``on``, for a combination of backwards compatibility
and identifying different processes easily. The recommended
setting is ``brief``, once the deployer has verified that none
of their tooling depends on the older strings.
upgrade:
- The change to the process title happens by default with the new
``setproctitle`` config option. The old string is still part of
the new process title, but any scripts looking for exact string
matches of the old string may need to be modified.
Loading…
Cancel
Save