openstack
/
neutron
Code Issues Proposed changes

fullstack: Gracefully stop neutron-server process 

There is possible scenario that neutron-server loses connection to AMQP
bus and that can cause timeouts when hanging on rpc calls. On the other
hand, we should be able to stop service gracefully even if AMQP server
isn't running.

During teardown there were still neutron-server orphaned api workers
processes running that had open connection to database which caused
unexpected failures.

Note that this change applies only for neutron-server because previous
investigations showed agents using oslo service can hang on rpc causing
other failures. Next step should be making sure rabbitmq is stopped as
last or decrease rpc timeouts in agents.

Change-Id: I3d793ba924dc3fd229d8588f4be1e943614f22bb
Related-bug: 1494363
Closes-bug: 1541742
This commit is contained in:
Jakub Libosvar 2016-02-10 16:35:30 +00:00
parent 5b7fd5f0f1
commit a5a7b892df
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
neutron/tests/fullstack/resources/process.py
View File

@ -15,6 +15,7 @@
import datetime import datetime
from distutils import spawn from distutils import spawn
import os import os
import signal
import fixtures import fixtures
from neutronclient.common import exceptions as nc_exc from neutronclient.common import exceptions as nc_exc
@ -34,13 +35,15 @@ DEFAULT_LOG_DIR = '/tmp/dsvm-fullstack-logs/'
class ProcessFixture(fixtures.Fixture): class ProcessFixture(fixtures.Fixture):
def __init__(self, test_name, process_name, exec_name, config_filenames): def __init__(self, test_name, process_name, exec_name, config_filenames,
kill_signal=signal.SIGKILL):
super(ProcessFixture, self).__init__() super(ProcessFixture, self).__init__()
self.test_name = test_name self.test_name = test_name
self.process_name = process_name self.process_name = process_name
self.exec_name = exec_name self.exec_name = exec_name
self.config_filenames = config_filenames self.config_filenames = config_filenames
self.process = None self.process = None
self.kill_signal = kill_signal
def _setUp(self): def _setUp(self):
self.start() self.start()
@ -63,7 +66,7 @@ class ProcessFixture(fixtures.Fixture):
self.process.start(block=True) self.process.start(block=True)
def stop(self): def stop(self):
self.process.stop(block=True) self.process.stop(block=True, kill_signal=self.kill_signal)
class RabbitmqEnvironmentFixture(fixtures.Fixture): class RabbitmqEnvironmentFixture(fixtures.Fixture):
@ -108,7 +111,8 @@ class NeutronServerFixture(fixtures.Fixture):
test_name=self.test_name, test_name=self.test_name,
process_name=self.NEUTRON_SERVER, process_name=self.NEUTRON_SERVER,
exec_name=self.NEUTRON_SERVER, exec_name=self.NEUTRON_SERVER,
config_filenames=config_filenames)) config_filenames=config_filenames,
kill_signal=signal.SIGTERM))
utils.wait_until_true(self.server_is_live) utils.wait_until_true(self.server_is_live)