Browse Source

Merge "Fix controller worker graceful shutdown"

changes/36/660236/23
Zuul 2 weeks ago
parent
commit
2a60c13863
9 changed files with 30 additions and 33 deletions
  1. +5
    -0
      devstack/plugin.sh
  2. +1
    -0
      doc/source/configuration/configref.rst
  3. +3
    -0
      etc/octavia.conf
  4. +5
    -5
      octavia/controller/queue/v1/consumer.py
  5. +5
    -5
      octavia/controller/queue/v2/consumer.py
  6. +0
    -11
      octavia/tests/unit/controller/queue/v1/test_consumer.py
  7. +0
    -11
      octavia/tests/unit/controller/queue/v2/test_consumer.py
  8. +9
    -0
      releasenotes/notes/fix-worker-graceful-shutdown-c44b6797637aa1b3.yaml
  9. +2
    -1
      tox.ini

+ 5
- 0
devstack/plugin.sh View File

@@ -385,6 +385,11 @@ function octavia_configure {
iniset $OCTAVIA_CONF api_settings bind_port ${OCTAVIA_HA_PORT}
iniset $OCTAVIA_CONF api_settings bind_host 0.0.0.0
fi

# set default graceful_shutdown_timeout to 300 sec (5 minutes)
# TODO(gthiemonge) update this value after persistant taskflow commits are
# merged
iniset $OCTAVIA_CONF DEFAULT graceful_shutdown_timeout 300
}

function create_mgmt_network_interface {

+ 1
- 0
doc/source/configuration/configref.rst View File

@@ -26,3 +26,4 @@ Octavia Configuration Options
oslo.db
oslo.log
oslo.messaging
cotyledon

+ 3
- 0
etc/octavia.conf View File

@@ -16,6 +16,9 @@
# transport_url = rabbit://<user>:<pass>@server01,<user>:<pass>@server02/<vhost>
# transport_url =

# How long in seconds to wait for octavia worker to exit before killing them.
# graceful_shutdown_timeout = 60

[api_settings]
# bind_host = 127.0.0.1
# bind_port = 9876

+ 5
- 5
octavia/controller/queue/v1/consumer.py View File

@@ -46,14 +46,14 @@ class ConsumerService(cotyledon.Service):
)
self.message_listener.start()

def terminate(self, graceful=False):
def terminate(self):
if self.message_listener:
LOG.info('Stopping consumer...')
self.message_listener.stop()
if graceful:
LOG.info('Consumer successfully stopped. Waiting for final '
'messages to be processed...')
self.message_listener.wait()
LOG.info('Consumer successfully stopped. Waiting for final '
'messages to be processed...')
self.message_listener.wait()
if self.endpoints:
LOG.info('Shutting down endpoint worker executors...')
for e in self.endpoints:

+ 5
- 5
octavia/controller/queue/v2/consumer.py View File

@@ -47,14 +47,14 @@ class ConsumerService(cotyledon.Service):
)
self.message_listener.start()

def terminate(self, graceful=False):
def terminate(self):
if self.message_listener:
LOG.info('Stopping V2 consumer...')
self.message_listener.stop()
if graceful:
LOG.info('V2 Consumer successfully stopped. Waiting for '
'final messages to be processed...')
self.message_listener.wait()
LOG.info('V2 Consumer successfully stopped. Waiting for '
'final messages to be processed...')
self.message_listener.wait()
if self.endpoints:
LOG.info('Shutting down V2 endpoint worker executors...')
for e in self.endpoints:

+ 0
- 11
octavia/tests/unit/controller/queue/v1/test_consumer.py View File

@@ -58,15 +58,4 @@ class TestConsumer(base.TestRpc):
cons.run()
cons.terminate()
mock_rpc_server_rv.stop.assert_called_once_with()
self.assertFalse(mock_rpc_server_rv.wait.called)

@mock.patch.object(messaging, 'get_rpc_server')
def test_consumer_graceful_terminate(self, mock_rpc_server):
mock_rpc_server_rv = mock.Mock()
mock_rpc_server.return_value = mock_rpc_server_rv

cons = consumer.ConsumerService(1, self.conf)
cons.run()
cons.terminate(graceful=True)
mock_rpc_server_rv.stop.assert_called_once_with()
mock_rpc_server_rv.wait.assert_called_once_with()

+ 0
- 11
octavia/tests/unit/controller/queue/v2/test_consumer.py View File

@@ -58,15 +58,4 @@ class TestConsumer(base.TestRpc):
cons.run()
cons.terminate()
mock_rpc_server_rv.stop.assert_called_once_with()
self.assertFalse(mock_rpc_server_rv.wait.called)

@mock.patch.object(messaging, 'get_rpc_server')
def test_consumer_graceful_terminate(self, mock_rpc_server):
mock_rpc_server_rv = mock.Mock()
mock_rpc_server.return_value = mock_rpc_server_rv

cons = consumer.ConsumerService(1, self.conf)
cons.run()
cons.terminate(graceful=True)
mock_rpc_server_rv.stop.assert_called_once_with()
mock_rpc_server_rv.wait.assert_called_once_with()

+ 9
- 0
releasenotes/notes/fix-worker-graceful-shutdown-c44b6797637aa1b3.yaml View File

@@ -0,0 +1,9 @@
---
fixes:
- |
Fix a bug that could interrupt resource creation when performing a graceful
shutdown of the controller worker and leave resources in a
PENDING_CREATE/PENDING_UPDATE/PENDING_DELETE provisioning status. If the
duration of an Octavia flow is greater than the 'graceful_shutdown_timeout'
configuration value, stopping the Octavia worker can still interrupt the
creation of resources.

+ 2
- 1
tox.ini View File

@@ -129,7 +129,8 @@ commands =
--namespace oslo.db \
--namespace oslo.log \
--namespace oslo.messaging \
--namespace keystonemiddleware.auth_token
--namespace keystonemiddleware.auth_token \
--namespace cotyledon

[testenv:genpolicy]
basepython = python3

Loading…
Cancel
Save