openstack
/
octavia
Code Issues Proposed changes

Provide better ComputeBuildExcpetion messages 

Octavia was not exposing the underlying compute driver exception
information to the operator in the octavia logs.  This meant debugging
required examining the compute service logs.

This patch will pass through the fault message provided by the compute
driver in the exceptions raised and clarify in the logs when the
nova driver caused the failure.

Change-Id: If54656c635aac907d78b387a9b191320385d9852
Closes-Bug: #1658900
This commit is contained in:
Michael Johnson 2017-06-21 21:49:09 -07:00
parent dcf4885a16
commit 32819ecc8d
7 changed files with 24 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
octavia/common/exceptions.py
View File

@ -166,7 +166,7 @@ class TooManyL7RulesOnL7Policy(APIException):
class ComputeBuildException(OctaviaException):
message = _('Failed to build compute instance.')
message = _("Failed to build compute instance due to: %(fault)s")
class ComputeBuildQueueTimeoutException(OctaviaException):

1
octavia/compute/compute_base.py
View File

@ -73,6 +73,7 @@ class ComputeBase(object):
:param compute_id: the id of the desired amphora
:returns: the amphora object
:returns: fault message or None
"""
pass

2
octavia/compute/drivers/noop_driver/driver.py
View File

@ -68,7 +68,7 @@ class NoopManager(object):
compute_id=compute_id,
status=constants.ACTIVE,
lb_network_ip='192.0.2.1'
)
), None
def create_server_group(self, name, policy):
LOG.debug("Create Server Group %s no-op, name %s, policy %s ",

14
octavia/compute/drivers/nova_driver.py
View File

@ -152,9 +152,9 @@ class VirtualMachineManager(compute_base.ComputeBase):
)
return amphora.id
except Exception:
LOG.exception("Error building nova virtual machine.")
raise exceptions.ComputeBuildException()
except Exception as e:
LOG.exception("Nova failed to build the instance due to: %s", e)
raise exceptions.ComputeBuildException(fault=e)
def delete(self, compute_id):
'''Delete a virtual machine.
@ -177,7 +177,7 @@ class VirtualMachineManager(compute_base.ComputeBase):
:returns: constant of amphora status
'''
try:
amphora = self.get_amphora(compute_id)
amphora, fault = self.get_amphora(compute_id)
if amphora and amphora.status == 'ACTIVE':
return constants.UP
except Exception:
@ -190,6 +190,7 @@ class VirtualMachineManager(compute_base.ComputeBase):
:param amphora_id: virtual machine UUID
:returns: an amphora object
:returns: fault message or None
'''
# utilize nova client ServerManager 'get' method to retrieve info
try:
@ -204,11 +205,13 @@ class VirtualMachineManager(compute_base.ComputeBase):
:param nova_response: JSON response from nova
:returns: an amphora object
:returns: fault message or None
'''
# Extract interfaces of virtual machine to populate desired amphora
# fields
lb_network_ip = None
fault = None
try:
inf_list = nova_response.interface_list()
@ -223,6 +226,7 @@ class VirtualMachineManager(compute_base.ComputeBase):
if is_boot_network or no_boot_networks:
lb_network_ip = interface.fixed_ips[0]['ip_address']
break
fault = getattr(nova_response, 'fault', None)
except Exception:
LOG.debug('Extracting virtual interfaces through nova '
'os-interfaces extension failed.')
@ -232,7 +236,7 @@ class VirtualMachineManager(compute_base.ComputeBase):
status=nova_response.status,
lb_network_ip=lb_network_ip
)
return response
return response, fault
def create_server_group(self, name, policy):
"""Create a server group object

4
octavia/controller/worker/tasks/compute_tasks.py
View File

@ -183,13 +183,13 @@ class ComputeWait(BaseComputeTask):
:returns: An amphora object
"""
for i in range(CONF.controller_worker.amp_active_retries):
amp = self.compute.get_amphora(compute_id)
amp, fault = self.compute.get_amphora(compute_id)
if amp.status == constants.ACTIVE:
if CONF.haproxy_amphora.build_rate_limit != -1:
self.rate_limit.remove_from_build_req_queue(amphora_id)
return amp
elif amp.status == constants.ERROR:
raise exceptions.ComputeBuildException()
raise exceptions.ComputeBuildException(fault=fault)
time.sleep(CONF.controller_worker.amp_active_wait_sec)
raise exceptions.ComputeWaitTimeoutException()

11
octavia/tests/unit/compute/drivers/test_nova_driver.py
View File

@ -106,6 +106,7 @@ class TestNovaClient(base.TestCase):
self.nova_response = mock.Mock()
self.nova_response.id = self.amphora.compute_id
self.nova_response.status = 'ACTIVE'
self.nova_response.fault = 'FAKE_FAULT'
self.interface_list = mock.MagicMock()
self.interface_list.net_id = '1'
@ -269,8 +270,9 @@ class TestNovaClient(base.TestCase):
self.manager.status, self.amphora.id)
def test_get_amphora(self):
amphora = self.manager.get_amphora(self.amphora.compute_id)
amphora, fault = self.manager.get_amphora(self.amphora.compute_id)
self.assertEqual(self.amphora, amphora)
self.assertEqual(self.nova_response.fault, fault)
self.manager.manager.get.called_with(server=amphora.id)
def test_bad_get_amphora(self):
@ -279,15 +281,16 @@ class TestNovaClient(base.TestCase):
self.manager.get_amphora, self.amphora.id)
def test_translate_amphora(self):
amphora = self.manager._translate_amphora(self.nova_response)
amphora, fault = self.manager._translate_amphora(self.nova_response)
self.assertEqual(self.amphora, amphora)
self.assertEqual(self.nova_response.fault, fault)
self.nova_response.interface_list.called_with()
def test_bad_translate_amphora(self):
self.nova_response.interface_list.side_effect = Exception
self.manager._nova_client.networks.get.side_effect = Exception
self.assertIsNone(
self.manager._translate_amphora(self.nova_response).lb_network_ip)
amphora, fault = self.manager._translate_amphora(self.nova_response)
self.assertIsNone(amphora.lb_network_ip)
self.nova_response.interface_list.called_with()
def test_create_server_group(self):

6
octavia/tests/unit/controller/worker/tasks/test_compute_tasks.py
View File

@ -337,7 +337,7 @@ class TestComputeTasks(base.TestCase):
_amphora_mock.status = constants.ACTIVE
_amphora_mock.lb_network_ip = LB_NET_IP
mock_driver.get_amphora.return_value = _amphora_mock
mock_driver.get_amphora.return_value = _amphora_mock, None
computewait = compute_tasks.ComputeWait()
computewait.execute(COMPUTE_ID, AMPHORA_ID)
@ -364,7 +364,7 @@ class TestComputeTasks(base.TestCase):
_amphora_mock.status = constants.ACTIVE
_amphora_mock.lb_network_ip = LB_NET_IP
mock_driver.get_amphora.return_value = _amphora_mock
mock_driver.get_amphora.return_value = _amphora_mock, None
computewait = compute_tasks.ComputeWait()
computewait.execute(COMPUTE_ID, AMPHORA_ID)
@ -389,7 +389,7 @@ class TestComputeTasks(base.TestCase):
_amphora_mock.status = constants.ACTIVE
_amphora_mock.lb_network_ip = LB_NET_IP
mock_driver.get_amphora.return_value = _amphora_mock
mock_driver.get_amphora.return_value = _amphora_mock, None
computewait = compute_tasks.ComputeWait()
computewait.execute(COMPUTE_ID, AMPHORA_ID)