openstack
/
octavia
Code Issues Proposed changes

Force PCI bus rescan if interface is not found 

On rare occasions the linux kernel is not seeing a hot plugged network
interface after nova attaches it to an instance.
This patch adds a PCI bus rescan if we can't find a network interface
we expect to see during a vip/network plug workflow.

Story: 1706836
Task: 5214

Change-Id: I3b15d52233a8111e9667a4d4a39e977098cf92e0
This commit is contained in:
Michael Johnson 2017-09-27 14:23:04 -07:00
parent c1a3c630af
commit 75c2d99983
2 changed files with 115 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
octavia/amphorae/backends/agent/api_server/plug.py
View File

@ -219,6 +219,14 @@ class Plug(object):
interface)[netifaces.AF_LINK]:
if link.get('addr', '').lower() == mac.lower():
return interface
# Poke the kernel to re-enumerate the PCI bus.
# We have had cases where nova hot plugs the interface but
# the kernel doesn't get the memo.
filename = '/sys/bus/pci/rescan'
flags = os.O_WRONLY
if os.path.isfile(filename):
with os.fdopen(os.open(filename, flags), 'w') as rescan_file:
rescan_file.write('1')
raise exceptions.HTTPException(
response=webob.Response(json=dict(
details="No suitable network interface found"), status=404))

167
octavia/tests/functional/amphorae/backend/agent/api_server/test_server.py
View File

@ -1034,33 +1034,48 @@ class TestServerTestCase(base.TestCase):
# No interface at all
mock_interfaces.side_effect = [[]]
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/network",
content_type='application/json',
data=json.dumps(port_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/network",
content_type='application/json',
data=json.dumps(port_info))
file_name = '/sys/bus/pci/rescan'
m = self.useFixture(test_utils.OpenFixture(file_name)).mock_open
with mock.patch('os.open') as mock_open, mock.patch.object(
os, 'fdopen', m) as mock_fdopen:
mock_open.return_value = 123
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/network",
content_type='application/json',
data=json.dumps(port_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/network",
content_type='application/json',
data=json.dumps(port_info))
mock_open.assert_called_with(file_name, os.O_WRONLY)
mock_fdopen.assert_called_with(123, 'w')
m().write.assert_called_once_with('1')
self.assertEqual(404, rv.status_code)
self.assertEqual(dict(details="No suitable network interface found"),
json.loads(rv.data.decode('utf-8')))
# No interface down
m().reset_mock()
mock_interfaces.side_effect = [['blah']]
mock_ifaddress.side_effect = [[netifaces.AF_INET]]
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/network",
content_type='application/json',
data=json.dumps(port_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/network",
content_type='application/json',
data=json.dumps(port_info))
with mock.patch('os.open') as mock_open, mock.patch.object(
os, 'fdopen', m) as mock_fdopen:
mock_open.return_value = 123
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/network",
content_type='application/json',
data=json.dumps(port_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/network",
content_type='application/json',
data=json.dumps(port_info))
mock_open.assert_called_with(file_name, os.O_WRONLY)
mock_fdopen.assert_called_with(123, 'w')
m().write.assert_called_once_with('1')
self.assertEqual(404, rv.status_code)
self.assertEqual(dict(details="No suitable network interface found"),
json.loads(rv.data.decode('utf-8')))
@ -1548,33 +1563,50 @@ class TestServerTestCase(base.TestCase):
# No interface at all
mock_interfaces.side_effect = [[]]
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/vip/203.0.113.2",
content_type='application/json',
data=json.dumps(subnet_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/vip/203.0.113.2",
content_type='application/json',
data=json.dumps(subnet_info))
file_name = '/sys/bus/pci/rescan'
m = self.useFixture(test_utils.OpenFixture(file_name)).mock_open
with mock.patch('os.open') as mock_open, mock.patch.object(
os, 'fdopen', m) as mock_fdopen:
mock_open.return_value = 123
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/vip/203.0.113.2",
content_type='application/json',
data=json.dumps(subnet_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/vip/203.0.113.2",
content_type='application/json',
data=json.dumps(subnet_info))
mock_open.assert_called_with(file_name, os.O_WRONLY)
mock_fdopen.assert_called_with(123, 'w')
m().write.assert_called_once_with('1')
self.assertEqual(404, rv.status_code)
self.assertEqual(dict(details="No suitable network interface found"),
json.loads(rv.data.decode('utf-8')))
# Two interfaces down
m().reset_mock()
mock_interfaces.side_effect = [['blah', 'blah2']]
mock_ifaddress.side_effect = [['blabla'], ['blabla']]
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/vip/203.0.113.2",
content_type='application/json',
data=json.dumps(subnet_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/vip/203.0.113.2",
content_type='application/json',
data=json.dumps(subnet_info))
with mock.patch('os.open') as mock_open, mock.patch.object(
os, 'fdopen', m) as mock_fdopen:
mock_open.return_value = 123
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/vip/203.0.113.2",
content_type='application/json',
data=json.dumps(subnet_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/vip/203.0.113.2",
content_type='application/json',
data=json.dumps(subnet_info))
mock_open.assert_called_with(file_name, os.O_WRONLY)
mock_fdopen.assert_called_with(123, 'w')
m().write.assert_called_once_with('1')
self.assertEqual(404, rv.status_code)
self.assertEqual(dict(details="No suitable network interface found"),
json.loads(rv.data.decode('utf-8')))
@ -1866,33 +1898,48 @@ class TestServerTestCase(base.TestCase):
# No interface at all
mock_interfaces.side_effect = [[]]
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/vip/2001:db8::2",
content_type='application/json',
data=json.dumps(subnet_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/vip/2001:db8::2",
content_type='application/json',
data=json.dumps(subnet_info))
file_name = '/sys/bus/pci/rescan'
m = self.useFixture(test_utils.OpenFixture(file_name)).mock_open
with mock.patch('os.open') as mock_open, mock.patch.object(
os, 'fdopen', m) as mock_fdopen:
mock_open.return_value = 123
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/vip/2001:db8::2",
content_type='application/json',
data=json.dumps(subnet_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/vip/2001:db8::2",
content_type='application/json',
data=json.dumps(subnet_info))
mock_open.assert_called_with(file_name, os.O_WRONLY)
mock_fdopen.assert_called_with(123, 'w')
m().write.assert_called_once_with('1')
self.assertEqual(404, rv.status_code)
self.assertEqual(dict(details="No suitable network interface found"),
json.loads(rv.data.decode('utf-8')))
# Two interfaces down
m().reset_mock()
mock_interfaces.side_effect = [['blah', 'blah2']]
mock_ifaddress.side_effect = [['blabla'], ['blabla']]
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/vip/2001:db8::2",
content_type='application/json',
data=json.dumps(subnet_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/vip/2001:db8::2",
content_type='application/json',
data=json.dumps(subnet_info))
with mock.patch('os.open') as mock_open, mock.patch.object(
os, 'fdopen', m) as mock_fdopen:
mock_open.return_value = 123
if distro == consts.UBUNTU:
rv = self.ubuntu_app.post('/' + api_server.VERSION +
"/plug/vip/2001:db8::2",
content_type='application/json',
data=json.dumps(subnet_info))
elif distro == consts.CENTOS:
rv = self.centos_app.post('/' + api_server.VERSION +
"/plug/vip/2001:db8::2",
content_type='application/json',
data=json.dumps(subnet_info))
mock_open.assert_called_with(file_name, os.O_WRONLY)
mock_fdopen.assert_called_with(123, 'w')
m().write.assert_called_once_with('1')
self.assertEqual(404, rv.status_code)
self.assertEqual(dict(details="No suitable network interface found"),
json.loads(rv.data.decode('utf-8')))