From 5c5328ccaabd0e6dfe8c4658b4ff738adcaba768 Mon Sep 17 00:00:00 2001 From: Dmitry Tantsur Date: Fri, 12 Apr 2019 10:57:03 +0200 Subject: [PATCH] Supports fetching API endpoints from mDNS This change enables IPA to receive API endpoints and configuration via multicast DNS. Story: #2005393 Task: #30382 Change-Id: Ibbf07052bea8f5c0305dda098b2879bcbc2fece5 --- ironic_python_agent/agent.py | 18 +++ ironic_python_agent/config.py | 44 +++++- ironic_python_agent/inspector.py | 13 +- ironic_python_agent/tests/unit/test_agent.py | 131 +++++++++++++++++- .../tests/unit/test_inspector.py | 28 +++- lower-constraints.txt | 2 +- releasenotes/notes/mdns-e020484e64d76edb.yaml | 12 ++ requirements.txt | 2 +- 8 files changed, 236 insertions(+), 14 deletions(-) create mode 100644 releasenotes/notes/mdns-e020484e64d76edb.yaml diff --git a/ironic_python_agent/agent.py b/ironic_python_agent/agent.py index 833d8fe14..390afce8e 100644 --- a/ironic_python_agent/agent.py +++ b/ironic_python_agent/agent.py @@ -21,6 +21,8 @@ import threading import time from wsgiref import simple_server +from ironic_lib import exception as lib_exc +from ironic_lib import mdns import netaddr from oslo_concurrency import processutils from oslo_config import cfg @@ -31,6 +33,7 @@ from six.moves.urllib import parse as urlparse from stevedore import extension from ironic_python_agent.api import app +from ironic_python_agent import config from ironic_python_agent import encoding from ironic_python_agent import errors from ironic_python_agent.extensions import base @@ -176,6 +179,21 @@ class IronicPythonAgent(base.ExecuteCommandMixin): invoke_kwds={'agent': self}, ) self.api_url = api_url + if not self.api_url or self.api_url == 'mdns': + try: + self.api_url, params = mdns.get_endpoint('baremetal') + except lib_exc.ServiceLookupFailure: + if self.api_url: + # mDNS explicitly requested, report failure. + raise + else: + # implicit fallback to mDNS, do not fail (maybe we're only + # running inspection). + LOG.warning('Could not get baremetal endpoint from mDNS, ' + 'will not heartbeat') + else: + config.override(params) + if self.api_url: self.api_client = ironic_api_client.APIClient(self.api_url) self.heartbeater = IronicPythonAgentHeartbeater(self) diff --git a/ironic_python_agent/config.py b/ironic_python_agent/config.py index 02c4cbe7d..5bd1cdc2d 100644 --- a/ironic_python_agent/config.py +++ b/ironic_python_agent/config.py @@ -13,23 +13,29 @@ # limitations under the License. from oslo_config import cfg +from oslo_log import log as logging -from ironic_python_agent import inspector from ironic_python_agent import netutils from ironic_python_agent import utils +LOG = logging.getLogger(__name__) CONF = cfg.CONF APARAMS = utils.get_agent_params() +INSPECTION_DEFAULT_COLLECTOR = 'default' +INSPECTION_DEFAULT_DHCP_WAIT_TIMEOUT = 60 + cli_opts = [ cfg.StrOpt('api_url', default=APARAMS.get('ipa-api-url'), deprecated_name='api-url', - regex='^http(s?):\\/\\/.+', + regex='^(mdns|http(s?):\\/\\/.+)', help='URL of the Ironic API. ' 'Can be supplied as "ipa-api-url" kernel parameter.' - 'The value must start with either http:// or https://.'), + 'The value must start with either http:// or https://. ' + 'A special value "mdns" can be specified to fetch the ' + 'URL using multicast DNS service discovery.'), cfg.StrOpt('listen_host', default=APARAMS.get('ipa-listen-host', @@ -133,12 +139,14 @@ cli_opts = [ help='Endpoint of ironic-inspector. If set, hardware inventory ' 'will be collected and sent to ironic-inspector ' 'on start up. ' + 'A special value "mdns" can be specified to fetch the ' + 'URL using multicast DNS service discovery. ' 'Can be supplied as "ipa-inspection-callback-url" ' 'kernel parameter.'), cfg.StrOpt('inspection_collectors', default=APARAMS.get('ipa-inspection-collectors', - inspector.DEFAULT_COLLECTOR), + INSPECTION_DEFAULT_COLLECTOR), help='Comma-separated list of plugins providing additional ' 'hardware data for inspection, empty value gives ' 'a minimum required set of plugins. ' @@ -148,7 +156,7 @@ cli_opts = [ cfg.IntOpt('inspection_dhcp_wait_timeout', min=0, default=APARAMS.get('ipa-inspection-dhcp-wait-timeout', - inspector.DEFAULT_DHCP_WAIT_TIMEOUT), + INSPECTION_DEFAULT_DHCP_WAIT_TIMEOUT), help='Maximum time (in seconds) to wait for the PXE NIC ' '(or all NICs if inspection_dhcp_all_interfaces is True) ' 'to get its IP address via DHCP before inspection. ' @@ -216,3 +224,29 @@ CONF.register_cli_opts(cli_opts) def list_opts(): return [('DEFAULT', cli_opts)] + + +def override(params): + """Override configuration with values from a dictionary. + + This is used for configuration overrides from mDNS. + + :param params: new configuration parameters as a dict. + """ + if not params: + return + + LOG.debug('Overriding configuration with %s', params) + for key, value in params.items(): + if key.startswith('ipa_'): + key = key[4:] + else: + LOG.warning('Skipping unknown configuration option %s', key) + continue + + try: + CONF.set_override(key, value) + except Exception as exc: + LOG.warning('Unable to override configuration option %(key)s ' + 'with %(value)r: %(exc)s', + {'key': key, 'value': value, 'exc': exc}) diff --git a/ironic_python_agent/inspector.py b/ironic_python_agent/inspector.py index 8195cdc6b..54de65d07 100644 --- a/ironic_python_agent/inspector.py +++ b/ironic_python_agent/inspector.py @@ -16,6 +16,7 @@ import os import time +from ironic_lib import mdns from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log as logging @@ -24,6 +25,7 @@ from oslo_utils import excutils import requests import stevedore +from ironic_python_agent import config from ironic_python_agent import encoding from ironic_python_agent import errors from ironic_python_agent import hardware @@ -32,8 +34,6 @@ from ironic_python_agent import utils LOG = logging.getLogger(__name__) CONF = cfg.CONF -DEFAULT_COLLECTOR = 'default' -DEFAULT_DHCP_WAIT_TIMEOUT = 60 _DHCP_RETRY_INTERVAL = 2 _COLLECTOR_NS = 'ironic_python_agent.inspector.collectors' @@ -63,6 +63,15 @@ def inspect(): if not CONF.inspection_callback_url: LOG.info('Inspection is disabled, skipping') return + + if CONF.inspection_callback_url == 'mdns': + LOG.debug('Fetching the inspection URL from mDNS') + url, params = mdns.get_endpoint('baremetal-introspection') + # We expect a proper catalog URL, which doesn't include any path. + CONF.set_override('inspection_callback_url', + url.rstrip('/') + '/v1/continue') + config.override(params) + collector_names = [x.strip() for x in CONF.inspection_collectors.split(',') if x.strip()] LOG.info('inspection is enabled with collectors %s', collector_names) diff --git a/ironic_python_agent/tests/unit/test_agent.py b/ironic_python_agent/tests/unit/test_agent.py index ba529bae4..8fbffe0ec 100644 --- a/ironic_python_agent/tests/unit/test_agent.py +++ b/ironic_python_agent/tests/unit/test_agent.py @@ -16,6 +16,7 @@ import socket import time from wsgiref import simple_server +from ironic_lib import exception as lib_exc import mock from oslo_concurrency import processutils from oslo_config import cfg @@ -217,6 +218,126 @@ class TestBaseAgent(ironic_agent_base.IronicAgentTest): mock_dispatch.call_args_list) self.agent.heartbeater.start.assert_called_once_with() + @mock.patch('ironic_lib.mdns.get_endpoint', autospec=True) + @mock.patch.object(hardware, '_check_for_iscsi', mock.Mock()) + @mock.patch( + 'ironic_python_agent.hardware_managers.cna._detect_cna_card', + mock.Mock()) + @mock.patch.object(hardware, 'dispatch_to_managers', autospec=True) + @mock.patch.object(agent.IronicPythonAgent, + '_wait_for_interface', autospec=True) + @mock.patch('wsgiref.simple_server.WSGIServer', autospec=True) + @mock.patch.object(hardware, 'load_managers', autospec=True) + def test_url_from_mdns_by_default(self, mock_load_managers, mock_wsgi, + mock_wait, mock_dispatch, mock_mdns): + CONF.set_override('inspection_callback_url', '') + mock_mdns.return_value = 'https://example.com', {} + + wsgi_server = mock_wsgi.return_value + + self.agent = agent.IronicPythonAgent(None, + agent.Host('203.0.113.1', 9990), + agent.Host('192.0.2.1', 9999), + 3, + 10, + 'eth0', + 300, + 1, + False) + + def set_serve_api(): + self.agent.serve_api = False + + wsgi_server.handle_request.side_effect = set_serve_api + self.agent.heartbeater = mock.Mock() + self.agent.api_client.lookup_node = mock.Mock() + self.agent.api_client.lookup_node.return_value = { + 'node': { + 'uuid': 'deadbeef-dabb-ad00-b105-f00d00bab10c' + }, + 'config': { + 'heartbeat_timeout': 300 + } + } + + self.agent.run() + + listen_addr = agent.Host('192.0.2.1', 9999) + mock_wsgi.assert_called_once_with( + (listen_addr.hostname, + listen_addr.port), + simple_server.WSGIRequestHandler) + wsgi_server.set_app.assert_called_once_with(self.agent.api) + self.assertTrue(wsgi_server.handle_request.called) + mock_wait.assert_called_once_with(mock.ANY) + self.assertEqual([mock.call('list_hardware_info'), + mock.call('wait_for_disks')], + mock_dispatch.call_args_list) + self.agent.heartbeater.start.assert_called_once_with() + + @mock.patch('ironic_lib.mdns.get_endpoint', autospec=True) + @mock.patch.object(hardware, '_check_for_iscsi', mock.Mock()) + @mock.patch( + 'ironic_python_agent.hardware_managers.cna._detect_cna_card', + mock.Mock()) + @mock.patch.object(hardware, 'dispatch_to_managers', autospec=True) + @mock.patch.object(agent.IronicPythonAgent, + '_wait_for_interface', autospec=True) + @mock.patch('wsgiref.simple_server.WSGIServer', autospec=True) + @mock.patch.object(hardware, 'load_managers', autospec=True) + def test_url_from_mdns_explicitly(self, mock_load_managers, mock_wsgi, + mock_wait, mock_dispatch, mock_mdns): + CONF.set_override('inspection_callback_url', '') + CONF.set_override('disk_wait_attempts', 0) + mock_mdns.return_value = 'https://example.com', { + # configuration via mdns + 'ipa_disk_wait_attempts': '42', + } + + wsgi_server = mock_wsgi.return_value + + self.agent = agent.IronicPythonAgent('mdns', + agent.Host('203.0.113.1', 9990), + agent.Host('192.0.2.1', 9999), + 3, + 10, + 'eth0', + 300, + 1, + False) + + def set_serve_api(): + self.agent.serve_api = False + + wsgi_server.handle_request.side_effect = set_serve_api + self.agent.heartbeater = mock.Mock() + self.agent.api_client.lookup_node = mock.Mock() + self.agent.api_client.lookup_node.return_value = { + 'node': { + 'uuid': 'deadbeef-dabb-ad00-b105-f00d00bab10c' + }, + 'config': { + 'heartbeat_timeout': 300 + } + } + + self.agent.run() + + listen_addr = agent.Host('192.0.2.1', 9999) + mock_wsgi.assert_called_once_with( + (listen_addr.hostname, + listen_addr.port), + simple_server.WSGIRequestHandler) + wsgi_server.set_app.assert_called_once_with(self.agent.api) + self.assertTrue(wsgi_server.handle_request.called) + mock_wait.assert_called_once_with(mock.ANY) + self.assertEqual([mock.call('list_hardware_info'), + mock.call('wait_for_disks')], + mock_dispatch.call_args_list) + self.agent.heartbeater.start.assert_called_once_with() + # changed via mdns + self.assertEqual(42, CONF.disk_wait_attempts) + @mock.patch.object(hardware, '_check_for_iscsi', mock.Mock()) @mock.patch( 'ironic_python_agent.hardware_managers.cna._detect_cna_card', @@ -314,6 +435,7 @@ class TestBaseAgent(ironic_agent_base.IronicAgentTest): mock_dispatch.call_args_list) self.agent.heartbeater.start.assert_called_once_with() + @mock.patch('ironic_lib.mdns.get_endpoint', autospec=True) @mock.patch.object(hardware, '_check_for_iscsi', mock.Mock()) @mock.patch( 'ironic_python_agent.hardware_managers.cna._detect_cna_card', @@ -330,7 +452,9 @@ class TestBaseAgent(ironic_agent_base.IronicAgentTest): mock_wsgi, mock_dispatch, mock_inspector, - mock_wait): + mock_wait, + mock_mdns): + mock_mdns.side_effect = lib_exc.ServiceLookupFailure() # If inspection_callback_url is configured and api_url is not when the # agent starts, ensure that the inspection will be called and wsgi # server will work as usual. Also, make sure api_client and heartbeater @@ -369,6 +493,7 @@ class TestBaseAgent(ironic_agent_base.IronicAgentTest): self.assertFalse(mock_wait.called) self.assertFalse(mock_dispatch.called) + @mock.patch('ironic_lib.mdns.get_endpoint', autospec=True) @mock.patch.object(hardware, '_check_for_iscsi', mock.Mock()) @mock.patch( 'ironic_python_agent.hardware_managers.cna._detect_cna_card', @@ -385,7 +510,9 @@ class TestBaseAgent(ironic_agent_base.IronicAgentTest): mock_wsgi, mock_dispatch, mock_inspector, - mock_wait): + mock_wait, + mock_mdns): + mock_mdns.side_effect = lib_exc.ServiceLookupFailure() # If both api_url and inspection_callback_url are not configured when # the agent starts, ensure that the inspection will be skipped and wsgi # server will work as usual. Also, make sure api_client and heartbeater diff --git a/ironic_python_agent/tests/unit/test_inspector.py b/ironic_python_agent/tests/unit/test_inspector.py index 2944bbc6f..501e8c1e5 100644 --- a/ironic_python_agent/tests/unit/test_inspector.py +++ b/ironic_python_agent/tests/unit/test_inspector.py @@ -24,6 +24,7 @@ from oslo_config import cfg import requests import stevedore +from ironic_python_agent import config from ironic_python_agent import errors from ironic_python_agent import hardware from ironic_python_agent import inspector @@ -48,8 +49,9 @@ class AcceptingFailure(mock.Mock): class TestMisc(base.IronicAgentTest): def test_default_collector_loadable(self): - ext = inspector.extension_manager([inspector.DEFAULT_COLLECTOR]) - self.assertIs(ext[inspector.DEFAULT_COLLECTOR].plugin, + ext = inspector.extension_manager( + [config.INSPECTION_DEFAULT_COLLECTOR]) + self.assertIs(ext[config.INSPECTION_DEFAULT_COLLECTOR].plugin, inspector.collect_default) def test_raise_on_wrong_collector(self): @@ -80,6 +82,26 @@ class TestInspect(base.IronicAgentTest): mock_call.assert_called_with_failure() self.assertEqual('uuid1', result) + @mock.patch('ironic_lib.mdns.get_endpoint', autospec=True) + def test_mdns(self, mock_mdns, mock_ext_mgr, mock_call): + CONF.set_override('inspection_callback_url', 'mdns') + mock_mdns.return_value = 'http://example', { + 'ipa_inspection_collectors': 'one,two' + } + mock_ext_mgr.return_value = [self.mock_ext] + mock_call.return_value = {'uuid': 'uuid1'} + + result = inspector.inspect() + + self.mock_collect.assert_called_with_failure() + mock_call.assert_called_with_failure() + self.assertEqual('uuid1', result) + + self.assertEqual('http://example/v1/continue', + CONF.inspection_callback_url) + self.assertEqual('one,two', CONF.inspection_collectors) + self.assertEqual(['one', 'two'], mock_ext_mgr.call_args[1]['names']) + def test_collectors_option(self, mock_ext_mgr, mock_call): CONF.set_override('inspection_collectors', 'foo,bar') mock_ext_mgr.return_value = [ @@ -362,7 +384,7 @@ class TestWaitForDhcp(base.IronicAgentTest): def setUp(self): super(TestWaitForDhcp, self).setUp() CONF.set_override('inspection_dhcp_wait_timeout', - inspector.DEFAULT_DHCP_WAIT_TIMEOUT) + config.INSPECTION_DEFAULT_DHCP_WAIT_TIMEOUT) @mock.patch.object(time, 'sleep', autospec=True) def test_all(self, mocked_sleep, mocked_dispatch): diff --git a/lower-constraints.txt b/lower-constraints.txt index f87f8b597..0f130f39d 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -24,7 +24,7 @@ greenlet==0.4.13 hacking==1.0.0 idna==2.6 imagesize==1.0.0 -ironic-lib==2.16.0 +ironic-lib==2.17.0 iso8601==0.1.11 Jinja2==2.10 keystoneauth1==3.4.0 diff --git a/releasenotes/notes/mdns-e020484e64d76edb.yaml b/releasenotes/notes/mdns-e020484e64d76edb.yaml new file mode 100644 index 000000000..a2dbb926b --- /dev/null +++ b/releasenotes/notes/mdns-e020484e64d76edb.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Supports fetching baremetal and baremetal introspection endpoints from + mDNS instead of providing them via kernel parameters or a configuration + file. See `story 2005393 + `_ for more details. +upgrade: + - | + When no baremetal API URL is provided (e.g. via the ``ipa-api-url`` kernel + parameter), ironic-python-agent now tries to get the URL using mDNS service + discovery. diff --git a/requirements.txt b/requirements.txt index 904fa91f7..6e1ac7da4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,4 +21,4 @@ rtslib-fb>=2.1.65 # Apache-2.0 six>=1.10.0 # MIT stevedore>=1.20.0 # Apache-2.0 WSME>=0.8.0 # MIT -ironic-lib>=2.16.0 # Apache-2.0 +ironic-lib>=2.17.0 # Apache-2.0