Add long-running provider agent support

This patch adds support for long-running provider driver agents to
the Octavia driver-agent.
It will fork a process for all of the enabled provider driver
agents at startup.

Change-Id: Ib7042bcc48b1dd5b37b671dd5e64728b71ab9542
Story: 2006250
Task: 35863

.pylintrc

@@ -9,6 +9,7 @@ ignore=.git,tests
 disable=
 # "F" Fatal errors that prevent further processing
 # "I" Informational noise
+ c-extension-no-member,
  locally-disabled,
 # "E" Error for important programming issues (likely bugs)
  import-error,

doc/source/contributor/guides/providers.rst

@@ -33,6 +33,10 @@ Octavia API functions not listed here will continue to be handled by the
 Octavia API and will not call into the driver. Examples would be show, list,
 and quota requests.
 
+In addition, drivers may provide a provider agent that the Octavia driver-agent
+will launch at start up. This is a long-running process that is intended to
+support the provider driver.
+
 Driver Entry Points
 -------------------
 
@@ -48,6 +52,18 @@ for the octavia reference driver would be:
 
   amphora = octavia.api.drivers.amphora_driver.driver:AmphoraProviderDriver
 
+In addition, provider drivers may provide a provider agent also defined by a
+setup tools entry point. The provider agent namespace is
+"octavia.driver_agent.provider_agents". This will be called once, at Octavia
+driver-agent start up, to launch a long-running process. Provider agents must
+be enabled in the Octavia configuration file. An example provider agent
+entry point would be:
+
+.. code-block:: python
+
+  amphora_agent = octavia.api.drivers.amphora_driver.agent:AmphoraProviderAgent
+
+
 Stable Provider Driver Interface
 ================================
 
@@ -1992,6 +2008,80 @@ references to the failed record if available.
         super(DriverAgentTimeout, self).__init__(self.fault_string,
                                                  *args, **kwargs)
 
+Provider Agents
+===============
+
+Provider agents are long-running processes started by the Octavia driver-agent
+process at start up. They are intended to allow provider drivers a long running
+process that can handle periodic jobs for the provider driver or receive events
+from another provider agent. Provider agents are optional and not required for
+a successful Octavia provider driver.
+
+Provider Agents have access to the same `Stable Provider Driver Interface`_
+as the provider driver. A provider agent must not access any other Octavia
+code.
+
+.. warning::
+
+  The methods listed in the `Driver Support Library`_ section are the only
+  Octavia callable methods for provider agents.
+  All other interfaces are not considered stable or safe for provider agents to
+  access. See `Stable Provider Driver Interface`_ for a list of acceptable
+  APIs for provider agents use.
+
+Declaring Your Provider Agent
+-----------------------------
+
+The Octavia driver-agent will use
+`stevedore <https://docs.openstack.org/stevedore/latest/>`_ to load enabled
+provider agents at start up. Provider agents are enabled in the Octavia
+configuration file. Provider agents that are installed, but not enabled, will
+not be loaded. An example configuration file entry for a provider agent is:
+
+.. code-block:: INI
+
+  [driver_agent]
+  enabled_provider_agents = amphora_agent, noop_agent
+
+The provider agent name must match the provider agent name declared in your
+python setup tools entry point. For example:
+
+.. code-block:: python
+
+  octavia.driver_agent.provider_agents =
+      amphora_agent = octavia.api.drivers.amphora_driver.agent:AmphoraProviderAgent
+      noop_agent = octavia.api.drivers.noop_driver.agent:noop_provider_agent
+
+Provider Agent Method Invocation
+--------------------------------
+
+On start up of the Octavia driver-agent, the method defined in the entry point
+will be launched in its own `multiprocessing Process <https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Process>`_.
+
+Your provider agent method will be passed a `multiprocessing Event <https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Event>`_ that will
+be used to signal that the provider agent should shutdown. When this event
+is "set", the provider agent should gracefully shutdown. If the provider agent
+fails to exit within the Octavia configuration file setting
+"provider_agent_shutdown_timeout" period, the driver-agent will forcefully
+shutdown the provider agent with a SIGKILL signal.
+
+Example Provider Agent Method
+-----------------------------
+
+If, for example, you declared a provider agent as "my_agent":
+
+.. code-block:: python
+
+  octavia.driver_agent.provider_agents =
+      my_agent = example_inc.drivers.my_driver.agent:my_provider_agent
+
+The signature of your "my_provider_agent" method would be:
+
+.. code-block:: python
+
+  def my_provider_agent(exit_event):
+
+
 Documenting the Driver
 ======================
 

etc/octavia.conf

@@ -521,3 +521,9 @@
 # Percentage of max_processes (both status and stats) in use to start
 # logging warning messages about an overloaded driver-agent.
 # max_process_warning_percent = .75
+
+# How long in seconds to wait for provider agents to exit before killing them.
+# provider_agent_shutdown_timeout = 60
+
+# List of enabled provider agents.
+# enabled_provider_agents =

octavia/api/drivers/driver_agent/driver_listener.py

@@ -15,7 +15,6 @@
 
 import errno
 import os
-import signal
 import threading
 
 import six.moves.socketserver as socketserver
@@ -103,10 +102,6 @@ class ForkingUDSServer(socketserver.ForkingMixIn,
     pass
 
 
-def _mutate_config(*args, **kwargs):
-    CONF.mutate_config_files()
-
-
 def _cleanup_socket_file(filename):
     # Remove the socket file if it already exists
     try:
@@ -117,9 +112,6 @@ def _cleanup_socket_file(filename):
 
 
 def status_listener(exit_event):
-    signal.signal(signal.SIGINT, signal.SIG_IGN)
-    signal.signal(signal.SIGHUP, _mutate_config)
-
     _cleanup_socket_file(CONF.driver_agent.status_socket_path)
 
     server = ForkingUDSServer(CONF.driver_agent.status_socket_path,
@@ -140,9 +132,6 @@ def status_listener(exit_event):
 
 
 def stats_listener(exit_event):
-    signal.signal(signal.SIGINT, signal.SIG_IGN)
-    signal.signal(signal.SIGHUP, _mutate_config)
-
     _cleanup_socket_file(CONF.driver_agent.stats_socket_path)
 
     server = ForkingUDSServer(CONF.driver_agent.stats_socket_path,
@@ -163,9 +152,6 @@ def stats_listener(exit_event):
 
 
 def get_listener(exit_event):
-    signal.signal(signal.SIGINT, signal.SIG_IGN)
-    signal.signal(signal.SIGHUP, _mutate_config)
-
     _cleanup_socket_file(CONF.driver_agent.get_socket_path)
 
     server = ForkingUDSServer(CONF.driver_agent.get_socket_path,

octavia/api/drivers/noop_driver/agent.py

@@ -0,0 +1,26 @@
+#    Copyright 2019 Red Hat, Inc. All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import time
+
+from oslo_log import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+def noop_provider_agent(exit_event):
+    LOG.info('No-Op provider agent has started.')
+    while not exit_event.is_set():
+        time.sleep(1)
+    LOG.info('No-Op provider agent is exiting.')

octavia/cmd/driver_agent.py

@@ -17,18 +17,21 @@ import multiprocessing
 import os
 import signal
 import sys
+import time
 
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_reports import guru_meditation_report as gmr
+import setproctitle
+from stevedore import enabled as stevedore_enabled
 
 from octavia.api.drivers.driver_agent import driver_listener
 from octavia.common import service
 from octavia import version
 
-
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
+PROVIDER_AGENT_PROCESSES = []
 
 
 def _mutate_config(*args, **kwargs):
@@ -42,6 +45,53 @@ def _handle_mutate_config(status_proc_pid, stats_proc_pid, *args, **kwargs):
     os.kill(stats_proc_pid, signal.SIGHUP)
 
 
+def _check_if_provider_agent_enabled(extension):
+    if extension.name in CONF.driver_agent.enabled_provider_agents:
+        return True
+    return False
+
+
+def _process_wrapper(exit_event, proc_name, function, agent_name=None):
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+    signal.signal(signal.SIGHUP, _mutate_config)
+    if agent_name:
+        process_title = 'octavia-driver-agent - {} -- {}'.format(
+            proc_name, agent_name)
+    else:
+        process_title = 'octavia-driver-agent - {}'.format(proc_name)
+    setproctitle.setproctitle(process_title)
+    while not exit_event.is_set():
+        try:
+            function(exit_event)
+        except Exception as e:
+            if agent_name:
+                LOG.exception('Provider agent "%s" raised exception: %s. '
+                              'Restarting the "%s" provider agent.',
+                              agent_name, str(e), agent_name)
+            else:
+                LOG.exception('%s raised exception: %s. '
+                              'Restarting %s.',
+                              proc_name, str(e), proc_name)
+            time.sleep(1)
+            continue
+        break
+
+
+def _start_provider_agents(exit_event):
+    extensions = stevedore_enabled.EnabledExtensionManager(
+        namespace='octavia.driver_agent.provider_agents',
+        check_func=_check_if_provider_agent_enabled)
+    for ext in extensions:
+        ext_process = multiprocessing.Process(
+            name=ext.name, target=_process_wrapper,
+            args=(exit_event, 'provider_agent', ext.plugin),
+            kwargs={'agent_name': ext.name})
+        PROVIDER_AGENT_PROCESSES.append(ext_process)
+        ext_process.start()
+        LOG.info('Started enabled provider agent: "%s" with PID: %d.',
+                 ext.name, ext_process.pid)
+
+
 def main():
     service.prepare_service(sys.argv)
 
@@ -51,36 +101,61 @@ def main():
     exit_event = multiprocessing.Event()
 
     status_listener_proc = multiprocessing.Process(
-        name='status_listener', target=driver_listener.status_listener,
+        name='status_listener', target=_process_wrapper,
-        args=(exit_event,))
+        args=(exit_event, 'status_listener', driver_listener.status_listener))
     processes.append(status_listener_proc)
 
     LOG.info("Driver agent status listener process starts:")
     status_listener_proc.start()
 
     stats_listener_proc = multiprocessing.Process(
-        name='stats_listener', target=driver_listener.stats_listener,
+        name='stats_listener', target=_process_wrapper,
-        args=(exit_event,))
+        args=(exit_event, 'stats_listener', driver_listener.stats_listener))
     processes.append(stats_listener_proc)
 
     LOG.info("Driver agent statistics listener process starts:")
     stats_listener_proc.start()
 
     get_listener_proc = multiprocessing.Process(
-        name='get_listener', target=driver_listener.get_listener,
+        name='get_listener', target=_process_wrapper,
-        args=(exit_event,))
+        args=(exit_event, 'get_listener', driver_listener.get_listener))
     processes.append(get_listener_proc)
 
     LOG.info("Driver agent get listener process starts:")
     get_listener_proc.start()
 
+    _start_provider_agents(exit_event)
+
     def process_cleanup(*args, **kwargs):
-        LOG.info("Driver agent exiting due to signal")
+        LOG.info("Driver agent exiting due to signal.")
         exit_event.set()
         status_listener_proc.join()
         stats_listener_proc.join()
         get_listener_proc.join()
 
+        for proc in PROVIDER_AGENT_PROCESSES:
+            LOG.info('Waiting up to %s seconds for provider agent "%s" to '
+                     'shutdown.',
+                     CONF.driver_agent.provider_agent_shutdown_timeout,
+                     proc.name)
+            try:
+                proc.join(CONF.driver_agent.provider_agent_shutdown_timeout)
+                if proc.exitcode is None:
+                    # TODO(johnsom) Change to proc.kill() once
+                    #               python 3.7 or newer only
+                    os.kill(proc.pid, signal.SIGKILL)
+                    LOG.warning(
+                        'Forcefully killed "%s" provider agent because it '
+                        'failed to shutdown in %s seconds.', proc.name,
+                        CONF.driver_agent.provider_agent_shutdown_timeout)
+            except Exception as e:
+                LOG.warning('Unknown error "%s" while shutting down "%s", '
+                            'ignoring and continuing shutdown process.',
+                            str(e), proc.name)
+            else:
+                LOG.info('Provider agent "%s" has succesfully shutdown.',
+                         proc.name)
+
     signal.signal(signal.SIGTERM, process_cleanup)
     signal.signal(signal.SIGHUP, partial(
         _handle_mutate_config, status_listener_proc.pid,

octavia/common/config.py

@@ -668,6 +668,13 @@ driver_agent_opts = [
                  help=_('Percentage of max_processes (both status and stats) '
                         'in use to start logging warning messages about an '
                         'overloaded driver-agent.')),
+    cfg.IntOpt('provider_agent_shutdown_timeout',
+               default=60,
+               help=_('The time, in seconds, to wait for provider agents '
+                      'to shutdown after the exit event has been set.')),
+    cfg.ListOpt('enabled_provider_agents', default='',
+                help=_('List of enabled provider agents. The driver-agent '
+                       'will launch these agents at startup.'))
 ]
 
 # Register the configuration options

octavia/tests/unit/api/drivers/driver_agent/test_driver_listener.py

@@ -135,11 +135,6 @@ class TestDriverListener(base.TestCase):
         mock_send.assert_called_with(b'15\n')
         mock_sendall.assert_called_with(jsonutils.dump_as_bytes(TEST_OBJECT))
 
-    @mock.patch('octavia.api.drivers.driver_agent.driver_listener.CONF')
-    def test_mutate_config(self, mock_conf):
-        driver_listener._mutate_config()
-        mock_conf.mutate_config_files.assert_called_once()
-
     @mock.patch('os.remove')
     def test_cleanup_socket_file(self, mock_remove):
         mock_remove.side_effect = [mock.DEFAULT, OSError,
@@ -154,11 +149,9 @@ class TestDriverListener(base.TestCase):
 
     @mock.patch('octavia.api.drivers.driver_agent.driver_listener.'
                 '_cleanup_socket_file')
-    @mock.patch('octavia.api.drivers.driver_agent.driver_listener.signal')
     @mock.patch('octavia.api.drivers.driver_agent.driver_listener.'
                 'ForkingUDSServer')
-    def test_status_listener(self, mock_forking_server,
+    def test_status_listener(self, mock_forking_server, mock_cleanup):
-                             mock_signal, mock_cleanup):
         mock_server = mock.MagicMock()
         mock_active_children = mock.PropertyMock(
             side_effect=['a', 'a', 'a',
@@ -176,11 +169,9 @@ class TestDriverListener(base.TestCase):
 
     @mock.patch('octavia.api.drivers.driver_agent.driver_listener.'
                 '_cleanup_socket_file')
-    @mock.patch('octavia.api.drivers.driver_agent.driver_listener.signal')
     @mock.patch('octavia.api.drivers.driver_agent.driver_listener.'
                 'ForkingUDSServer')
-    def test_stats_listener(self, mock_forking_server,
+    def test_stats_listener(self, mock_forking_server, mock_cleanup):
-                            mock_signal, mock_cleanup):
         mock_server = mock.MagicMock()
         mock_active_children = mock.PropertyMock(
             side_effect=['a', 'a', 'a',
@@ -197,11 +188,9 @@ class TestDriverListener(base.TestCase):
 
     @mock.patch('octavia.api.drivers.driver_agent.driver_listener.'
                 '_cleanup_socket_file')
-    @mock.patch('octavia.api.drivers.driver_agent.driver_listener.signal')
     @mock.patch('octavia.api.drivers.driver_agent.driver_listener.'
                 'ForkingUDSServer')
-    def test_get_listener(self, mock_forking_server,
+    def test_get_listener(self, mock_forking_server, mock_cleanup):
-                          mock_signal, mock_cleanup):
         mock_server = mock.MagicMock()
         mock_active_children = mock.PropertyMock(
             side_effect=['a', 'a', 'a',

octavia/tests/unit/api/drivers/test_provider_noop_agent.py

@@ -0,0 +1,33 @@
+#    Copyright 2019 Red Hat, Inc. All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from octavia.api.drivers.noop_driver import agent
+import octavia.tests.unit.base as base
+
+
+class TestNoopProviderAgent(base.TestCase):
+
+    def setUp(self):
+        super(TestNoopProviderAgent, self).setUp()
+
+    @mock.patch('time.sleep')
+    def test_noop_provider_agent(self, mock_sleep):
+        mock_exit_event = mock.MagicMock()
+        mock_exit_event.is_set.side_effect = [False, True]
+
+        agent.noop_provider_agent(mock_exit_event)
+
+        mock_sleep.assert_called_once_with(1)

octavia/tests/unit/cmd/test_driver_agent.py

@@ -15,16 +15,21 @@
 import signal
 
 import mock
+from oslo_config import cfg
+from oslo_config import fixture as oslo_fixture
 
 import octavia.api.drivers.driver_agent.driver_listener
 from octavia.cmd import driver_agent
 from octavia.tests.unit import base
 
+CONF = cfg.CONF
+
 
 class TestDriverAgentCMD(base.TestCase):
 
     def setUp(self):
         super(TestDriverAgentCMD, self).setUp()
+        self.CONF = self.useFixture(oslo_fixture.Config(cfg.CONF))
 
     @mock.patch('os.kill')
     @mock.patch('octavia.cmd.driver_agent.CONF')
@@ -34,23 +39,117 @@ class TestDriverAgentCMD(base.TestCase):
         os_calls = [mock.call(1, signal.SIGHUP), mock.call(2, signal.SIGHUP)]
         mock_os_kill.assert_has_calls(os_calls, any_order=True)
 
+    def test_check_if_provider_agent_enabled(self):
+        mock_extension = mock.MagicMock()
+        self.CONF.config(group="driver_agent",
+                         enabled_provider_agents=[
+                             'spiffy_agent', 'super_agent'])
+        mock_extension.name = 'super_agent'
+        self.assertTrue(
+            driver_agent._check_if_provider_agent_enabled(mock_extension))
+        mock_extension.name = 'bogus_agent'
+        self.assertFalse(
+            driver_agent._check_if_provider_agent_enabled(mock_extension))
+
+    @mock.patch('setproctitle.setproctitle')
     @mock.patch('signal.signal')
+    def test_process_wrapper(self, mock_signal, mock_setproctitle):
+        mock_exit_event = mock.MagicMock()
+        mock_function = mock.MagicMock()
+        mock_function.side_effect = [
+            mock.DEFAULT, Exception('boom'), mock.DEFAULT, Exception('boom'),
+            mock.DEFAULT]
+        mock_exit_event.is_set.side_effect = [False, False, True,
+                                              False, False, True]
+
+        signal_calls = [mock.call(signal.SIGINT, signal.SIG_IGN),
+                        mock.call(signal.SIGHUP, driver_agent._mutate_config)]
+        # With agent_name
+        driver_agent._process_wrapper(
+            mock_exit_event, 'test_proc_name', mock_function,
+            agent_name='test_agent_name')
+        mock_signal.assert_has_calls(signal_calls)
+        mock_setproctitle.assert_called_once_with(
+            'octavia-driver-agent - test_proc_name -- test_agent_name')
+        mock_function.assert_called_once_with(mock_exit_event)
+
+        # With agent_name - With function exception
+        mock_signal.reset_mock()
+        mock_setproctitle.reset_mock()
+        mock_function.reset_mock()
+        driver_agent._process_wrapper(
+            mock_exit_event, 'test_proc_name', mock_function,
+            agent_name='test_agent_name')
+        mock_signal.assert_has_calls(signal_calls)
+        mock_setproctitle.assert_called_once_with(
+            'octavia-driver-agent - test_proc_name -- test_agent_name')
+        mock_function.assert_called_once_with(mock_exit_event)
+
+        # Without agent_name
+        mock_signal.reset_mock()
+        mock_setproctitle.reset_mock()
+        mock_function.reset_mock()
+        driver_agent._process_wrapper(
+            mock_exit_event, 'test_proc_name', mock_function)
+        mock_signal.assert_has_calls(signal_calls)
+        mock_setproctitle.assert_called_once_with(
+            'octavia-driver-agent - test_proc_name')
+        mock_function.assert_called_once_with(mock_exit_event)
+
+        # Without agent_name - With function exception
+        mock_signal.reset_mock()
+        mock_setproctitle.reset_mock()
+        mock_function.reset_mock()
+        driver_agent._process_wrapper(
+            mock_exit_event, 'test_proc_name', mock_function)
+        mock_signal.assert_has_calls(signal_calls)
+        mock_setproctitle.assert_called_once_with(
+            'octavia-driver-agent - test_proc_name')
+        mock_function.assert_called_once_with(mock_exit_event)
+
+    @mock.patch('octavia.cmd.driver_agent.multiprocessing')
+    @mock.patch('stevedore.enabled.EnabledExtensionManager')
+    def test_start_provider_agents(self, mock_stevedore, mock_multiprocessing):
+        mock_extension = mock.MagicMock()
+        mock_extension.name = 'test_extension'
+        mock_exit_event = mock.MagicMock()
+        mock_stevedore.return_value = [mock_extension]
+        mock_ext_proc = mock.MagicMock()
+        mock_multiprocessing.Process.return_value = mock_ext_proc
+
+        driver_agent._start_provider_agents(mock_exit_event)
+
+        mock_stevedore.assert_called_once_with(
+            namespace='octavia.driver_agent.provider_agents',
+            check_func=driver_agent._check_if_provider_agent_enabled)
+        mock_multiprocessing.Process.assert_called_once_with(
+            name='test_extension', target=driver_agent._process_wrapper,
+            args=(mock_exit_event, 'provider_agent', mock_extension.plugin),
+            kwargs={'agent_name': 'test_extension'})
+        mock_ext_proc.start.assert_called_once_with()
+
+    @mock.patch('os.kill')
     @mock.patch('octavia.cmd.driver_agent.multiprocessing')
     @mock.patch('oslo_reports.guru_meditation_report.TextGuruMeditation.'
                 'setup_autorun')
     @mock.patch('octavia.common.service.prepare_service')
     def test_main(self, mock_prep_srvc, mock_gmr, mock_multiprocessing,
-                  mock_signal):
+                  mock_kill):
         mock_exit_event = mock.MagicMock()
         mock_multiprocessing.Event.return_value = mock_exit_event
         mock_status_listener_proc = mock.MagicMock()
         mock_stats_listener_proc = mock.MagicMock()
         mock_get_listener_proc = mock.MagicMock()
-        mock_multiprocessing.Process.side_effect = [mock_status_listener_proc,
+        mock_multiprocessing.Process.side_effect = [
-                                                    mock_stats_listener_proc,
+            mock_status_listener_proc, mock_stats_listener_proc,
             mock_get_listener_proc,
-                                                    mock_status_listener_proc,
+            mock_status_listener_proc, mock_stats_listener_proc,
-                                                    mock_stats_listener_proc,
+            mock_get_listener_proc,
+            mock_status_listener_proc, mock_stats_listener_proc,
+            mock_get_listener_proc,
+            mock_status_listener_proc, mock_stats_listener_proc,
+            mock_get_listener_proc,
+            mock_status_listener_proc, mock_stats_listener_proc,
             mock_get_listener_proc]
         driver_agent.main()
         mock_prep_srvc.assert_called_once()
@@ -76,3 +175,40 @@ class TestDriverAgentCMD(base.TestCase):
         mock_stats_listener_proc.join.side_effect = [KeyboardInterrupt, None]
         driver_agent.main()
         mock_exit_event.set.assert_called_once()
+
+        # Test keyboard interrupt with provider agents
+        mock_exit_event.reset_mock()
+        mock_stats_listener_proc.join.side_effect = [KeyboardInterrupt, None]
+        mock_provider_proc = mock.MagicMock()
+        mock_provider_proc.pid = 'not-valid-pid'
+        mock_provider_proc.exitcode = 1
+        driver_agent.PROVIDER_AGENT_PROCESSES = [mock_provider_proc]
+        driver_agent.main()
+        mock_exit_event.set.assert_called_once()
+        mock_provider_proc.join.assert_called_once_with(
+            CONF.driver_agent.provider_agent_shutdown_timeout)
+
+        # Test keyboard interrupt with provider agents fails to stop
+        mock_exit_event.reset_mock()
+        mock_stats_listener_proc.join.side_effect = [KeyboardInterrupt, None]
+        mock_provider_proc = mock.MagicMock()
+        mock_provider_proc.pid = 'not-valid-pid'
+        mock_provider_proc.exitcode = None
+        driver_agent.PROVIDER_AGENT_PROCESSES = [mock_provider_proc]
+        driver_agent.main()
+        mock_exit_event.set.assert_called_once()
+        mock_provider_proc.join.assert_called_once_with(
+            CONF.driver_agent.provider_agent_shutdown_timeout)
+        mock_kill.assert_called_once_with('not-valid-pid', signal.SIGKILL)
+
+        # Test keyboard interrupt with provider agents join exception
+        mock_exit_event.reset_mock()
+        mock_stats_listener_proc.join.side_effect = [KeyboardInterrupt, None]
+        mock_provider_proc = mock.MagicMock()
+        mock_provider_proc.pid = 'not-valid-pid'
+        mock_provider_proc.join.side_effect = Exception('boom')
+        driver_agent.PROVIDER_AGENT_PROCESSES = [mock_provider_proc]
+        driver_agent.main()
+        mock_exit_event.set.assert_called_once()
+        mock_provider_proc.join.assert_called_once_with(
+            CONF.driver_agent.provider_agent_shutdown_timeout)

releasenotes/notes/Add-provider-agent-support-a735806c4da4c470.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    The Octavia driver-agent now supports starting provider driver agents.
+    Provider driver agents are long running agent processes supporting
+    provider drivers.

requirements.txt

@@ -48,6 +48,7 @@ debtcollector>=1.19.0 # Apache-2.0
 octavia-lib>=1.3.1 # Apache-2.0
 netaddr>=0.7.19 # BSD
 simplejson>=3.13.2 # MIT
+setproctitle>=1.1.10 # BSD
 
 #for the amphora api
 Flask!=0.11,>=0.10 # BSD

setup.cfg

@@ -75,6 +75,8 @@ octavia.amphora.udp_api_server =
 octavia.compute.drivers =
     compute_noop_driver = octavia.compute.drivers.noop_driver.driver:NoopComputeDriver
     compute_nova_driver = octavia.compute.drivers.nova_driver:VirtualMachineManager
+octavia.driver_agent.provider_agents =
+    noop_agent = octavia.api.drivers.noop_driver.agent:noop_provider_agent
 octavia.network.drivers =
     network_noop_driver = octavia.network.drivers.noop_driver.driver:NoopNetworkDriver
     allowed_address_pairs_driver = octavia.network.drivers.neutron.allowed_address_pairs:AllowedAddressPairsDriver