From d700c00a90fd62b4f6cb9eb30ebe5f619dd6bfda Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 12 Jun 2019 15:20:41 -0700 Subject: [PATCH] Add get methods to the driver-lib This patch adds get methods to the driver-lib which allows provider drivers to query for objects by ID. In support of the get methods, this patch fixes a bug in the data model to_dict() where it may not properly recurse the objects. It also improves connecting to the driver agent sockets and adds a timeout while waiting to receive data from the driver-agent. Change-Id: Ia69d1f61571a1a65dee585037affb317999d7ade Story: 2005870 Task: 33682 --- lower-constraints.txt | 1 + octavia_lib/api/drivers/data_models.py | 1 + octavia_lib/api/drivers/driver_lib.py | 153 +++++++++++++++++- octavia_lib/api/drivers/exceptions.py | 48 +++++- octavia_lib/common/constants.py | 1 + .../tests/unit/api/drivers/test_driver_lib.py | 112 ++++++++++++- ...ethods-to-driver-lib-dae3c217e7ac9e82.yaml | 11 ++ requirements.txt | 1 + 8 files changed, 320 insertions(+), 8 deletions(-) create mode 100644 releasenotes/notes/Add-get-methods-to-driver-lib-dae3c217e7ac9e82.yaml diff --git a/lower-constraints.txt b/lower-constraints.txt index 5ce8f74..8dec028 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -13,4 +13,5 @@ pylint==1.9.2 python-subunit==1.0.0 six==1.10.0 stestr==2.0.0 +tenacity==5.0.2 testtools==2.2.0 diff --git a/octavia_lib/api/drivers/data_models.py b/octavia_lib/api/drivers/data_models.py index d318124..48ff2f6 100644 --- a/octavia_lib/api/drivers/data_models.py +++ b/octavia_lib/api/drivers/data_models.py @@ -51,6 +51,7 @@ class BaseDataModel(object): elif isinstance(getattr(self, attr), BaseDataModel): if type(self) not in calling_classes: ret[attr] = value.to_dict( + recurse=recurse, render_unsets=render_unsets, calling_classes=calling_classes + [type(self)]) else: diff --git a/octavia_lib/api/drivers/driver_lib.py b/octavia_lib/api/drivers/driver_lib.py index 117db38..473fefb 100644 --- a/octavia_lib/api/drivers/driver_lib.py +++ b/octavia_lib/api/drivers/driver_lib.py @@ -12,40 +12,77 @@ # License for the specific language governing permissions and limitations # under the License. +import os import socket +import time from oslo_serialization import jsonutils +import tenacity +from octavia_lib.api.drivers import data_models from octavia_lib.api.drivers import exceptions as driver_exceptions from octavia_lib.common import constants DEFAULT_STATUS_SOCKET = '/var/run/octavia/status.sock' DEFAULT_STATS_SOCKET = '/var/run/octavia/stats.sock' +DEFAULT_GET_SOCKET = '/var/run/octavia/get.sock' SOCKET_TIMEOUT = 5 +DRIVER_AGENT_TIMEOUT = 30 class DriverLibrary(object): + @tenacity.retry( + stop=tenacity.stop_after_attempt(30), reraise=True, + wait=tenacity.wait_exponential(multiplier=1, min=1, max=5), + retry=tenacity.retry_if_exception_type( + driver_exceptions.DriverAgentNotFound)) + def _check_for_socket_ready(self, socket): + if not os.path.exists(socket): + raise driver_exceptions.DriverAgentNotFound( + fault_string=('Unable to open the driver agent ' + 'socket: {}'.format(socket))) + def __init__(self, status_socket=DEFAULT_STATUS_SOCKET, - stats_socket=DEFAULT_STATS_SOCKET, **kwargs): + stats_socket=DEFAULT_STATS_SOCKET, + get_socket=DEFAULT_GET_SOCKET, **kwargs): self.status_socket = status_socket self.stats_socket = stats_socket + self.get_socket = get_socket + + self._check_for_socket_ready(status_socket) + self._check_for_socket_ready(stats_socket) + self._check_for_socket_ready(get_socket) super(DriverLibrary, self).__init__(**kwargs) def _recv(self, sock): size_str = b'' char = sock.recv(1) + begin = time.time() while char != b'\n': size_str += char char = sock.recv(1) + if time.time() - begin > DRIVER_AGENT_TIMEOUT: + raise driver_exceptions.DriverAgentTimeout( + fault_string=('The driver agent did not respond in {} ' + 'seconds.'.format(DRIVER_AGENT_TIMEOUT))) + # Give the CPU a break from polling + time.sleep(0.01) payload_size = int(size_str) mv_buffer = memoryview(bytearray(payload_size)) next_offset = 0 + begin = time.time() while payload_size - next_offset > 0: recv_size = sock.recv_into(mv_buffer[next_offset:], payload_size - next_offset) next_offset += recv_size + if time.time() - begin > DRIVER_AGENT_TIMEOUT: + raise driver_exceptions.DriverAgentTimeout( + fault_string=('The driver agent did not respond in {} ' + 'seconds.'.format(DRIVER_AGENT_TIMEOUT))) + # Give the CPU a break from polling + time.sleep(0.01) return jsonutils.loads(mv_buffer.tobytes()) def _send(self, socket_path, data): @@ -114,3 +151,117 @@ class DriverLibrary(object): stats_object=response.pop(constants.STATS_OBJECT, None), stats_object_id=response.pop(constants.STATS_OBJECT_ID, None), stats_record=response.pop(constants.STATS_RECORD, None)) + + def _get_resource(self, resource, id): + try: + return self._send(self.get_socket, {constants.OBJECT: resource, + constants.ID: id}) + except driver_exceptions.DriverAgentTimeout: + raise + except Exception: + raise driver_exceptions.DriverError() + + def get_loadbalancer(self, loadbalancer_id): + """Get a load balancer object. + + :param loadbalancer_id: The load balancer ID to lookup. + :type loadbalancer_id: UUID string + :raises DriverAgentTimeout: The driver agent did not respond + inside the timeout. + :raises DriverError: An unexpected error occurred. + :returns: A LoadBalancer object or None if not found. + """ + data = self._get_resource(constants.LOADBALANCERS, loadbalancer_id) + if data: + return data_models.LoadBalancer.from_dict(data) + return None + + def get_listener(self, listener_id): + """Get a listener object. + + :param listener_id: The listener ID to lookup. + :type listener_id: UUID string + :raises DriverAgentTimeout: The driver agent did not respond + inside the timeout. + :raises DriverError: An unexpected error occurred. + :returns: A Listener object or None if not found. + """ + data = self._get_resource(constants.LISTENERS, listener_id) + if data: + return data_models.Listener.from_dict(data) + return None + + def get_pool(self, pool_id): + """Get a pool object. + + :param pool_id: The pool ID to lookup. + :type pool_id: UUID string + :raises DriverAgentTimeout: The driver agent did not respond + inside the timeout. + :raises DriverError: An unexpected error occurred. + :returns: A Pool object or None if not found. + """ + data = self._get_resource(constants.POOLS, pool_id) + if data: + return data_models.Pool.from_dict(data) + return None + + def get_healthmonitor(self, healthmonitor_id): + """Get a health monitor object. + + :param healthmonitor_id: The health monitor ID to lookup. + :type healthmonitor_id: UUID string + :raises DriverAgentTimeout: The driver agent did not respond + inside the timeout. + :raises DriverError: An unexpected error occurred. + :returns: A HealthMonitor object or None if not found. + """ + data = self._get_resource(constants.HEALTHMONITORS, healthmonitor_id) + if data: + return data_models.HealthMonitor.from_dict(data) + return None + + def get_member(self, member_id): + """Get a member object. + + :param member_id: The member ID to lookup. + :type member_id: UUID string + :raises DriverAgentTimeout: The driver agent did not respond + inside the timeout. + :raises DriverError: An unexpected error occurred. + :returns: A Member object or None if not found. + """ + data = self._get_resource(constants.MEMBERS, member_id) + if data: + return data_models.Member.from_dict(data) + return None + + def get_l7policy(self, l7policy_id): + """Get a L7 policy object. + + :param l7policy_id: The L7 policy ID to lookup. + :type l7policy_id: UUID string + :raises DriverAgentTimeout: The driver agent did not respond + inside the timeout. + :raises DriverError: An unexpected error occurred. + :returns: A L7Policy object or None if not found. + """ + data = self._get_resource(constants.L7POLICIES, l7policy_id) + if data: + return data_models.L7Policy.from_dict(data) + return None + + def get_l7rule(self, l7rule_id): + """Get a L7 rule object. + + :param l7rule_id: The L7 rule ID to lookup. + :type l7rule_id: UUID string + :raises DriverAgentTimeout: The driver agent did not respond + inside the timeout. + :raises DriverError: An unexpected error occurred. + :returns: A L7Rule object or None if not found. + """ + data = self._get_resource(constants.L7RULES, l7rule_id) + if data: + return data_models.L7Rule.from_dict(data) + return None diff --git a/octavia_lib/api/drivers/exceptions.py b/octavia_lib/api/drivers/exceptions.py index ee2fa0f..425afe3 100644 --- a/octavia_lib/api/drivers/exceptions.py +++ b/octavia_lib/api/drivers/exceptions.py @@ -38,7 +38,8 @@ class DriverError(Exception): self.user_fault_string) self.operator_fault_string = kwargs.pop('operator_fault_string', self.operator_fault_string) - super(DriverError, self).__init__(*args, **kwargs) + super(DriverError, self).__init__(self.user_fault_string, + *args, **kwargs) class NotImplementedError(Exception): @@ -59,7 +60,8 @@ class NotImplementedError(Exception): self.user_fault_string) self.operator_fault_string = kwargs.pop('operator_fault_string', self.operator_fault_string) - super(NotImplementedError, self).__init__(*args, **kwargs) + super(NotImplementedError, self).__init__(self.user_fault_string, + *args, **kwargs) class UnsupportedOptionError(Exception): @@ -88,7 +90,8 @@ class UnsupportedOptionError(Exception): self.user_fault_string) self.operator_fault_string = kwargs.pop('operator_fault_string', self.operator_fault_string) - super(UnsupportedOptionError, self).__init__(*args, **kwargs) + super(UnsupportedOptionError, self).__init__(self.user_fault_string, + *args, **kwargs) class UpdateStatusError(Exception): @@ -116,7 +119,8 @@ class UpdateStatusError(Exception): self.status_object_id = kwargs.pop('status_object_id', None) self.status_record = kwargs.pop('status_record', None) - super(UpdateStatusError, self).__init__(*args, **kwargs) + super(UpdateStatusError, self).__init__(self.fault_string, + *args, **kwargs) class UpdateStatisticsError(Exception): @@ -145,4 +149,38 @@ class UpdateStatisticsError(Exception): self.stats_object_id = kwargs.pop('stats_object_id', None) self.stats_record = kwargs.pop('stats_record', None) - super(UpdateStatisticsError, self).__init__(*args, **kwargs) + super(UpdateStatisticsError, self).__init__(self.fault_string, + *args, **kwargs) + + +class DriverAgentNotFound(Exception): + """Exception raised when the driver agent cannot be reached. + + Each exception will include a message field that describes the + error. + :param fault_string: String describing the fault. + :type fault_string: string + """ + fault_string = _("The driver-agent process was not found or not ready.") + + def __init__(self, *args, **kwargs): + self.fault_string = kwargs.pop('fault_string', self.fault_string) + super(DriverAgentNotFound, self).__init__(self.fault_string, + *args, **kwargs) + + +class DriverAgentTimeout(Exception): + """Exception raised when the driver agent does not respond. + + Raised when communication with the driver agent times out. + Each exception will include a message field that describes the + error. + :param fault_string: String describing the fault. + :type fault_string: string + """ + fault_string = _("The driver-agent timeout.") + + def __init__(self, *args, **kwargs): + self.fault_string = kwargs.pop('fault_string', self.fault_string) + super(DriverAgentTimeout, self).__init__(self.fault_string, + *args, **kwargs) diff --git a/octavia_lib/common/constants.py b/octavia_lib/common/constants.py index aa8720d..43d7d24 100644 --- a/octavia_lib/common/constants.py +++ b/octavia_lib/common/constants.py @@ -34,6 +34,7 @@ MEMBERS = 'members' L7POLICIES = 'l7policies' L7RULES = 'l7rules' FLAVOR = 'flavor' +OBJECT = 'object' # ID fields ID = 'id' diff --git a/octavia_lib/tests/unit/api/drivers/test_driver_lib.py b/octavia_lib/tests/unit/api/drivers/test_driver_lib.py index 2e8d4a2..0e3ecd4 100644 --- a/octavia_lib/tests/unit/api/drivers/test_driver_lib.py +++ b/octavia_lib/tests/unit/api/drivers/test_driver_lib.py @@ -16,19 +16,37 @@ import mock from octavia_lib.api.drivers import driver_lib from octavia_lib.api.drivers import exceptions as driver_exceptions +from octavia_lib.common import constants from octavia_lib.tests.unit import base class TestDriverLib(base.TestCase): - def setUp(self): + + @mock.patch('octavia_lib.api.drivers.driver_lib.DriverLibrary.' + '_check_for_socket_ready') + def setUp(self, mock_check_ready): self.driver_lib = driver_lib.DriverLibrary() super(TestDriverLib, self).setUp() + @mock.patch('octavia_lib.api.drivers.driver_lib.DriverLibrary.' + '_check_for_socket_ready.retry.sleep') + @mock.patch('os.path.exists') + def test_check_for_socket_ready(self, mock_path_exists, mock_sleep): + mock_path_exists.return_value = True + + # should not raise an exception + self.driver_lib._check_for_socket_ready('bogus') + + mock_path_exists.return_value = False + self.assertRaises(driver_exceptions.DriverAgentNotFound, + self.driver_lib._check_for_socket_ready, + 'bogus') + @mock.patch('six.moves.builtins.memoryview') def test_recv(self, mock_memoryview): mock_socket = mock.MagicMock() - mock_socket.recv.side_effect = [b'1', b'\n'] + mock_socket.recv.side_effect = [b'1', b'\n', b'2', b'\n', b'3', b'\n'] mock_socket.recv_into.return_value = 1 mv_mock = mock.MagicMock() mock_memoryview.return_value = mv_mock @@ -43,6 +61,17 @@ class TestDriverLib(base.TestCase): mv_mock.__getitem__(), 1) self.assertEqual('test data', response) + # Test size recv timeout + with mock.patch('octavia_lib.api.drivers.driver_lib.' + 'time') as mock_time: + mock_time.time.side_effect = [0, 1000, 0, 0, 0, 0, 1000] + self.assertRaises(driver_exceptions.DriverAgentTimeout, + self.driver_lib._recv, mock_socket) + + # Test payload recv timeout + self.assertRaises(driver_exceptions.DriverAgentTimeout, + self.driver_lib._recv, mock_socket) + @mock.patch('octavia_lib.api.drivers.driver_lib.DriverLibrary._recv') def test_send(self, mock_recv): mock_socket = mock.MagicMock() @@ -105,3 +134,82 @@ class TestDriverLib(base.TestCase): self.assertRaises(driver_exceptions.UpdateStatisticsError, self.driver_lib.update_listener_statistics, 'fake_stats') + + @mock.patch('octavia_lib.api.drivers.driver_lib.DriverLibrary._send') + def test_get_resource(self, mock_send): + fake_resource = 'fake resource' + fake_id = 'fake id' + + mock_send.side_effect = ['some result', + driver_exceptions.DriverAgentTimeout, + Exception('boom')] + + result = self.driver_lib._get_resource(fake_resource, fake_id) + + data = {constants.OBJECT: fake_resource, constants.ID: fake_id} + mock_send.assert_called_once_with('/var/run/octavia/get.sock', data) + self.assertEqual('some result', result) + + # Test with driver_exceptions.DriverAgentTimeout + self.assertRaises(driver_exceptions.DriverAgentTimeout, + self.driver_lib._get_resource, + fake_resource, fake_id) + + # Test with random exception + self.assertRaises(driver_exceptions.DriverError, + self.driver_lib._get_resource, + fake_resource, fake_id) + + @mock.patch('octavia_lib.api.drivers.driver_lib.DriverLibrary.' + '_get_resource') + def _test_get_object(self, get_method, name, mock_from_dict, + mock_get_resource): + + mock_get_resource.side_effect = ['some data', None] + mock_from_dict.return_value = 'object' + + result = get_method('fake id') + + mock_get_resource.assert_called_once_with(name, 'fake id') + mock_from_dict.assert_called_once_with('some data') + self.assertEqual('object', result) + + # Test not found + result = get_method('fake id') + + self.assertIsNone(result) + + @mock.patch('octavia_lib.api.drivers.data_models.LoadBalancer.from_dict') + def test_get_loadbalancer(self, mock_from_dict): + self._test_get_object(self.driver_lib.get_loadbalancer, + constants.LOADBALANCERS, mock_from_dict) + + @mock.patch('octavia_lib.api.drivers.data_models.Listener.from_dict') + def test_get_listener(self, mock_from_dict): + self._test_get_object(self.driver_lib.get_listener, + constants.LISTENERS, mock_from_dict) + + @mock.patch('octavia_lib.api.drivers.data_models.Pool.from_dict') + def test_get_pool(self, mock_from_dict): + self._test_get_object(self.driver_lib.get_pool, + constants.POOLS, mock_from_dict) + + @mock.patch('octavia_lib.api.drivers.data_models.HealthMonitor.from_dict') + def test_get_healthmonitor(self, mock_from_dict): + self._test_get_object(self.driver_lib.get_healthmonitor, + constants.HEALTHMONITORS, mock_from_dict) + + @mock.patch('octavia_lib.api.drivers.data_models.Member.from_dict') + def test_get_member(self, mock_from_dict): + self._test_get_object(self.driver_lib.get_member, + constants.MEMBERS, mock_from_dict) + + @mock.patch('octavia_lib.api.drivers.data_models.L7Policy.from_dict') + def test_get_l7policy(self, mock_from_dict): + self._test_get_object(self.driver_lib.get_l7policy, + constants.L7POLICIES, mock_from_dict) + + @mock.patch('octavia_lib.api.drivers.data_models.L7Rule.from_dict') + def test_get_l7rule(self, mock_from_dict): + self._test_get_object(self.driver_lib.get_l7rule, + constants.L7RULES, mock_from_dict) diff --git a/releasenotes/notes/Add-get-methods-to-driver-lib-dae3c217e7ac9e82.yaml b/releasenotes/notes/Add-get-methods-to-driver-lib-dae3c217e7ac9e82.yaml new file mode 100644 index 0000000..0e3307f --- /dev/null +++ b/releasenotes/notes/Add-get-methods-to-driver-lib-dae3c217e7ac9e82.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + The driver-lib now provides "get" methods for drivers to be able to + query for objects by id. For example, get_loadbalancer(loadbalancer_id). +fixes: + - Improved the driver_lib connecting to the driver-agent sockets. + - Fixed a bug where the data model to_dict() may not recurse properly. + - | + Message receiving for the driver_lib will timeout after no response + from the driver-agent. diff --git a/requirements.txt b/requirements.txt index 584f574..0a6f803 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ oslo.log>=3.36.0 # Apache-2.0 oslo.serialization>=2.28.1 # Apache-2.0 pbr!=2.1.0,>=2.0.0 # Apache-2.0 six>=1.10.0 # MIT +tenacity>=5.0.2 # Apache-2.0