# Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import random import time from unittest import mock from oslo_config import cfg from oslo_utils import uuidutils import sqlalchemy from octavia.common import constants from octavia.common import data_models from octavia.controller.healthmanager.health_drivers import update_db from octavia.db import models as db_models from octavia.tests.unit import base class TestException(Exception): def __init__(self, value): self.value = value def __str__(self): return repr(self.value) class TestUpdateHealthDb(base.TestCase): FAKE_UUID_1 = uuidutils.generate_uuid() def setUp(self): super(TestUpdateHealthDb, self).setUp() session_patch = mock.patch('octavia.db.api.get_session') self.addCleanup(session_patch.stop) self.mock_session = session_patch.start() self.session_mock = mock.MagicMock() self.mock_session.return_value = self.session_mock self.hm = update_db.UpdateHealthDb() self.amphora_repo = mock.MagicMock() self.amphora_health_repo = mock.MagicMock() self.listener_repo = mock.MagicMock() self.loadbalancer_repo = mock.MagicMock() self.member_repo = mock.MagicMock() self.pool_repo = mock.MagicMock() self.hm.amphora_repo = self.amphora_repo self.hm.amphora_health_repo = self.amphora_health_repo self.hm.listener_repo = self.listener_repo self.hm.listener_repo.count.return_value = 1 self.hm.loadbalancer_repo = self.loadbalancer_repo self.hm.member_repo = self.member_repo self.hm.pool_repo = self.pool_repo def _make_mock_lb_tree(self, listener=True, pool=True, health_monitor=True, members=1, lb_prov_status=constants.ACTIVE): mock_lb = mock.Mock() mock_lb.id = self.FAKE_UUID_1 mock_lb.pools = [] mock_lb.listeners = [] mock_lb.provisioning_status = lb_prov_status mock_lb.operating_status = 'blah' mock_listener1 = None mock_pool1 = None mock_members = None if listener: mock_listener1 = mock.Mock() mock_listener1.id = 'listener-id-1' mock_lb.listeners = [mock_listener1] if pool: mock_pool1 = mock.Mock() mock_pool1.id = "pool-id-1" mock_pool1.members = [] if health_monitor: mock_hm1 = mock.Mock() mock_pool1.health_monitor = mock_hm1 else: mock_pool1.health_monitor = None mock_lb.pools = [mock_pool1] if mock_listener1: mock_listener1.pools = [mock_pool1] mock_listener1.default_pool = mock_pool1 for i in range(members): mock_member_x = mock.Mock() mock_member_x.id = 'member-id-%s' % (i + 1) if health_monitor: mock_member_x.operating_status = 'NOTHING_MATCHABLE' else: mock_member_x.operating_status = constants.NO_MONITOR mock_pool1.members.append(mock_member_x) mock_members = mock_pool1.members return mock_lb, mock_listener1, mock_pool1, mock_members def _make_fake_lb_health_dict(self, listener=True, pool=True, health_monitor=True, members=1, lb_prov_status=constants.ACTIVE, listener_protocol=constants.PROTOCOL_TCP, enabled=True): lb_ref = {'enabled': enabled, 'id': self.FAKE_UUID_1, constants.OPERATING_STATUS: 'bogus', constants.PROVISIONING_STATUS: lb_prov_status} if pool: members_dict = {} if health_monitor: member_operating_status = 'NOTHING_MATCHABLE' else: member_operating_status = constants.NO_MONITOR for i in range(members): member_id = 'member-id-%s' % (i + 1) members_dict[member_id] = { constants.OPERATING_STATUS: member_operating_status} pool_ref = {'pool-id-1': {'members': members_dict, constants.OPERATING_STATUS: 'bogus'}} lb_ref['pools'] = pool_ref if listener: listener_ref = {'listener-id-1': { constants.OPERATING_STATUS: 'bogus', 'protocol': listener_protocol, 'enabled': True}} lb_ref['listeners'] = listener_ref return lb_ref def test_update_health_no_listener(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": {}, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict(listener=False, pool=False) self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.assertTrue(self.amphora_health_repo.replace.called) def test_update_health_lb_disabled(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": {}, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict( listener=True, pool=True, enabled=False) self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.assertTrue(self.amphora_health_repo.replace.called) def test_update_health_lb_pending_no_listener(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": {}, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict( listener=True, pool=False, lb_prov_status=constants.PENDING_UPDATE) self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.assertTrue(self.amphora_health_repo.replace.called) def test_update_health_missing_listener(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": {}, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict(listener=True, pool=False) self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.assertFalse(self.amphora_health_repo.replace.called) def test_update_health_recv_time_stale(self): hb_interval = cfg.CONF.health_manager.heartbeat_interval health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": {}, "recv_time": time.time() - hb_interval - 1 # extra -1 for buffer } lb_ref = self._make_fake_lb_health_dict(listener=False, pool=False) self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) # Receive time is stale, so we shouldn't see this called self.assertFalse(self.loadbalancer_repo.update.called) def test_update_health_replace_error(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.UP} } } } }, "recv_time": time.time() } self.session_mock.commit.side_effect = TestException('boom') lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) self.session_mock.rollback.assert_called_once() def test_update_health_online(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.UP} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ONLINE) # If the listener count is wrong, make sure we don't update lb_ref['listeners']['listener-id-2'] = { constants.OPERATING_STATUS: 'bogus'} self.amphora_health_repo.replace.reset_mock() self.hm.update_health(health, '192.0.2.1') self.assertTrue(not self.amphora_health_repo.replace.called) def test_update_health_listener_disabled(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.UP} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() lb_ref['listeners']['listener-id-2'] = { 'enabled': False, constants.OPERATING_STATUS: constants.OFFLINE} self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ONLINE) def test_update_lb_pool_health_offline(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": {}} }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) self.pool_repo.update.assert_any_call( self.session_mock, 'pool-id-1', operating_status=constants.OFFLINE ) def test_update_lb_multiple_listeners_one_error_pool(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.DOWN, "members": {"member-id-1": constants.ERROR}} }}, "listener-id-2": {"status": constants.OPEN, "pools": { "pool-id-2": {"status": constants.UP, "members": {"member-id-2": constants.UP}} }} }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() lb_ref['pools']['pool-id-2'] = { constants.OPERATING_STATUS: 'bogus', 'members': {'member-id-2': {constants.OPERATING_STATUS: 'bogus'}}} lb_ref['listeners']['listener-id-2'] = { constants.OPERATING_STATUS: 'bogus', 'protocol': constants.PROTOCOL_TCP, 'enabled': True} self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners').items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) # Call count should be exactly 2, as each pool should be processed once self.assertEqual(2, self.pool_repo.update.call_count) self.pool_repo.update.assert_has_calls([ mock.call(self.session_mock, 'pool-id-1', operating_status=constants.ERROR), mock.call(self.session_mock, 'pool-id-2', operating_status=constants.ONLINE) ], any_order=True) def test_update_lb_and_list_pool_health_online(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.UP} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): # We should not double process a shared pool self.hm.pool_repo.update.assert_called_once_with( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ONLINE) def test_update_v2_lb_and_list_pool_health_online(self): health = { "id": self.FAKE_UUID_1, "ver": 2, "listeners": { "listener-id-1": {"status": constants.OPEN} }, "pools": { "pool-id-1:listener-id-1": { "status": constants.UP, "members": {"member-id-1": constants.UP}}, "pool-id-1:listener-id-2": { "status": constants.UP, "members": {"member-id-1": constants.UP}}}, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in health.get('pools', {}).items(): # We should not double process a shared pool self.hm.pool_repo.update.assert_called_once_with( self.session_mock, 'pool-id-1', operating_status=constants.ONLINE) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ONLINE) def test_update_pool_offline(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-5": {"status": constants.UP, "members": {"member-id-1": constants.UP} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() lb_ref['pools']['pool-id-2'] = { constants.OPERATING_STATUS: constants.OFFLINE} self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) self.hm.pool_repo.update.assert_any_call( self.session_mock, "pool-id-1", operating_status=constants.OFFLINE) def test_update_health_member_drain(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": { "status": constants.OPEN, "pools": { "pool-id-1": { "status": constants.UP, "members": {"member-id-1": constants.DRAIN}}}}}, "recv_time": time.time()} lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.DRAINING) def test_update_health_member_maint(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": { "status": constants.OPEN, "pools": { "pool-id-1": { "status": constants.UP, "members": {"member-id-1": constants.MAINT}}}}}, "recv_time": time.time()} lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.OFFLINE) def test_update_health_member_unknown(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": { "status": constants.OPEN, "pools": { "pool-id-1": { "status": constants.UP, "members": {"member-id-1": "blah"}}}}}, "recv_time": time.time()} lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) self.assertTrue(not self.member_repo.update.called) def test_update_health_member_down(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.DOWN} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.DEGRADED) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ERROR) def test_update_health_member_missing_no_hm(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict(health_monitor=False) self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) self.member_repo.update.assert_not_called() def test_update_health_member_down_no_hm(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.MAINT} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict(health_monitor=False) member1 = lb_ref['pools']['pool-id-1']['members']['member-id-1'] member1[constants.OPERATING_STATUS] = constants.NO_MONITOR self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) self.member_repo.update.assert_any_call( self.session_mock, 'member-id-1', operating_status=constants.OFFLINE) def test_update_health_member_no_check(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.NO_CHECK} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.NO_MONITOR) def test_update_health_member_admin_down(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": { "status": constants.OPEN, "pools": { "pool-id-1": { "status": constants.UP, "members": { "member-id-1": constants.UP}}}}}, "recv_time": time.time()} lb_ref = self._make_fake_lb_health_dict(members=2) self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ONLINE) self.member_repo.update.assert_any_call( self.session_mock, 'member-id-2', operating_status=constants.OFFLINE) def test_update_health_list_full_member_down(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.FULL, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.DOWN} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.DEGRADED) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.DEGRADED) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ERROR) lb_ref['listeners']['listener-id-2'] = { constants.OPERATING_STATUS: 'bogus'} self.amphora_health_repo.replace.reset_mock() self.hm.update_health(health, '192.0.2.1') self.assertTrue(not self.amphora_health_repo.replace.called) def test_update_health_error(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.DOWN, "members": {"member-id-1": constants.DOWN} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ERROR) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ERROR) # Test the logic code paths def test_update_health_full(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.FULL, "pools": { "pool-id-1": {"status": constants.DOWN, "members": {"member-id-1": constants.DOWN} } } }, "listener-id-2": {"status": constants.FULL, "pools": { "pool-id-2": {"status": constants.UP, "members": {"member-id-2": constants.UP} } } }, "listener-id-3": {"status": constants.OPEN, "pools": { "pool-id-3": {"status": constants.UP, "members": {"member-id-3": constants.UP, "member-id-31": constants.DOWN} } } }, "listener-id-4": { "status": constants.OPEN, "pools": { "pool-id-4": { "status": constants.UP, "members": {"member-id-4": constants.DRAINING} } } }, "listener-id-5": { "status": "bogus", "pools": { "pool-id-5": { "status": "bogus", "members": {"member-id-5": "bogus"} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() # Build our own custom listeners/pools/members for i in [1, 2, 3, 4, 5]: lb_ref['listeners']['listener-id-%s' % i] = { constants.OPERATING_STATUS: 'bogus', 'protocol': constants.PROTOCOL_TCP, 'enabled': True} if i == 3: members_dict = {'member-id-3': { constants.OPERATING_STATUS: 'bogus'}, 'member-id-31': { constants.OPERATING_STATUS: 'bogus'}} else: members_dict = {'member-id-%s' % i: { constants.OPERATING_STATUS: 'bogus'}} lb_ref['pools']['pool-id-%s' % i] = { 'members': members_dict, constants.OPERATING_STATUS: 'bogus'} self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') # test listener self.listener_repo.update.assert_any_call( self.session_mock, "listener-id-1", operating_status=constants.DEGRADED) self.listener_repo.update.assert_any_call( self.session_mock, "listener-id-2", operating_status=constants.DEGRADED) self.pool_repo.update.assert_any_call( self.session_mock, "pool-id-1", operating_status=constants.ERROR) self.pool_repo.update.assert_any_call( self.session_mock, "pool-id-2", operating_status=constants.ONLINE) self.pool_repo.update.assert_any_call( self.session_mock, "pool-id-3", operating_status=constants.DEGRADED) self.pool_repo.update.assert_any_call( self.session_mock, "pool-id-4", operating_status=constants.ONLINE) # Test code paths where objects are not found in the database def test_update_health_not_found(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.UP} } } } }, "recv_time": time.time() } self.hm.listener_repo.update.side_effect = ( [sqlalchemy.orm.exc.NoResultFound]) self.hm.member_repo.update.side_effect = ( [sqlalchemy.orm.exc.NoResultFound]) self.hm.pool_repo.update.side_effect = ( sqlalchemy.orm.exc.NoResultFound) self.hm.loadbalancer_repo.update.side_effect = ( [sqlalchemy.orm.exc.NoResultFound]) lb_ref = self._make_fake_lb_health_dict() lb_ref['pools']['pool-id-2'] = {constants.OPERATING_STATUS: 'bogus'} self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in health.get('listeners', {}).items(): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in listener.get('pools', {}).items(): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in pool.get('members', {}).items(): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ONLINE) @mock.patch('stevedore.driver.DriverManager.driver') def test_update_health_zombie(self, mock_driver): health = {"id": self.FAKE_UUID_1, "listeners": {}} self.amphora_repo.get_lb_for_health_update.return_value = None amp_mock = mock.MagicMock() self.amphora_repo.get.return_value = amp_mock self.hm.update_health(health, '192.0.2.1') mock_driver.delete.assert_called_once_with( amp_mock.compute_id) def test_update_health_no_status_change(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": { "listener-id-1": { "status": constants.OPEN, "pools": { "pool-id-1": { "status": constants.UP, "members": { "member-id-1": constants.UP } } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() # Start everything ONLINE lb_ref[constants.OPERATING_STATUS] = constants.ONLINE listener1 = lb_ref['listeners']['listener-id-1'] listener1[constants.OPERATING_STATUS] = constants.ONLINE pool1 = lb_ref['pools']['pool-id-1'] pool1[constants.OPERATING_STATUS] = constants.ONLINE member1 = pool1['members']['member-id-1'] member1[constants.OPERATING_STATUS] = constants.ONLINE self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.loadbalancer_repo.update.assert_not_called() self.listener_repo.update.assert_not_called() self.pool_repo.update.assert_not_called() self.member_repo.update.assert_not_called() def test_update_health_lb_admin_down(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": {}, "recv_time": time.time()} lb_ref = self._make_fake_lb_health_dict(listener=False, pool=False) lb_ref['enabled'] = False self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.loadbalancer_repo.update.assert_called_with( self.mock_session(), self.FAKE_UUID_1, operating_status='OFFLINE') def test_update_health_lb_admin_up(self): health = { "id": self.FAKE_UUID_1, "listeners": {}, "recv_time": time.time(), "ver": 1} lb_ref = self._make_fake_lb_health_dict(listener=False, pool=False) self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.loadbalancer_repo.update.assert_called_with( self.mock_session(), self.FAKE_UUID_1, operating_status='ONLINE') def test_update_health_forbid_to_stale_udp_listener_amphora(self): health = { "id": self.FAKE_UUID_1, "listeners": {}, "recv_time": time.time() } mock_lb = mock.Mock() mock_lb.id = self.FAKE_UUID_1 mock_lb.pools = [] mock_lb.listeners = [] mock_lb.provisioning_status = constants.ACTIVE mock_lb.operating_status = 'blah' # The default pool of udp listener1 has no enabled member mock_member1 = mock.Mock() mock_member1.id = 'member-id-1' mock_member1.enabled = False mock_pool1 = mock.Mock() mock_pool1.id = "pool-id-1" mock_pool1.members = [mock_member1] mock_listener1 = mock.Mock() mock_listener1.id = 'listener-id-1' mock_listener1.default_pool = mock_pool1 mock_listener1.protocol = constants.PROTOCOL_UDP # The default pool of udp listener2 has no member mock_pool2 = mock.Mock() mock_pool2.id = "pool-id-2" mock_pool2.members = [] mock_listener2 = mock.Mock() mock_listener2.id = 'listener-id-2' mock_listener2.default_pool = mock_pool2 mock_listener2.protocol = constants.PROTOCOL_UDP # The udp listener3 has no default_pool mock_listener3 = mock.Mock() mock_listener3.id = 'listener-id-3' mock_listener3.default_pool = None mock_listener3.protocol = constants.PROTOCOL_UDP mock_lb.listeners.extend([mock_listener1, mock_listener2, mock_listener3]) mock_lb.pools.extend([mock_pool1, mock_pool2]) self.loadbalancer_repo.get.return_value = mock_lb lb_ref = self._make_fake_lb_health_dict( listener_protocol=constants.PROTOCOL_UDP) lb_ref['listeners']['listener-id-2'] = { constants.OPERATING_STATUS: 'bogus', 'protocol': constants.PROTOCOL_UDP, 'enabled': True} lb_ref['listeners']['listener-id-3'] = { constants.OPERATING_STATUS: 'bogus', 'protocol': constants.PROTOCOL_UDP, 'enabled': True} self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.assertTrue(self.amphora_health_repo.replace.called) def test_update_health_no_db_lb(self): health = { "id": self.FAKE_UUID_1, "ver": 1, "listeners": {}, "recv_time": time.time() } self.hm.amphora_repo.get_lb_for_health_update.return_value = {} self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertFalse(self.amphora_health_repo.replace.called) # Test missing amp in addition to missing lb DB record self.amphora_repo.get_lb_for_health_update.reset_mock() self.amphora_health_repo.replace.reset_mock() mock_amphora = mock.MagicMock() mock_amphora.load_balancer_id = None self.amphora_repo.get.return_value = mock_amphora self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.amphora_repo.get.called) self.assertTrue(self.amphora_health_repo.replace.called) def test_update_health_with_without_udp_listeners(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.DOWN} } }}}, "recv_time": time.time() } # Test with a TCP listener lb_ref = self._make_fake_lb_health_dict( listener_protocol=constants.PROTOCOL_TCP) self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') # We should have no calls to listener_repo.get, because we skip # running the extra UDP function self.assertFalse(self.listener_repo.get.called) # Reset the mocks to try again self.listener_repo.reset_mock() # Test with a UDP listener lb_ref = self._make_fake_lb_health_dict( listener_protocol=constants.PROTOCOL_UDP) self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') # This time we should have a call to listener_repo.get because the # UDP helper function is triggered self.assertTrue(self.listener_repo.get.called) def test_update_listener_count_for_UDP(self): mock_lb, mock_listener1, mock_pool1, mock_members = ( self._make_mock_lb_tree()) mock_listener1.protocol = constants.PROTOCOL_TCP self.hm.listener_repo.get.return_value = mock_listener1 # Test only TCP listeners lb_ref = self._make_fake_lb_health_dict( listener_protocol=constants.PROTOCOL_TCP) result = self.hm._update_listener_count_for_UDP( 'bogus_session', lb_ref, 0) self.assertEqual(0, result) # Test with a valid member lb_ref = self._make_fake_lb_health_dict( listener_protocol=constants.PROTOCOL_UDP) mock_listener1.protocol = constants.PROTOCOL_UDP result = self.hm._update_listener_count_for_UDP( 'bogus_session', lb_ref, 1) self.assertEqual(1, result) # Test with a disabled member mock_listener1.protocol = constants.PROTOCOL_UDP mock_members[0].enabled = False result = self.hm._update_listener_count_for_UDP( 'bogus_session', lb_ref, 1) self.assertEqual(0, result) def test_update_status(self): # Test update with the same operating status self.hm._update_status( 'fake_session', self.loadbalancer_repo, constants.LOADBALANCER, 1, 'ONLINE', 'ONLINE') self.assertFalse(self.loadbalancer_repo.update.called) self.loadbalancer_repo.update.reset_mock() # Test stream with provisioning sync self.hm._update_status( 'fake_session', self.loadbalancer_repo, constants.LOADBALANCER, 1, 'ONLINE', 'OFFLINE') self.assertTrue(self.loadbalancer_repo.update.called) class TestUpdateStatsDb(base.TestCase): def setUp(self): super(TestUpdateStatsDb, self).setUp() self.sm = update_db.UpdateStatsDb() self.listener_stats_repo = mock.MagicMock() self.sm.listener_stats_repo = self.listener_stats_repo self.loadbalancer_id = uuidutils.generate_uuid() self.listener_id = uuidutils.generate_uuid() self.listener_stats = data_models.ListenerStatistics( listener_id=self.listener_id, bytes_in=random.randrange(1000000000), bytes_out=random.randrange(1000000000), active_connections=random.randrange(1000000000), total_connections=random.randrange(1000000000), request_errors=random.randrange(1000000000)) self.sm.get_listener_stats = mock.MagicMock() self.sm.get_listener_stats.return_value = self.listener_stats self.loadbalancer_id = uuidutils.generate_uuid() self.amphora_id = uuidutils.generate_uuid() self.listener_id = uuidutils.generate_uuid() self.listener = db_models.Listener( load_balancer_id=self.loadbalancer_id) self.listener_repo = mock.MagicMock() self.sm.repo_listener = self.listener_repo self.sm.repo_listener.get.return_value = self.listener self.loadbalancer_repo = mock.MagicMock() self.sm.repo_loadbalancer = self.loadbalancer_repo self.loadbalancer = db_models.LoadBalancer( id=self.loadbalancer_id, listeners=[self.listener]) self.loadbalancer_repo.get.return_value = self.loadbalancer @mock.patch('octavia.db.api.get_session') def test_update_stats_v1(self, mock_session): health = { "id": self.amphora_id, "ver": 1, "listeners": { self.listener_id: { "status": constants.OPEN, "stats": { "ereq": self.listener_stats.request_errors, "conns": self.listener_stats.active_connections, "totconns": self.listener_stats.total_connections, "rx": self.listener_stats.bytes_in, "tx": self.listener_stats.bytes_out, }, "pools": { "pool-id-1": { "status": constants.UP, "members": {"member-id-1": constants.ONLINE} } } } } } mock_session.return_value = 'blah' self.sm.update_stats(health, '192.0.2.1') self.listener_stats_repo.replace.assert_called_once_with( 'blah', self.listener_id, self.amphora_id, bytes_in=self.listener_stats.bytes_in, bytes_out=self.listener_stats.bytes_out, active_connections=self.listener_stats.active_connections, total_connections=self.listener_stats.total_connections, request_errors=self.listener_stats.request_errors) # Test with update failure mock_session.side_effect = Exception self.sm.update_stats(health, '192.0.2.1') @mock.patch('octavia.db.api.get_session') def test_update_stats_v3(self, mock_session): health = { "id": self.amphora_id, "seq": random.randint(0, 100), "ver": 3, "listeners": { self.listener_id: { "status": constants.OPEN, "stats": { "ereq": self.listener_stats.request_errors, "conns": self.listener_stats.active_connections, "totconns": self.listener_stats.total_connections, "rx": self.listener_stats.bytes_in, "tx": self.listener_stats.bytes_out, }, "pools": { "pool-id-1": { "status": constants.UP, "members": {"member-id-1": constants.ONLINE} } } } } } delta_stats_model = data_models.ListenerStatistics( listener_id=self.listener_id, amphora_id=self.amphora_id, bytes_in=self.listener_stats.bytes_in, bytes_out=self.listener_stats.bytes_out, active_connections=self.listener_stats.active_connections, total_connections=self.listener_stats.total_connections, request_errors=self.listener_stats.request_errors ) mock_session.return_value = 'blah' self.sm.update_stats(health, '192.0.2.1') self.listener_stats_repo.increment.assert_called_once_with( 'blah', delta_stats_model) # Test with update failure mock_session.side_effect = Exception self.sm.update_stats(health, '192.0.2.1')