diff --git a/octavia/controller/healthmanager/update_db.py b/octavia/controller/healthmanager/update_db.py index 09486c271b..416e885d26 100644 --- a/octavia/controller/healthmanager/update_db.py +++ b/octavia/controller/healthmanager/update_db.py @@ -50,24 +50,24 @@ class UpdateHealthDb(object): self.event_streamer.emit(cnt) def _update_status_and_emit_event(self, session, repo, entity_type, - entity_id, new_op_status): - entity = repo.get(session, id=entity_id) + entity_id, new_op_status, old_op_status, + current_prov_status): message = {} - if entity.operating_status.lower() != new_op_status.lower(): + if old_op_status.lower() != new_op_status.lower(): LOG.debug("%s %s status has changed from %s to " "%s. Updating db and sending event.", - entity_type, entity_id, entity.operating_status, + entity_type, entity_id, old_op_status, new_op_status) repo.update(session, entity_id, operating_status=new_op_status) if new_op_status == constants.DRAINING: new_op_status = constants.ONLINE message.update({constants.OPERATING_STATUS: new_op_status}) if self.sync_prv_status: - LOG.debug("%s %s provisioning_status %s. Updating db and sending" - " event.", entity_type, entity_id, - entity.provisioning_status) + LOG.debug("%s %s provisioning_status %s. " + "Updating db and sending event.", + entity_type, entity_id, current_prov_status) message.update( - {constants.PROVISIONING_STATUS: entity.provisioning_status}) + {constants.PROVISIONING_STATUS: current_prov_status}) if message: self.emit(entity_type, entity_id, message) @@ -98,15 +98,16 @@ class UpdateHealthDb(object): # We need to see if all of the listeners are reporting in expected_listener_count = 0 - lbs_on_amp = self.amphora_repo.get_all_lbs_on_amphora(session, - health['id']) - for lb in lbs_on_amp: - listener_count = self.listener_repo.count(session, - load_balancer_id=lb.id) - expected_listener_count += listener_count - + db_lbs_on_amp = self.amphora_repo.get_all_lbs_on_amphora(session, + health['id']) listeners = health['listeners'] + # We need to loop over the lbs here to make sure we update the + # amphora_health record as soon as possible to prevent unnecessary + # failovers. Unfortunately that means looping over this list twice. + for db_lb in db_lbs_on_amp: + expected_listener_count += len(db_lb.listeners) + # Do not update amphora health if the reporting listener count # does not match the expected listener count if len(listeners) == expected_listener_count: @@ -127,64 +128,123 @@ class UpdateHealthDb(object): {'id': health['id'], 'found': len(listeners), 'expected': expected_listener_count}) - # We got a heartbeat so lb is healthy until proven otherwise - # TODO(johnsom) Fix this if we have more than one LB on an amp - if lbs_on_amp[0].enabled is False: - lb_status = constants.OFFLINE - else: - lb_status = constants.ONLINE + for db_lb in db_lbs_on_amp: - # update listener and nodes db information - for listener_id, listener in listeners.items(): + processed_pools = [] - listener_status = None - # OPEN = HAProxy listener status nbconn < maxconn - if listener.get('status') == constants.OPEN: - listener_status = constants.ONLINE - # FULL = HAProxy listener status not nbconn < maxconn - elif listener.get('status') == constants.FULL: - listener_status = constants.DEGRADED - if lb_status == constants.ONLINE: - lb_status = constants.DEGRADED + # We got a heartbeat so lb is healthy until proven otherwise + if db_lb.enabled is False: + lb_status = constants.OFFLINE else: - LOG.warning(('Listener %(list)s reported status of ' - '%(status)s'), {'list': listener_id, - 'status': listener.get('status')}) + lb_status = constants.ONLINE - try: - if listener_status is not None: - self._update_status_and_emit_event( - session, self.listener_repo, constants.LISTENER, - listener_id, listener_status - ) - except sqlalchemy.orm.exc.NoResultFound: - LOG.error("Listener %s is not in DB", listener_id) + for db_listener in db_lb.listeners: + listener_status = None + listener_id = db_listener.id + listener = None - pools = listener['pools'] - for pool_id, pool in pools.items(): - - pool_status = None - # UP = HAProxy backend has working or no servers - if pool.get('status') == constants.UP: - pool_status = constants.ONLINE - # DOWN = HAProxy backend has no working servers - elif pool.get('status') == constants.DOWN: - pool_status = constants.ERROR - lb_status = constants.ERROR + if listener_id not in listeners: + listener_status = constants.OFFLINE else: - LOG.warning(('Pool %(pool)s reported status of ' - '%(status)s'), {'pool': pool_id, - 'status': pool.get('status')}) + listener = listeners[listener_id] - # Deal with the members that are reporting from the Amphora - members = pool['members'] - for member_id, status in members.items(): + # OPEN = HAProxy listener status nbconn < maxconn + if listener.get('status') == constants.OPEN: + listener_status = constants.ONLINE + # FULL = HAProxy listener status not nbconn < maxconn + elif listener.get('status') == constants.FULL: + listener_status = constants.DEGRADED + if lb_status == constants.ONLINE: + lb_status = constants.DEGRADED + else: + LOG.warning(('Listener %(list)s reported status of ' + '%(status)s'), + {'list': listener_id, + 'status': listener.get('status')}) - member_status = None - # Member status can be "UP" or "UP #/#" (transitional) + try: + if listener_status is not None: + self._update_status_and_emit_event( + session, self.listener_repo, constants.LISTENER, + listener_id, listener_status, + db_listener.operating_status, + db_listener.provisioning_status + ) + except sqlalchemy.orm.exc.NoResultFound: + LOG.error("Listener %s is not in DB", listener_id) + + if not listener: + continue + + pools = listener['pools'] + + # Process pools bound to listeners + for db_pool in db_listener.pools: + lb_status, processed_pools = self._process_pool_status( + session, db_pool, pools, lb_status) + + # Process pools bound to the load balancer + for db_pool in db_lb.pools: + # Don't re-process pools shared with listeners + if db_pool.id in processed_pools: + continue + lb_status, processed_pools = self._process_pool_status( + session, db_pool, pools, lb_status) + + # Update the load balancer status last + try: + self._update_status_and_emit_event( + session, self.loadbalancer_repo, + constants.LOADBALANCER, db_lb.id, lb_status, + db_lb.operating_status, db_lb.provisioning_status + ) + except sqlalchemy.orm.exc.NoResultFound: + LOG.error("Load balancer %s is not in DB", db_lb.id) + + def _process_pool_status(self, session, db_pool, pools, lb_status): + pool_status = None + pool_id = db_pool.id + + processed_pools = [] + + if pool_id not in pools: + pool_status = constants.OFFLINE + else: + pool = pools[pool_id] + + processed_pools.append(pool_id) + + # UP = HAProxy backend has working or no servers + if pool.get('status') == constants.UP: + pool_status = constants.ONLINE + # DOWN = HAProxy backend has no working servers + elif pool.get('status') == constants.DOWN: + pool_status = constants.ERROR + lb_status = constants.ERROR + else: + LOG.warning(('Pool %(pool)s reported status of ' + '%(status)s'), + {'pool': pool_id, + 'status': pool.get('status')}) + + # Deal with the members that are reporting from + # the Amphora + members = pool['members'] + for db_member in db_pool.members: + member_status = None + member_id = db_member.id + + if member_id not in members: + member_status = constants.OFFLINE + else: + status = members[member_id] + + # Member status can be "UP" or "UP #/#" + # (transitional) if status.startswith(constants.UP): member_status = constants.ONLINE - # Member status can be "DOWN" or "DOWN #/#" (transitional) + # Member status can be "DOWN" or "DOWN #/#" + # (transitional) elif status.startswith(constants.DOWN): member_status = constants.ERROR if pool_status == constants.ONLINE: @@ -198,53 +258,35 @@ class UpdateHealthDb(object): elif status == constants.NO_CHECK: member_status = constants.NO_MONITOR else: - LOG.warning('Member %(mem)s reported status of ' - '%(status)s', {'mem': member_id, - 'status': status}) - - try: - if member_status is not None: - self._update_status_and_emit_event( - session, self.member_repo, constants.MEMBER, - member_id, member_status - ) - except sqlalchemy.orm.exc.NoResultFound: - LOG.error("Member %s is not able to update " - "in DB", member_id) - - # Now deal with the members that didn't report from the Amphora - db_pool = self.pool_repo.get(session, id=pool_id) - real_members = [member.id for member in db_pool.members] - reported_members = [member for member in members.keys()] - missing_members = set(real_members) - set(reported_members) - for member_id in missing_members: - self._update_status_and_emit_event( - session, self.member_repo, constants.MEMBER, - member_id, constants.OFFLINE - ) + LOG.warning('Member %(mem)s reported ' + 'status of %(status)s', + {'mem': member_id, + 'status': status}) try: - if pool_status is not None: + if member_status is not None: self._update_status_and_emit_event( - session, self.pool_repo, constants.POOL, - pool_id, pool_status + session, self.member_repo, + constants.MEMBER, + member_id, member_status, + db_member.operating_status, + db_member.provisioning_status ) except sqlalchemy.orm.exc.NoResultFound: - LOG.error("Pool %s is not in DB", pool_id) + LOG.error("Member %s is not able to update " + "in DB", member_id) - # Update the load balancer status last - # TODO(sbalukoff): This logic will need to be adjusted if we - # start supporting multiple load balancers per amphora - lb_id = self.amphora_repo.get( - session, id=health['id']).load_balancer_id - if lb_id is not None: - try: + try: + if pool_status is not None: self._update_status_and_emit_event( - session, self.loadbalancer_repo, - constants.LOADBALANCER, lb_id, lb_status + session, self.pool_repo, constants.POOL, + pool_id, pool_status, db_pool.operating_status, + db_pool.provisioning_status ) - except sqlalchemy.orm.exc.NoResultFound: - LOG.error("Load balancer %s is not in DB", lb_id) + except sqlalchemy.orm.exc.NoResultFound: + LOG.error("Pool %s is not in DB", pool_id) + + return lb_status, processed_pools class UpdateStatsDb(stats.StatsMixin): diff --git a/octavia/db/repositories.py b/octavia/db/repositories.py index 1f6fe70f73..b965d7b5f8 100644 --- a/octavia/db/repositories.py +++ b/octavia/db/repositories.py @@ -908,7 +908,8 @@ class AmphoraRepository(BaseRepository): lb_subquery = (session.query(self.model_class.load_balancer_id). filter_by(id=amphora_id).subquery()) lb_list = (session.query(models.LoadBalancer). - filter(models.LoadBalancer.id.in_(lb_subquery)).all()) + filter(models.LoadBalancer.id.in_(lb_subquery)). + options(joinedload('*')).all()) data_model_list = [model.to_data_model() for model in lb_list] return data_model_list diff --git a/octavia/tests/unit/controller/healthmanager/test_update_db.py b/octavia/tests/unit/controller/healthmanager/test_update_db.py index 9dc2e63cfc..054da89962 100644 --- a/octavia/tests/unit/controller/healthmanager/test_update_db.py +++ b/octavia/tests/unit/controller/healthmanager/test_update_db.py @@ -83,6 +83,22 @@ class TestUpdateHealthDb(base.TestCase): } } } + + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] + self.hm.update_health(health) self.event_client.cast.assert_any_call( {}, 'update_info', container={ @@ -103,34 +119,30 @@ class TestUpdateHealthDb(base.TestCase): "id": self.FAKE_UUID_1, "listeners": {}} - lb = mock.MagicMock() - lb.operating_status.lower.return_value = 'blah' - self.amphora_repo.get.load_balancer_id.return_value = self.FAKE_UUID_1 - self.loadbalancer_repo.get.return_value = lb + fake_lb = mock.MagicMock() + fake_lb.enabled = False + fake_lb.listeners = [] + fake_lb.id = self.FAKE_UUID_1 + fake_lb.operating_status = 'blah' + self.hm.amphora_repo.get_all_lbs_on_amphora.return_value = [fake_lb] self.hm.update_health(health) - self.assertTrue(self.amphora_repo.get.called) - self.assertTrue(lb.operating_status.lower.called) + self.assertTrue(self.amphora_repo.get_all_lbs_on_amphora.called) self.assertTrue(self.loadbalancer_repo.update.called) def test_update_health_replace_error(self): health = { "id": self.FAKE_UUID_1, - "listeners": { - "listener-id-1": {"status": constants.OPEN, "pools": { - "pool-id-1": {"status": constants.UP, - "members": {"member-id-1": constants.UP} - } - } - } - } + "listeners": {} } session_mock = mock.MagicMock() session_mock.commit.side_effect = TestException('boom') self.mock_session.return_value = session_mock + self.amphora_repo.get_all_lbs_on_amphora.return_value = [] + self.assertRaises(TestException, self.hm.update_health, health) self.assertTrue(self.amphora_health_repo.replace.called) session_mock.rollback.assert_called_once() @@ -152,6 +164,20 @@ class TestUpdateHealthDb(base.TestCase): session_mock = mock.MagicMock() self.mock_session.return_value = session_mock + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] self.hm.update_health(health) self.assertTrue(self.amphora_health_repo.replace.called) @@ -174,12 +200,165 @@ class TestUpdateHealthDb(base.TestCase): operating_status=constants.ONLINE) # If the listener count is wrong, make sure we don't update - self.hm.listener_repo.count.return_value = 2 + mock_listener2 = mock.Mock() + mock_listener2.id = 'listener-id-2' + mock_listener2.pools = [mock_pool1] + mock_lb.listeners = [mock_listener1, mock_listener2] self.amphora_health_repo.replace.reset_mock() self.hm.update_health(health) self.assertTrue(not self.amphora_health_repo.replace.called) + def test_update_lb_pool_health_online(self): + + health = { + "id": self.FAKE_UUID_1, + "listeners": { + "listener-id-1": {"status": constants.OPEN, "pools": { + "pool-id-1": {"status": constants.UP, + "members": {"member-id-1": constants.UP} + } + } + } + } + } + + session_mock = mock.MagicMock() + self.mock_session.return_value = session_mock + + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [mock_pool1] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] + self.hm.update_health(health) + self.assertTrue(self.amphora_health_repo.replace.called) + + # test listener, member + for listener_id, listener in six.iteritems( + health.get('listeners', {})): + + self.listener_repo.update.assert_any_call( + session_mock, listener_id, operating_status=constants.ONLINE) + + for pool_id, pool in six.iteritems(listener.get('pools', {})): + + # We should not double process a shared pool + self.hm.pool_repo.update.assert_called_once_with( + session_mock, pool_id, operating_status=constants.ONLINE) + + for member_id, member in six.iteritems( + pool.get('members', {})): + self.member_repo.update.assert_any_call( + session_mock, member_id, + operating_status=constants.ONLINE) + + def test_update_lb_and_list_pool_health_online(self): + + health = { + "id": self.FAKE_UUID_1, + "listeners": { + "listener-id-1": {"status": constants.OPEN, "pools": { + "pool-id-1": {"status": constants.UP, + "members": {"member-id-1": constants.UP} + } + } + } + } + } + + session_mock = mock.MagicMock() + self.mock_session.return_value = session_mock + + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [mock_pool1] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] + self.hm.update_health(health) + self.assertTrue(self.amphora_health_repo.replace.called) + + # test listener, member + for listener_id, listener in six.iteritems( + health.get('listeners', {})): + + self.listener_repo.update.assert_any_call( + session_mock, listener_id, operating_status=constants.ONLINE) + + for pool_id, pool in six.iteritems(listener.get('pools', {})): + + # We should not double process a shared pool + self.hm.pool_repo.update.assert_called_once_with( + session_mock, pool_id, operating_status=constants.ONLINE) + + for member_id, member in six.iteritems( + pool.get('members', {})): + self.member_repo.update.assert_any_call( + session_mock, member_id, + operating_status=constants.ONLINE) + + def test_update_pool_offline(self): + + health = { + "id": self.FAKE_UUID_1, + "listeners": { + "listener-id-1": {"status": constants.OPEN, "pools": { + "pool-id-5": {"status": constants.UP, + "members": {"member-id-1": constants.UP} + } + } + } + } + } + + session_mock = mock.MagicMock() + self.mock_session.return_value = session_mock + + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] + self.hm.update_health(health) + self.assertTrue(self.amphora_health_repo.replace.called) + + # test listener, member + for listener_id, listener in six.iteritems( + health.get('listeners', {})): + + self.listener_repo.update.assert_any_call( + session_mock, listener_id, operating_status=constants.ONLINE) + + self.hm.pool_repo.update.assert_any_call( + session_mock, "pool-id-1", operating_status=constants.OFFLINE) + def test_update_health_member_drain(self): health = { @@ -195,6 +374,20 @@ class TestUpdateHealthDb(base.TestCase): session_mock = mock.MagicMock() self.mock_session.return_value = session_mock + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] self.hm.update_health(health) self.assertTrue(self.amphora_health_repo.replace.called) @@ -232,6 +425,20 @@ class TestUpdateHealthDb(base.TestCase): session_mock = mock.MagicMock() self.mock_session.return_value = session_mock + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] self.hm.update_health(health) self.assertTrue(self.amphora_health_repo.replace.called) @@ -269,6 +476,20 @@ class TestUpdateHealthDb(base.TestCase): session_mock = mock.MagicMock() self.mock_session.return_value = session_mock + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] self.hm.update_health(health) self.assertTrue(self.amphora_health_repo.replace.called) @@ -302,6 +523,21 @@ class TestUpdateHealthDb(base.TestCase): session_mock = mock.MagicMock() self.mock_session.return_value = session_mock + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] + self.hm.update_health(health) self.assertTrue(self.amphora_health_repo.replace.called) @@ -342,6 +578,20 @@ class TestUpdateHealthDb(base.TestCase): session_mock = mock.MagicMock() self.mock_session.return_value = session_mock + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] self.hm.update_health(health) self.assertTrue(self.amphora_health_repo.replace.called) @@ -380,15 +630,24 @@ class TestUpdateHealthDb(base.TestCase): session_mock = mock.MagicMock() self.mock_session.return_value = session_mock - mock_pool = mock.Mock() + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" mock_member1 = mock.Mock() mock_member1.id = 'member-id-1' mock_member2 = mock.Mock() mock_member2.id = 'member-id-2' - mock_pool.members = [mock_member1, mock_member2] - self.pool_repo.get.return_value = mock_pool + mock_pool1.members = [mock_member1, mock_member2] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] self.hm.update_health(health) - self.assertTrue(self.amphora_health_repo.replace.called) + self.assertTrue(self.amphora_repo.get_all_lbs_on_amphora.called) # test listener, member for listener_id, listener in six.iteritems( @@ -428,6 +687,20 @@ class TestUpdateHealthDb(base.TestCase): session_mock = mock.MagicMock() self.mock_session.return_value = session_mock + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] self.hm.update_health(health) self.assertTrue(self.amphora_health_repo.replace.called) @@ -450,7 +723,10 @@ class TestUpdateHealthDb(base.TestCase): session_mock, member_id, operating_status=constants.ERROR) - self.hm.listener_repo.count.return_value = 2 + mock_listener2 = mock.Mock() + mock_listener2.id = 'listener-id-2' + mock_listener2.pools = [mock_pool1] + mock_lb.listeners.append(mock_listener2) self.amphora_health_repo.replace.reset_mock() self.hm.update_health(health) @@ -473,6 +749,21 @@ class TestUpdateHealthDb(base.TestCase): session_mock = mock.MagicMock() self.mock_session.return_value = session_mock + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] + self.hm.update_health(health) self.assertTrue(self.amphora_health_repo.replace.called) @@ -541,23 +832,46 @@ class TestUpdateHealthDb(base.TestCase): } } - self.mock_session.return_value = 'blah' + session_mock = mock.MagicMock() + self.mock_session.return_value = session_mock + + mock_lb = mock.Mock() + mock_lb.listeners = [] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + for i in [1, 2, 3, 4, 5]: + mock_member = mock.Mock() + mock_member.id = 'member-id-%s' % i + mock_pool = mock.Mock() + mock_pool.id = 'pool-id-%s' % i + mock_pool.members = [mock_member] + if i == 3: + mock_member = mock.Mock() + mock_member.id = 'member-id-31' + mock_pool.members.append(mock_member) + mock_listener = mock.Mock() + mock_listener.id = 'listener-id-%s' % i + mock_listener.pools = [mock_pool] + mock_lb.listeners.append(mock_listener) + + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] self.hm.update_health(health) # test listener self.listener_repo.update.assert_any_call( - 'blah', "listener-id-1", operating_status=constants.DEGRADED) + session_mock, "listener-id-1", operating_status=constants.DEGRADED) self.listener_repo.update.assert_any_call( - 'blah', "listener-id-2", operating_status=constants.DEGRADED) + session_mock, "listener-id-2", operating_status=constants.DEGRADED) self.pool_repo.update.assert_any_call( - 'blah', "pool-id-1", operating_status=constants.ERROR) + session_mock, "pool-id-1", operating_status=constants.ERROR) self.pool_repo.update.assert_any_call( - 'blah', "pool-id-2", operating_status=constants.ONLINE) + session_mock, "pool-id-2", operating_status=constants.ONLINE) self.pool_repo.update.assert_any_call( - 'blah', "pool-id-3", operating_status=constants.DEGRADED) + session_mock, "pool-id-3", operating_status=constants.DEGRADED) self.pool_repo.update.assert_any_call( - 'blah', "pool-id-4", operating_status=constants.ONLINE) + session_mock, "pool-id-4", operating_status=constants.ONLINE) # Test code paths where objects are not found in the database def test_update_health_not_found(self): @@ -586,6 +900,21 @@ class TestUpdateHealthDb(base.TestCase): session_mock = mock.MagicMock() self.mock_session.return_value = session_mock + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = 'blah' + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] + self.hm.update_health(health) self.assertTrue(self.amphora_health_repo.replace.called) @@ -623,23 +952,25 @@ class TestUpdateHealthDb(base.TestCase): } } } - db_lb = data_models.LoadBalancer( - id=self.FAKE_UUID_1, operating_status=constants.ONLINE - ) - db_listener = data_models.Listener( - id='listener-id-', operating_status=constants.ONLINE, - load_balancer_id=self.FAKE_UUID_1 - ) - db_pool = data_models.Pool( - id='pool-id-1', operating_status=constants.ONLINE - ) - db_member = data_models.Member( - id='member-id-1', operating_status=constants.ONLINE - ) - self.listener_repo.get.return_value = db_listener - self.pool_repo.get.return_value = db_pool - self.member_repo.get.return_value = db_member - self.loadbalancer_repo.get.return_value = db_lb + + mock_pool1 = mock.Mock() + mock_pool1.id = "pool-id-1" + mock_pool1.operating_status = constants.ONLINE + mock_member1 = mock.Mock() + mock_member1.id = 'member-id-1' + mock_member1.operating_status = constants.ONLINE + mock_pool1.members = [mock_member1] + mock_listener1 = mock.Mock() + mock_listener1.id = 'listener-id-1' + mock_listener1.pools = [mock_pool1] + mock_listener1.operating_status = constants.ONLINE + mock_lb = mock.Mock() + mock_lb.listeners = [mock_listener1] + mock_lb.id = self.FAKE_UUID_1 + mock_lb.operating_status = constants.ONLINE + mock_lb.pools = [] + self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb] + self.hm.update_health(health) self.event_client.cast.assert_not_called() self.loadbalancer_repo.update.assert_not_called() @@ -655,20 +986,17 @@ class TestUpdateHealthDb(base.TestCase): fake_lb = mock.MagicMock() fake_lb.enabled = False + fake_lb.listeners = [] + fake_lb.id = self.FAKE_UUID_1 + fake_lb.operating_status = 'blah' self.hm.amphora_repo.get_all_lbs_on_amphora.return_value = [fake_lb] - lb = mock.MagicMock() - lb.operating_status.lower.return_value = 'blah' - self.amphora_repo.get.load_balancer_id.return_value = self.FAKE_UUID_1 - self.loadbalancer_repo.get.return_value = lb - self.hm.update_health(health) - self.assertTrue(self.amphora_repo.get.called) - self.assertTrue(lb.operating_status.lower.called) + self.assertTrue(self.amphora_repo.get_all_lbs_on_amphora.called) self.assertTrue(self.loadbalancer_repo.update.called) self.loadbalancer_repo.update.assert_called_with( self.mock_session(), - self.amphora_repo.get().load_balancer_id, + fake_lb.id, operating_status='OFFLINE') def test_update_health_lb_admin_up(self): @@ -679,20 +1007,17 @@ class TestUpdateHealthDb(base.TestCase): fake_lb = mock.MagicMock() fake_lb.enabled = True + fake_lb.listeners = [] + fake_lb.id = self.FAKE_UUID_1 + fake_lb.operating_status = 'blah' self.hm.amphora_repo.get_all_lbs_on_amphora.return_value = [fake_lb] - lb = mock.MagicMock() - lb.operating_status.lower.return_value = 'blah' - self.amphora_repo.get.load_balancer_id.return_value = self.FAKE_UUID_1 - self.loadbalancer_repo.get.return_value = lb - self.hm.update_health(health) - self.assertTrue(self.amphora_repo.get.called) - self.assertTrue(lb.operating_status.lower.called) + self.assertTrue(self.amphora_repo.get_all_lbs_on_amphora.called) self.assertTrue(self.loadbalancer_repo.update.called) self.loadbalancer_repo.update.assert_called_with( self.mock_session(), - self.amphora_repo.get().load_balancer_id, + fake_lb.id, operating_status='ONLINE')