Optimize update_health process
1:reduce database interaction 2:support more than one LB on an amp Co-Authored-By: Michael Johnson <johnsomor@gmail.com> Change-Id: If4f5f93fb8cfd4db8ab3eb9f0518a16da9068160 story: 2001202 task: 5707 (cherry picked from commit 4bddaf6bf2bc85a882006a006b3619e4ed8a82ea)
This commit is contained in:
wei
Adam Harwell
parent
c49959d2ab
commit
38f076249c
octavia
controller/healthmanager
db
tests/unit/controller/healthmanager
@ -50,24 +50,24 @@ class UpdateHealthDb(object):
|
||||
self.event_streamer.emit(cnt)
|
||||
|
||||
def _update_status_and_emit_event(self, session, repo, entity_type,
|
||||
entity_id, new_op_status):
|
||||
entity = repo.get(session, id=entity_id)
|
||||
entity_id, new_op_status, old_op_status,
|
||||
current_prov_status):
|
||||
message = {}
|
||||
if entity.operating_status.lower() != new_op_status.lower():
|
||||
if old_op_status.lower() != new_op_status.lower():
|
||||
LOG.debug("%s %s status has changed from %s to "
|
||||
"%s. Updating db and sending event.",
|
||||
entity_type, entity_id, entity.operating_status,
|
||||
entity_type, entity_id, old_op_status,
|
||||
new_op_status)
|
||||
repo.update(session, entity_id, operating_status=new_op_status)
|
||||
if new_op_status == constants.DRAINING:
|
||||
new_op_status = constants.ONLINE
|
||||
message.update({constants.OPERATING_STATUS: new_op_status})
|
||||
if self.sync_prv_status:
|
||||
LOG.debug("%s %s provisioning_status %s. Updating db and sending"
|
||||
" event.", entity_type, entity_id,
|
||||
entity.provisioning_status)
|
||||
LOG.debug("%s %s provisioning_status %s. "
|
||||
"Updating db and sending event.",
|
||||
entity_type, entity_id, current_prov_status)
|
||||
message.update(
|
||||
{constants.PROVISIONING_STATUS: entity.provisioning_status})
|
||||
{constants.PROVISIONING_STATUS: current_prov_status})
|
||||
if message:
|
||||
self.emit(entity_type, entity_id, message)
|
||||
|
||||
@ -106,15 +106,16 @@ class UpdateHealthDb(object):
|
||||
|
||||
# We need to see if all of the listeners are reporting in
|
||||
expected_listener_count = 0
|
||||
lbs_on_amp = self.amphora_repo.get_all_lbs_on_amphora(session,
|
||||
health['id'])
|
||||
for lb in lbs_on_amp:
|
||||
listener_count = self.listener_repo.count(session,
|
||||
load_balancer_id=lb.id)
|
||||
expected_listener_count += listener_count
|
||||
|
||||
db_lbs_on_amp = self.amphora_repo.get_all_lbs_on_amphora(session,
|
||||
health['id'])
|
||||
listeners = health['listeners']
|
||||
|
||||
# We need to loop over the lbs here to make sure we update the
|
||||
# amphora_health record as soon as possible to prevent unnecessary
|
||||
# failovers. Unfortunately that means looping over this list twice.
|
||||
for db_lb in db_lbs_on_amp:
|
||||
expected_listener_count += len(db_lb.listeners)
|
||||
|
||||
# Do not update amphora health if the reporting listener count
|
||||
# does not match the expected listener count
|
||||
if len(listeners) == expected_listener_count:
|
||||
@ -135,64 +136,123 @@ class UpdateHealthDb(object):
|
||||
{'id': health['id'], 'found': len(listeners),
|
||||
'expected': expected_listener_count})
|
||||
|
||||
# We got a heartbeat so lb is healthy until proven otherwise
|
||||
# TODO(johnsom) Fix this if we have more than one LB on an amp
|
||||
if lbs_on_amp[0].enabled is False:
|
||||
lb_status = constants.OFFLINE
|
||||
else:
|
||||
lb_status = constants.ONLINE
|
||||
for db_lb in db_lbs_on_amp:
|
||||
|
||||
# update listener and nodes db information
|
||||
for listener_id, listener in listeners.items():
|
||||
processed_pools = []
|
||||
|
||||
listener_status = None
|
||||
# OPEN = HAProxy listener status nbconn < maxconn
|
||||
if listener.get('status') == constants.OPEN:
|
||||
listener_status = constants.ONLINE
|
||||
# FULL = HAProxy listener status not nbconn < maxconn
|
||||
elif listener.get('status') == constants.FULL:
|
||||
listener_status = constants.DEGRADED
|
||||
if lb_status == constants.ONLINE:
|
||||
lb_status = constants.DEGRADED
|
||||
# We got a heartbeat so lb is healthy until proven otherwise
|
||||
if db_lb.enabled is False:
|
||||
lb_status = constants.OFFLINE
|
||||
else:
|
||||
LOG.warning(('Listener %(list)s reported status of '
|
||||
'%(status)s'), {'list': listener_id,
|
||||
'status': listener.get('status')})
|
||||
lb_status = constants.ONLINE
|
||||
|
||||
try:
|
||||
if listener_status is not None:
|
||||
self._update_status_and_emit_event(
|
||||
session, self.listener_repo, constants.LISTENER,
|
||||
listener_id, listener_status
|
||||
)
|
||||
except sqlalchemy.orm.exc.NoResultFound:
|
||||
LOG.error("Listener %s is not in DB", listener_id)
|
||||
for db_listener in db_lb.listeners:
|
||||
listener_status = None
|
||||
listener_id = db_listener.id
|
||||
listener = None
|
||||
|
||||
pools = listener['pools']
|
||||
for pool_id, pool in pools.items():
|
||||
|
||||
pool_status = None
|
||||
# UP = HAProxy backend has working or no servers
|
||||
if pool.get('status') == constants.UP:
|
||||
pool_status = constants.ONLINE
|
||||
# DOWN = HAProxy backend has no working servers
|
||||
elif pool.get('status') == constants.DOWN:
|
||||
pool_status = constants.ERROR
|
||||
lb_status = constants.ERROR
|
||||
if listener_id not in listeners:
|
||||
listener_status = constants.OFFLINE
|
||||
else:
|
||||
LOG.warning(('Pool %(pool)s reported status of '
|
||||
'%(status)s'), {'pool': pool_id,
|
||||
'status': pool.get('status')})
|
||||
listener = listeners[listener_id]
|
||||
|
||||
# Deal with the members that are reporting from the Amphora
|
||||
members = pool['members']
|
||||
for member_id, status in members.items():
|
||||
# OPEN = HAProxy listener status nbconn < maxconn
|
||||
if listener.get('status') == constants.OPEN:
|
||||
listener_status = constants.ONLINE
|
||||
# FULL = HAProxy listener status not nbconn < maxconn
|
||||
elif listener.get('status') == constants.FULL:
|
||||
listener_status = constants.DEGRADED
|
||||
if lb_status == constants.ONLINE:
|
||||
lb_status = constants.DEGRADED
|
||||
else:
|
||||
LOG.warning(('Listener %(list)s reported status of '
|
||||
'%(status)s'),
|
||||
{'list': listener_id,
|
||||
'status': listener.get('status')})
|
||||
|
||||
member_status = None
|
||||
# Member status can be "UP" or "UP #/#" (transitional)
|
||||
try:
|
||||
if listener_status is not None:
|
||||
self._update_status_and_emit_event(
|
||||
session, self.listener_repo, constants.LISTENER,
|
||||
listener_id, listener_status,
|
||||
db_listener.operating_status,
|
||||
db_listener.provisioning_status
|
||||
)
|
||||
except sqlalchemy.orm.exc.NoResultFound:
|
||||
LOG.error("Listener %s is not in DB", listener_id)
|
||||
|
||||
if not listener:
|
||||
continue
|
||||
|
||||
pools = listener['pools']
|
||||
|
||||
# Process pools bound to listeners
|
||||
for db_pool in db_listener.pools:
|
||||
lb_status, processed_pools = self._process_pool_status(
|
||||
session, db_pool, pools, lb_status)
|
||||
|
||||
# Process pools bound to the load balancer
|
||||
for db_pool in db_lb.pools:
|
||||
# Don't re-process pools shared with listeners
|
||||
if db_pool.id in processed_pools:
|
||||
continue
|
||||
lb_status, processed_pools = self._process_pool_status(
|
||||
session, db_pool, pools, lb_status)
|
||||
|
||||
# Update the load balancer status last
|
||||
try:
|
||||
self._update_status_and_emit_event(
|
||||
session, self.loadbalancer_repo,
|
||||
constants.LOADBALANCER, db_lb.id, lb_status,
|
||||
db_lb.operating_status, db_lb.provisioning_status
|
||||
)
|
||||
except sqlalchemy.orm.exc.NoResultFound:
|
||||
LOG.error("Load balancer %s is not in DB", db_lb.id)
|
||||
|
||||
def _process_pool_status(self, session, db_pool, pools, lb_status):
|
||||
pool_status = None
|
||||
pool_id = db_pool.id
|
||||
|
||||
processed_pools = []
|
||||
|
||||
if pool_id not in pools:
|
||||
pool_status = constants.OFFLINE
|
||||
else:
|
||||
pool = pools[pool_id]
|
||||
|
||||
processed_pools.append(pool_id)
|
||||
|
||||
# UP = HAProxy backend has working or no servers
|
||||
if pool.get('status') == constants.UP:
|
||||
pool_status = constants.ONLINE
|
||||
# DOWN = HAProxy backend has no working servers
|
||||
elif pool.get('status') == constants.DOWN:
|
||||
pool_status = constants.ERROR
|
||||
lb_status = constants.ERROR
|
||||
else:
|
||||
LOG.warning(('Pool %(pool)s reported status of '
|
||||
'%(status)s'),
|
||||
{'pool': pool_id,
|
||||
'status': pool.get('status')})
|
||||
|
||||
# Deal with the members that are reporting from
|
||||
# the Amphora
|
||||
members = pool['members']
|
||||
for db_member in db_pool.members:
|
||||
member_status = None
|
||||
member_id = db_member.id
|
||||
|
||||
if member_id not in members:
|
||||
member_status = constants.OFFLINE
|
||||
else:
|
||||
status = members[member_id]
|
||||
|
||||
# Member status can be "UP" or "UP #/#"
|
||||
# (transitional)
|
||||
if status.startswith(constants.UP):
|
||||
member_status = constants.ONLINE
|
||||
# Member status can be "DOWN" or "DOWN #/#" (transitional)
|
||||
# Member status can be "DOWN" or "DOWN #/#"
|
||||
# (transitional)
|
||||
elif status.startswith(constants.DOWN):
|
||||
member_status = constants.ERROR
|
||||
if pool_status == constants.ONLINE:
|
||||
@ -206,53 +266,35 @@ class UpdateHealthDb(object):
|
||||
elif status == constants.NO_CHECK:
|
||||
member_status = constants.NO_MONITOR
|
||||
else:
|
||||
LOG.warning('Member %(mem)s reported status of '
|
||||
'%(status)s', {'mem': member_id,
|
||||
'status': status})
|
||||
|
||||
try:
|
||||
if member_status is not None:
|
||||
self._update_status_and_emit_event(
|
||||
session, self.member_repo, constants.MEMBER,
|
||||
member_id, member_status
|
||||
)
|
||||
except sqlalchemy.orm.exc.NoResultFound:
|
||||
LOG.error("Member %s is not able to update "
|
||||
"in DB", member_id)
|
||||
|
||||
# Now deal with the members that didn't report from the Amphora
|
||||
db_pool = self.pool_repo.get(session, id=pool_id)
|
||||
real_members = [member.id for member in db_pool.members]
|
||||
reported_members = [member for member in members.keys()]
|
||||
missing_members = set(real_members) - set(reported_members)
|
||||
for member_id in missing_members:
|
||||
self._update_status_and_emit_event(
|
||||
session, self.member_repo, constants.MEMBER,
|
||||
member_id, constants.OFFLINE
|
||||
)
|
||||
LOG.warning('Member %(mem)s reported '
|
||||
'status of %(status)s',
|
||||
{'mem': member_id,
|
||||
'status': status})
|
||||
|
||||
try:
|
||||
if pool_status is not None:
|
||||
if member_status is not None:
|
||||
self._update_status_and_emit_event(
|
||||
session, self.pool_repo, constants.POOL,
|
||||
pool_id, pool_status
|
||||
session, self.member_repo,
|
||||
constants.MEMBER,
|
||||
member_id, member_status,
|
||||
db_member.operating_status,
|
||||
db_member.provisioning_status
|
||||
)
|
||||
except sqlalchemy.orm.exc.NoResultFound:
|
||||
LOG.error("Pool %s is not in DB", pool_id)
|
||||
LOG.error("Member %s is not able to update "
|
||||
"in DB", member_id)
|
||||
|
||||
# Update the load balancer status last
|
||||
# TODO(sbalukoff): This logic will need to be adjusted if we
|
||||
# start supporting multiple load balancers per amphora
|
||||
lb_id = self.amphora_repo.get(
|
||||
session, id=health['id']).load_balancer_id
|
||||
if lb_id is not None:
|
||||
try:
|
||||
try:
|
||||
if pool_status is not None:
|
||||
self._update_status_and_emit_event(
|
||||
session, self.loadbalancer_repo,
|
||||
constants.LOADBALANCER, lb_id, lb_status
|
||||
session, self.pool_repo, constants.POOL,
|
||||
pool_id, pool_status, db_pool.operating_status,
|
||||
db_pool.provisioning_status
|
||||
)
|
||||
except sqlalchemy.orm.exc.NoResultFound:
|
||||
LOG.error("Load balancer %s is not in DB", lb_id)
|
||||
except sqlalchemy.orm.exc.NoResultFound:
|
||||
LOG.error("Pool %s is not in DB", pool_id)
|
||||
|
||||
return lb_status, processed_pools
|
||||
|
||||
|
||||
class UpdateStatsDb(stats.StatsMixin):
|
||||
|
||||
@ -891,7 +891,8 @@ class AmphoraRepository(BaseRepository):
|
||||
lb_subquery = (session.query(self.model_class.load_balancer_id).
|
||||
filter_by(id=amphora_id).subquery())
|
||||
lb_list = (session.query(models.LoadBalancer).
|
||||
filter(models.LoadBalancer.id.in_(lb_subquery)).all())
|
||||
filter(models.LoadBalancer.id.in_(lb_subquery)).
|
||||
options(joinedload('*')).all())
|
||||
data_model_list = [model.to_data_model() for model in lb_list]
|
||||
return data_model_list
|
||||
|
||||
|
||||
@ -83,6 +83,22 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
|
||||
self.hm.update_health(health)
|
||||
self.event_client.cast.assert_any_call(
|
||||
{}, 'update_info', container={
|
||||
@ -103,34 +119,30 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
"id": self.FAKE_UUID_1,
|
||||
"listeners": {}}
|
||||
|
||||
lb = mock.MagicMock()
|
||||
lb.operating_status.lower.return_value = 'blah'
|
||||
self.amphora_repo.get.load_balancer_id.return_value = self.FAKE_UUID_1
|
||||
self.loadbalancer_repo.get.return_value = lb
|
||||
fake_lb = mock.MagicMock()
|
||||
fake_lb.enabled = False
|
||||
fake_lb.listeners = []
|
||||
fake_lb.id = self.FAKE_UUID_1
|
||||
fake_lb.operating_status = 'blah'
|
||||
self.hm.amphora_repo.get_all_lbs_on_amphora.return_value = [fake_lb]
|
||||
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_repo.get.called)
|
||||
self.assertTrue(lb.operating_status.lower.called)
|
||||
self.assertTrue(self.amphora_repo.get_all_lbs_on_amphora.called)
|
||||
self.assertTrue(self.loadbalancer_repo.update.called)
|
||||
|
||||
def test_update_health_replace_error(self):
|
||||
|
||||
health = {
|
||||
"id": self.FAKE_UUID_1,
|
||||
"listeners": {
|
||||
"listener-id-1": {"status": constants.OPEN, "pools": {
|
||||
"pool-id-1": {"status": constants.UP,
|
||||
"members": {"member-id-1": constants.UP}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"listeners": {}
|
||||
}
|
||||
|
||||
session_mock = mock.MagicMock()
|
||||
session_mock.commit.side_effect = TestException('boom')
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = []
|
||||
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
session_mock.rollback.assert_called_once()
|
||||
@ -152,6 +164,20 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
@ -174,12 +200,165 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
operating_status=constants.ONLINE)
|
||||
|
||||
# If the listener count is wrong, make sure we don't update
|
||||
self.hm.listener_repo.count.return_value = 2
|
||||
mock_listener2 = mock.Mock()
|
||||
mock_listener2.id = 'listener-id-2'
|
||||
mock_listener2.pools = [mock_pool1]
|
||||
mock_lb.listeners = [mock_listener1, mock_listener2]
|
||||
self.amphora_health_repo.replace.reset_mock()
|
||||
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(not self.amphora_health_repo.replace.called)
|
||||
|
||||
def test_update_lb_pool_health_online(self):
|
||||
|
||||
health = {
|
||||
"id": self.FAKE_UUID_1,
|
||||
"listeners": {
|
||||
"listener-id-1": {"status": constants.OPEN, "pools": {
|
||||
"pool-id-1": {"status": constants.UP,
|
||||
"members": {"member-id-1": constants.UP}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = []
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = [mock_pool1]
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
# test listener, member
|
||||
for listener_id, listener in six.iteritems(
|
||||
health.get('listeners', {})):
|
||||
|
||||
self.listener_repo.update.assert_any_call(
|
||||
session_mock, listener_id, operating_status=constants.ONLINE)
|
||||
|
||||
for pool_id, pool in six.iteritems(listener.get('pools', {})):
|
||||
|
||||
# We should not double process a shared pool
|
||||
self.hm.pool_repo.update.assert_called_once_with(
|
||||
session_mock, pool_id, operating_status=constants.ONLINE)
|
||||
|
||||
for member_id, member in six.iteritems(
|
||||
pool.get('members', {})):
|
||||
self.member_repo.update.assert_any_call(
|
||||
session_mock, member_id,
|
||||
operating_status=constants.ONLINE)
|
||||
|
||||
def test_update_lb_and_list_pool_health_online(self):
|
||||
|
||||
health = {
|
||||
"id": self.FAKE_UUID_1,
|
||||
"listeners": {
|
||||
"listener-id-1": {"status": constants.OPEN, "pools": {
|
||||
"pool-id-1": {"status": constants.UP,
|
||||
"members": {"member-id-1": constants.UP}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = [mock_pool1]
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
# test listener, member
|
||||
for listener_id, listener in six.iteritems(
|
||||
health.get('listeners', {})):
|
||||
|
||||
self.listener_repo.update.assert_any_call(
|
||||
session_mock, listener_id, operating_status=constants.ONLINE)
|
||||
|
||||
for pool_id, pool in six.iteritems(listener.get('pools', {})):
|
||||
|
||||
# We should not double process a shared pool
|
||||
self.hm.pool_repo.update.assert_called_once_with(
|
||||
session_mock, pool_id, operating_status=constants.ONLINE)
|
||||
|
||||
for member_id, member in six.iteritems(
|
||||
pool.get('members', {})):
|
||||
self.member_repo.update.assert_any_call(
|
||||
session_mock, member_id,
|
||||
operating_status=constants.ONLINE)
|
||||
|
||||
def test_update_pool_offline(self):
|
||||
|
||||
health = {
|
||||
"id": self.FAKE_UUID_1,
|
||||
"listeners": {
|
||||
"listener-id-1": {"status": constants.OPEN, "pools": {
|
||||
"pool-id-5": {"status": constants.UP,
|
||||
"members": {"member-id-1": constants.UP}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
# test listener, member
|
||||
for listener_id, listener in six.iteritems(
|
||||
health.get('listeners', {})):
|
||||
|
||||
self.listener_repo.update.assert_any_call(
|
||||
session_mock, listener_id, operating_status=constants.ONLINE)
|
||||
|
||||
self.hm.pool_repo.update.assert_any_call(
|
||||
session_mock, "pool-id-1", operating_status=constants.OFFLINE)
|
||||
|
||||
def test_update_health_member_drain(self):
|
||||
|
||||
health = {
|
||||
@ -195,6 +374,20 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
@ -232,6 +425,20 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
@ -269,6 +476,20 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
@ -302,6 +523,21 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
@ -342,6 +578,20 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
@ -380,15 +630,24 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_pool = mock.Mock()
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_member2 = mock.Mock()
|
||||
mock_member2.id = 'member-id-2'
|
||||
mock_pool.members = [mock_member1, mock_member2]
|
||||
self.pool_repo.get.return_value = mock_pool
|
||||
mock_pool1.members = [mock_member1, mock_member2]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
self.assertTrue(self.amphora_repo.get_all_lbs_on_amphora.called)
|
||||
|
||||
# test listener, member
|
||||
for listener_id, listener in six.iteritems(
|
||||
@ -428,6 +687,20 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
@ -450,7 +723,10 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
session_mock, member_id,
|
||||
operating_status=constants.ERROR)
|
||||
|
||||
self.hm.listener_repo.count.return_value = 2
|
||||
mock_listener2 = mock.Mock()
|
||||
mock_listener2.id = 'listener-id-2'
|
||||
mock_listener2.pools = [mock_pool1]
|
||||
mock_lb.listeners.append(mock_listener2)
|
||||
self.amphora_health_repo.replace.reset_mock()
|
||||
|
||||
self.hm.update_health(health)
|
||||
@ -473,6 +749,21 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
@ -541,23 +832,46 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
}
|
||||
}
|
||||
|
||||
self.mock_session.return_value = 'blah'
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = []
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
for i in [1, 2, 3, 4, 5]:
|
||||
mock_member = mock.Mock()
|
||||
mock_member.id = 'member-id-%s' % i
|
||||
mock_pool = mock.Mock()
|
||||
mock_pool.id = 'pool-id-%s' % i
|
||||
mock_pool.members = [mock_member]
|
||||
if i == 3:
|
||||
mock_member = mock.Mock()
|
||||
mock_member.id = 'member-id-31'
|
||||
mock_pool.members.append(mock_member)
|
||||
mock_listener = mock.Mock()
|
||||
mock_listener.id = 'listener-id-%s' % i
|
||||
mock_listener.pools = [mock_pool]
|
||||
mock_lb.listeners.append(mock_listener)
|
||||
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
|
||||
self.hm.update_health(health)
|
||||
|
||||
# test listener
|
||||
self.listener_repo.update.assert_any_call(
|
||||
'blah', "listener-id-1", operating_status=constants.DEGRADED)
|
||||
session_mock, "listener-id-1", operating_status=constants.DEGRADED)
|
||||
self.listener_repo.update.assert_any_call(
|
||||
'blah', "listener-id-2", operating_status=constants.DEGRADED)
|
||||
session_mock, "listener-id-2", operating_status=constants.DEGRADED)
|
||||
self.pool_repo.update.assert_any_call(
|
||||
'blah', "pool-id-1", operating_status=constants.ERROR)
|
||||
session_mock, "pool-id-1", operating_status=constants.ERROR)
|
||||
self.pool_repo.update.assert_any_call(
|
||||
'blah', "pool-id-2", operating_status=constants.ONLINE)
|
||||
session_mock, "pool-id-2", operating_status=constants.ONLINE)
|
||||
self.pool_repo.update.assert_any_call(
|
||||
'blah', "pool-id-3", operating_status=constants.DEGRADED)
|
||||
session_mock, "pool-id-3", operating_status=constants.DEGRADED)
|
||||
self.pool_repo.update.assert_any_call(
|
||||
'blah', "pool-id-4", operating_status=constants.ONLINE)
|
||||
session_mock, "pool-id-4", operating_status=constants.ONLINE)
|
||||
|
||||
# Test code paths where objects are not found in the database
|
||||
def test_update_health_not_found(self):
|
||||
@ -586,6 +900,21 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
session_mock = mock.MagicMock()
|
||||
self.mock_session.return_value = session_mock
|
||||
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = 'blah'
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_health_repo.replace.called)
|
||||
|
||||
@ -623,23 +952,25 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
}
|
||||
}
|
||||
}
|
||||
db_lb = data_models.LoadBalancer(
|
||||
id=self.FAKE_UUID_1, operating_status=constants.ONLINE
|
||||
)
|
||||
db_listener = data_models.Listener(
|
||||
id='listener-id-', operating_status=constants.ONLINE,
|
||||
load_balancer_id=self.FAKE_UUID_1
|
||||
)
|
||||
db_pool = data_models.Pool(
|
||||
id='pool-id-1', operating_status=constants.ONLINE
|
||||
)
|
||||
db_member = data_models.Member(
|
||||
id='member-id-1', operating_status=constants.ONLINE
|
||||
)
|
||||
self.listener_repo.get.return_value = db_listener
|
||||
self.pool_repo.get.return_value = db_pool
|
||||
self.member_repo.get.return_value = db_member
|
||||
self.loadbalancer_repo.get.return_value = db_lb
|
||||
|
||||
mock_pool1 = mock.Mock()
|
||||
mock_pool1.id = "pool-id-1"
|
||||
mock_pool1.operating_status = constants.ONLINE
|
||||
mock_member1 = mock.Mock()
|
||||
mock_member1.id = 'member-id-1'
|
||||
mock_member1.operating_status = constants.ONLINE
|
||||
mock_pool1.members = [mock_member1]
|
||||
mock_listener1 = mock.Mock()
|
||||
mock_listener1.id = 'listener-id-1'
|
||||
mock_listener1.pools = [mock_pool1]
|
||||
mock_listener1.operating_status = constants.ONLINE
|
||||
mock_lb = mock.Mock()
|
||||
mock_lb.listeners = [mock_listener1]
|
||||
mock_lb.id = self.FAKE_UUID_1
|
||||
mock_lb.operating_status = constants.ONLINE
|
||||
mock_lb.pools = []
|
||||
self.amphora_repo.get_all_lbs_on_amphora.return_value = [mock_lb]
|
||||
|
||||
self.hm.update_health(health)
|
||||
self.event_client.cast.assert_not_called()
|
||||
self.loadbalancer_repo.update.assert_not_called()
|
||||
@ -655,20 +986,17 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
|
||||
fake_lb = mock.MagicMock()
|
||||
fake_lb.enabled = False
|
||||
fake_lb.listeners = []
|
||||
fake_lb.id = self.FAKE_UUID_1
|
||||
fake_lb.operating_status = 'blah'
|
||||
self.hm.amphora_repo.get_all_lbs_on_amphora.return_value = [fake_lb]
|
||||
|
||||
lb = mock.MagicMock()
|
||||
lb.operating_status.lower.return_value = 'blah'
|
||||
self.amphora_repo.get.load_balancer_id.return_value = self.FAKE_UUID_1
|
||||
self.loadbalancer_repo.get.return_value = lb
|
||||
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_repo.get.called)
|
||||
self.assertTrue(lb.operating_status.lower.called)
|
||||
self.assertTrue(self.amphora_repo.get_all_lbs_on_amphora.called)
|
||||
self.assertTrue(self.loadbalancer_repo.update.called)
|
||||
self.loadbalancer_repo.update.assert_called_with(
|
||||
self.mock_session(),
|
||||
self.amphora_repo.get().load_balancer_id,
|
||||
fake_lb.id,
|
||||
operating_status='OFFLINE')
|
||||
|
||||
def test_update_health_lb_admin_up(self):
|
||||
@ -679,20 +1007,17 @@ class TestUpdateHealthDb(base.TestCase):
|
||||
|
||||
fake_lb = mock.MagicMock()
|
||||
fake_lb.enabled = True
|
||||
fake_lb.listeners = []
|
||||
fake_lb.id = self.FAKE_UUID_1
|
||||
fake_lb.operating_status = 'blah'
|
||||
self.hm.amphora_repo.get_all_lbs_on_amphora.return_value = [fake_lb]
|
||||
|
||||
lb = mock.MagicMock()
|
||||
lb.operating_status.lower.return_value = 'blah'
|
||||
self.amphora_repo.get.load_balancer_id.return_value = self.FAKE_UUID_1
|
||||
self.loadbalancer_repo.get.return_value = lb
|
||||
|
||||
self.hm.update_health(health)
|
||||
self.assertTrue(self.amphora_repo.get.called)
|
||||
self.assertTrue(lb.operating_status.lower.called)
|
||||
self.assertTrue(self.amphora_repo.get_all_lbs_on_amphora.called)
|
||||
self.assertTrue(self.loadbalancer_repo.update.called)
|
||||
self.loadbalancer_repo.update.assert_called_with(
|
||||
self.mock_session(),
|
||||
self.amphora_repo.get().load_balancer_id,
|
||||
fake_lb.id,
|
||||
operating_status='ONLINE')
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user