0b1723ed77
There's a lot of methods implementing the common watcher tools that are inherited from all drivers whereas there is no need for that. The mixin is actually not correctly designed, as it mixes the hooks and the cache code. This splits out the common implementation into its common class that can be inherited to implement this model of driver. The Kazoo and Zake drivers are also merged to simplify code, as the base class was not used. Change-Id: I1acbf1983165d00e68b96cbca2bc5c1e2acadfa8
959 lines
38 KiB
Python
959 lines
38 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright © 2013-2015 eNovance Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import os
|
|
import threading
|
|
import time
|
|
|
|
from concurrent import futures
|
|
import fixtures
|
|
import mock
|
|
from six.moves.urllib import parse
|
|
from testtools import matchers
|
|
from testtools import testcase
|
|
|
|
import tooz.coordination
|
|
from tooz import tests
|
|
|
|
|
|
def try_to_lock_job(name, coord, url, member_id):
|
|
if not coord:
|
|
coord = tooz.coordination.get_coordinator(
|
|
url, member_id)
|
|
coord.start()
|
|
lock2 = coord.get_lock(name)
|
|
return lock2.acquire(blocking=False)
|
|
|
|
|
|
class TestAPI(tests.TestCaseSkipNotImplemented):
|
|
|
|
url = os.getenv("TOOZ_TEST_URL")
|
|
|
|
def assertRaisesAny(self, exc_classes, callable_obj, *args, **kwargs):
|
|
checkers = [matchers.MatchesException(exc_class)
|
|
for exc_class in exc_classes]
|
|
matcher = matchers.Raises(matchers.MatchesAny(*checkers))
|
|
callable_obj = testcase.Nullary(callable_obj, *args, **kwargs)
|
|
self.assertThat(callable_obj, matcher)
|
|
|
|
def setUp(self):
|
|
super(TestAPI, self).setUp()
|
|
if self.url is None:
|
|
self.skipTest("No URL set for this driver")
|
|
self.useFixture(fixtures.NestedTempfile())
|
|
self.group_id = tests.get_random_uuid()
|
|
self.member_id = tests.get_random_uuid()
|
|
self._coord = tooz.coordination.get_coordinator(self.url,
|
|
self.member_id)
|
|
self._coord.start()
|
|
|
|
def tearDown(self):
|
|
self._coord.stop()
|
|
super(TestAPI, self).tearDown()
|
|
|
|
def test_connection_error_bad_host(self):
|
|
if (tooz.coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS
|
|
not in self._coord.CHARACTERISTICS):
|
|
self.skipTest("This driver is not distributed across hosts")
|
|
scheme = parse.urlparse(self.url).scheme
|
|
coord = tooz.coordination.get_coordinator(
|
|
"%s://localhost:1/f00" % scheme,
|
|
self.member_id)
|
|
self.assertRaises(tooz.coordination.ToozConnectionError,
|
|
coord.start)
|
|
|
|
def test_stop_first(self):
|
|
c = tooz.coordination.get_coordinator(self.url,
|
|
self.member_id)
|
|
self.assertRaises(tooz.coordination.ToozError,
|
|
c.stop)
|
|
|
|
def test_create_group(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
all_group_ids = self._coord.get_groups().get()
|
|
self.assertIn(self.group_id, all_group_ids)
|
|
|
|
def test_get_lock_release_broken(self):
|
|
name = tests.get_random_uuid()
|
|
memberid2 = tests.get_random_uuid()
|
|
coord2 = tooz.coordination.get_coordinator(self.url,
|
|
memberid2)
|
|
coord2.start()
|
|
lock1 = self._coord.get_lock(name)
|
|
lock2 = coord2.get_lock(name)
|
|
self.assertTrue(lock1.acquire(blocking=False))
|
|
self.assertFalse(lock2.acquire(blocking=False))
|
|
self.assertTrue(lock2.break_())
|
|
self.assertTrue(lock2.acquire(blocking=False))
|
|
self.assertFalse(lock1.release())
|
|
# Assert lock is not accidentally broken now
|
|
memberid3 = tests.get_random_uuid()
|
|
coord3 = tooz.coordination.get_coordinator(self.url,
|
|
memberid3)
|
|
coord3.start()
|
|
lock3 = coord3.get_lock(name)
|
|
self.assertFalse(lock3.acquire(blocking=False))
|
|
|
|
def test_create_group_already_exist(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
create_group = self._coord.create_group(self.group_id)
|
|
self.assertRaises(tooz.coordination.GroupAlreadyExist,
|
|
create_group.get)
|
|
|
|
def test_get_groups(self):
|
|
groups_ids = [tests.get_random_uuid() for _ in range(0, 5)]
|
|
for group_id in groups_ids:
|
|
self._coord.create_group(group_id).get()
|
|
created_groups = self._coord.get_groups().get()
|
|
for group_id in groups_ids:
|
|
self.assertIn(group_id, created_groups)
|
|
|
|
def test_delete_group(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
all_group_ids = self._coord.get_groups().get()
|
|
self.assertIn(self.group_id, all_group_ids)
|
|
self._coord.delete_group(self.group_id).get()
|
|
all_group_ids = self._coord.get_groups().get()
|
|
self.assertNotIn(self.group_id, all_group_ids)
|
|
join_group = self._coord.join_group(self.group_id)
|
|
self.assertRaises(tooz.coordination.GroupNotCreated,
|
|
join_group.get)
|
|
|
|
def test_delete_group_non_existent(self):
|
|
delete = self._coord.delete_group(self.group_id)
|
|
self.assertRaises(tooz.coordination.GroupNotCreated,
|
|
delete.get)
|
|
|
|
def test_delete_group_non_empty(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.join_group(self.group_id).get()
|
|
delete = self._coord.delete_group(self.group_id)
|
|
self.assertRaises(tooz.coordination.GroupNotEmpty,
|
|
delete.get)
|
|
self._coord.leave_group(self.group_id)
|
|
self._coord.delete_group(self.group_id).get()
|
|
|
|
def test_join_group(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.join_group(self.group_id).get()
|
|
member_list = self._coord.get_members(self.group_id).get()
|
|
self.assertIn(self.member_id, member_list)
|
|
|
|
def test_join_nonexistent_group(self):
|
|
join_group = self._coord.join_group(self.group_id)
|
|
self.assertRaises(tooz.coordination.GroupNotCreated,
|
|
join_group.get)
|
|
|
|
def test_join_group_create(self):
|
|
self._coord.join_group_create(self.group_id)
|
|
member_list = self._coord.get_members(self.group_id).get()
|
|
self.assertIn(self.member_id, member_list)
|
|
|
|
def test_join_group_with_member_id_already_exists(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.join_group(self.group_id).get()
|
|
client = tooz.coordination.get_coordinator(self.url,
|
|
self.member_id)
|
|
client.start()
|
|
join_group = client.join_group(self.group_id)
|
|
self.assertRaises(tooz.coordination.MemberAlreadyExist,
|
|
join_group.get)
|
|
|
|
def test_leave_group(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
all_group_ids = self._coord.get_groups().get()
|
|
self.assertIn(self.group_id, all_group_ids)
|
|
self._coord.join_group(self.group_id).get()
|
|
member_list = self._coord.get_members(self.group_id).get()
|
|
self.assertIn(self.member_id, member_list)
|
|
member_ids = self._coord.get_members(self.group_id).get()
|
|
self.assertIn(self.member_id, member_ids)
|
|
self._coord.leave_group(self.group_id).get()
|
|
new_member_objects = self._coord.get_members(self.group_id).get()
|
|
new_member_list = [member.member_id for member in new_member_objects]
|
|
self.assertNotIn(self.member_id, new_member_list)
|
|
|
|
def test_leave_nonexistent_group(self):
|
|
all_group_ids = self._coord.get_groups().get()
|
|
self.assertNotIn(self.group_id, all_group_ids)
|
|
leave_group = self._coord.leave_group(self.group_id)
|
|
# Drivers raise one of those depending on their capability
|
|
self.assertRaisesAny([tooz.coordination.MemberNotJoined,
|
|
tooz.coordination.GroupNotCreated],
|
|
leave_group.get)
|
|
|
|
def test_leave_group_not_joined_by_member(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
all_group_ids = self._coord.get_groups().get()
|
|
self.assertIn(self.group_id, all_group_ids)
|
|
leave_group = self._coord.leave_group(self.group_id)
|
|
self.assertRaises(tooz.coordination.MemberNotJoined,
|
|
leave_group.get)
|
|
|
|
def test_get_lock_twice_locked_one_released_two(self):
|
|
name = tests.get_random_uuid()
|
|
lock1 = self._coord.get_lock(name)
|
|
lock2 = self._coord.get_lock(name)
|
|
self.assertTrue(lock1.acquire())
|
|
self.assertFalse(lock2.acquire(blocking=False))
|
|
self.assertFalse(lock2.release())
|
|
self.assertTrue(lock1.release())
|
|
self.assertFalse(lock2.release())
|
|
|
|
def test_get_members(self):
|
|
group_id_test2 = tests.get_random_uuid()
|
|
member_id_test2 = tests.get_random_uuid()
|
|
client2 = tooz.coordination.get_coordinator(self.url,
|
|
member_id_test2)
|
|
client2.start()
|
|
|
|
self._coord.create_group(group_id_test2).get()
|
|
self._coord.join_group(group_id_test2).get()
|
|
client2.join_group(group_id_test2).get()
|
|
members_ids = self._coord.get_members(group_id_test2).get()
|
|
self.assertEqual({self.member_id, member_id_test2}, members_ids)
|
|
|
|
def test_get_member_capabilities(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.join_group(self.group_id, b"test_capabilities")
|
|
|
|
capa = self._coord.get_member_capabilities(self.group_id,
|
|
self.member_id).get()
|
|
self.assertEqual(capa, b"test_capabilities")
|
|
|
|
def test_get_member_capabilities_complex(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
caps = {
|
|
'type': 'warrior',
|
|
'abilities': ['fight', 'flight', 'double-hit-damage'],
|
|
}
|
|
self._coord.join_group(self.group_id, caps)
|
|
capa = self._coord.get_member_capabilities(self.group_id,
|
|
self.member_id).get()
|
|
self.assertEqual(capa, caps)
|
|
self.assertEqual(capa['type'], caps['type'])
|
|
|
|
def test_get_member_capabilities_nonexistent_group(self):
|
|
capa = self._coord.get_member_capabilities(self.group_id,
|
|
self.member_id)
|
|
# Drivers raise one of those depending on their capability
|
|
self.assertRaisesAny([tooz.coordination.MemberNotJoined,
|
|
tooz.coordination.GroupNotCreated],
|
|
capa.get)
|
|
|
|
def test_get_member_capabilities_nonjoined_member(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
capa = self._coord.get_member_capabilities(self.group_id,
|
|
self.member_id)
|
|
self.assertRaises(tooz.coordination.MemberNotJoined,
|
|
capa.get)
|
|
|
|
def test_get_member_info(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.join_group(self.group_id, b"test_capabilities")
|
|
|
|
member_info = self._coord.get_member_info(self.group_id,
|
|
self.member_id).get()
|
|
self.assertEqual(member_info['capabilities'], b"test_capabilities")
|
|
|
|
def test_get_member_info_complex(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
caps = {
|
|
'type': 'warrior',
|
|
'abilities': ['fight', 'flight', 'double-hit-damage'],
|
|
}
|
|
member_info = {'capabilities': 'caps',
|
|
'created_at': '0',
|
|
'updated_at': '0'}
|
|
self._coord.join_group(self.group_id, caps)
|
|
member_info = self._coord.get_member_info(self.group_id,
|
|
self.member_id).get()
|
|
self.assertEqual(member_info['capabilities'], caps)
|
|
|
|
def test_get_member_info_nonexistent_group(self):
|
|
member_info = self._coord.get_member_info(self.group_id,
|
|
self.member_id)
|
|
# Drivers raise one of those depending on their capability
|
|
self.assertRaisesAny([tooz.coordination.MemberNotJoined,
|
|
tooz.coordination.GroupNotCreated],
|
|
member_info.get)
|
|
|
|
def test_get_member_info_nonjoined_member(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
member_id = tests.get_random_uuid()
|
|
member_info = self._coord.get_member_info(self.group_id,
|
|
member_id)
|
|
self.assertRaises(tooz.coordination.MemberNotJoined,
|
|
member_info.get)
|
|
|
|
def test_update_capabilities(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.join_group(self.group_id, b"test_capabilities1").get()
|
|
|
|
capa = self._coord.get_member_capabilities(self.group_id,
|
|
self.member_id).get()
|
|
self.assertEqual(capa, b"test_capabilities1")
|
|
self._coord.update_capabilities(self.group_id,
|
|
b"test_capabilities2").get()
|
|
|
|
capa2 = self._coord.get_member_capabilities(self.group_id,
|
|
self.member_id).get()
|
|
self.assertEqual(capa2, b"test_capabilities2")
|
|
|
|
def test_update_capabilities_with_group_id_nonexistent(self):
|
|
update_cap = self._coord.update_capabilities(self.group_id,
|
|
b'test_capabilities')
|
|
# Drivers raise one of those depending on their capability
|
|
self.assertRaisesAny([tooz.coordination.MemberNotJoined,
|
|
tooz.coordination.GroupNotCreated],
|
|
update_cap.get)
|
|
|
|
def test_heartbeat(self):
|
|
if not self._coord.requires_beating:
|
|
raise testcase.TestSkipped("Test not applicable (heartbeating"
|
|
" not required)")
|
|
self._coord.heartbeat()
|
|
|
|
def test_heartbeat_loop(self):
|
|
if not self._coord.requires_beating:
|
|
raise testcase.TestSkipped("Test not applicable (heartbeating"
|
|
" not required)")
|
|
|
|
heart = self._coord.heart
|
|
self.assertFalse(heart.is_alive())
|
|
heart.start()
|
|
|
|
# This will timeout if nothing ever is done...
|
|
try:
|
|
while not heart.beats:
|
|
time.sleep(1)
|
|
finally:
|
|
heart.stop()
|
|
heart.wait()
|
|
|
|
def test_disconnect_leave_group(self):
|
|
member_id_test2 = tests.get_random_uuid()
|
|
client2 = tooz.coordination.get_coordinator(self.url,
|
|
member_id_test2)
|
|
client2.start()
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.join_group(self.group_id).get()
|
|
client2.join_group(self.group_id).get()
|
|
members_ids = self._coord.get_members(self.group_id).get()
|
|
self.assertIn(self.member_id, members_ids)
|
|
self.assertIn(member_id_test2, members_ids)
|
|
client2.stop()
|
|
members_ids = self._coord.get_members(self.group_id).get()
|
|
self.assertIn(self.member_id, members_ids)
|
|
self.assertNotIn(member_id_test2, members_ids)
|
|
|
|
def test_timeout(self):
|
|
if (tooz.coordination.Characteristics.NON_TIMEOUT_BASED
|
|
in self._coord.CHARACTERISTICS):
|
|
self.skipTest("This driver is not based on timeout")
|
|
self._coord.stop()
|
|
if "?" in self.url:
|
|
sep = "&"
|
|
else:
|
|
sep = "?"
|
|
url = self.url + sep + "timeout=5"
|
|
self._coord = tooz.coordination.get_coordinator(url, self.member_id)
|
|
self._coord.start()
|
|
|
|
member_id_test2 = tests.get_random_uuid()
|
|
client2 = tooz.coordination.get_coordinator(url, member_id_test2)
|
|
client2.start()
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.join_group(self.group_id).get()
|
|
client2.join_group(self.group_id).get()
|
|
members_ids = self._coord.get_members(self.group_id).get()
|
|
self.assertIn(self.member_id, members_ids)
|
|
self.assertIn(member_id_test2, members_ids)
|
|
|
|
# Watch the group, we want to be sure that when client2 is kicked out
|
|
# we get an event.
|
|
self._coord.watch_leave_group(self.group_id, self._set_event)
|
|
|
|
time.sleep(3)
|
|
self._coord.heartbeat()
|
|
time.sleep(3)
|
|
|
|
# Now client2 has timed out!
|
|
|
|
members_ids = self._coord.get_members(self.group_id).get()
|
|
while True:
|
|
if self._coord.run_watchers():
|
|
break
|
|
self.assertIn(self.member_id, members_ids)
|
|
self.assertNotIn(member_id_test2, members_ids)
|
|
# Check that the event has been triggered
|
|
self.assertIsInstance(self.event,
|
|
tooz.coordination.MemberLeftGroup)
|
|
self.assertEqual(member_id_test2,
|
|
self.event.member_id)
|
|
self.assertEqual(self.group_id,
|
|
self.event.group_id)
|
|
|
|
def _set_event(self, event):
|
|
self.event = event
|
|
return 42
|
|
|
|
def test_watch_group_join(self):
|
|
member_id_test2 = tests.get_random_uuid()
|
|
client2 = tooz.coordination.get_coordinator(self.url,
|
|
member_id_test2)
|
|
client2.start()
|
|
self._coord.create_group(self.group_id).get()
|
|
|
|
# Watch the group
|
|
self._coord.watch_join_group(self.group_id, self._set_event)
|
|
|
|
# Join the group
|
|
client2.join_group(self.group_id).get()
|
|
members_ids = self._coord.get_members(self.group_id).get()
|
|
self.assertIn(member_id_test2, members_ids)
|
|
while True:
|
|
if self._coord.run_watchers():
|
|
break
|
|
self.assertIsInstance(self.event,
|
|
tooz.coordination.MemberJoinedGroup)
|
|
self.assertEqual(member_id_test2,
|
|
self.event.member_id)
|
|
self.assertEqual(self.group_id,
|
|
self.event.group_id)
|
|
|
|
# Stop watching
|
|
self._coord.unwatch_join_group(self.group_id, self._set_event)
|
|
self.event = None
|
|
|
|
# Leave and rejoin group
|
|
client2.leave_group(self.group_id).get()
|
|
client2.join_group(self.group_id).get()
|
|
self._coord.run_watchers()
|
|
self.assertIsNone(self.event)
|
|
|
|
def test_watch_leave_group(self):
|
|
member_id_test2 = tests.get_random_uuid()
|
|
client2 = tooz.coordination.get_coordinator(self.url,
|
|
member_id_test2)
|
|
client2.start()
|
|
self._coord.create_group(self.group_id).get()
|
|
|
|
# Watch the group: this can leads to race conditions in certain
|
|
# driver that are not able to see all events, so we join, wait for
|
|
# the join to be seen, and then we leave, and wait for the leave to
|
|
# be seen.
|
|
self._coord.watch_join_group(self.group_id, lambda children: True)
|
|
self._coord.watch_leave_group(self.group_id, self._set_event)
|
|
|
|
# Join and leave the group
|
|
client2.join_group(self.group_id).get()
|
|
# Consumes join event
|
|
while True:
|
|
if self._coord.run_watchers():
|
|
break
|
|
client2.leave_group(self.group_id).get()
|
|
# Consumes leave event
|
|
while True:
|
|
if self._coord.run_watchers():
|
|
break
|
|
|
|
self.assertIsInstance(self.event,
|
|
tooz.coordination.MemberLeftGroup)
|
|
self.assertEqual(member_id_test2,
|
|
self.event.member_id)
|
|
self.assertEqual(self.group_id,
|
|
self.event.group_id)
|
|
|
|
# Stop watching
|
|
self._coord.unwatch_leave_group(self.group_id, self._set_event)
|
|
self.event = None
|
|
|
|
# Rejoin and releave group
|
|
client2.join_group(self.group_id).get()
|
|
client2.leave_group(self.group_id).get()
|
|
self._coord.run_watchers()
|
|
self.assertIsNone(self.event)
|
|
|
|
def test_watch_join_group_disappear(self):
|
|
if not hasattr(self._coord, '_destroy_group'):
|
|
self.skipTest("This test only works with coordinators"
|
|
" that have the ability to destroy groups.")
|
|
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.watch_join_group(self.group_id, self._set_event)
|
|
self._coord.watch_leave_group(self.group_id, self._set_event)
|
|
|
|
member_id_test2 = tests.get_random_uuid()
|
|
client2 = tooz.coordination.get_coordinator(self.url,
|
|
member_id_test2)
|
|
client2.start()
|
|
client2.join_group(self.group_id).get()
|
|
|
|
while True:
|
|
if self._coord.run_watchers():
|
|
break
|
|
self.assertIsInstance(self.event,
|
|
tooz.coordination.MemberJoinedGroup)
|
|
self.event = None
|
|
|
|
# Force the group to disappear...
|
|
self._coord._destroy_group(self.group_id)
|
|
|
|
while True:
|
|
if self._coord.run_watchers():
|
|
break
|
|
|
|
self.assertIsInstance(self.event,
|
|
tooz.coordination.MemberLeftGroup)
|
|
|
|
def test_watch_join_group_non_existent(self):
|
|
self.assertRaises(tooz.coordination.GroupNotCreated,
|
|
self._coord.watch_join_group,
|
|
self.group_id,
|
|
lambda: None)
|
|
self.assertEqual(0, len(self._coord._hooks_join_group[self.group_id]))
|
|
|
|
def test_watch_join_group_booted_out(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.join_group(self.group_id).get()
|
|
self._coord.watch_join_group(self.group_id, self._set_event)
|
|
self._coord.watch_leave_group(self.group_id, self._set_event)
|
|
|
|
member_id_test2 = tests.get_random_uuid()
|
|
client2 = tooz.coordination.get_coordinator(self.url,
|
|
member_id_test2)
|
|
client2.start()
|
|
client2.join_group(self.group_id).get()
|
|
|
|
while True:
|
|
if self._coord.run_watchers():
|
|
break
|
|
|
|
client3 = tooz.coordination.get_coordinator(self.url, self.member_id)
|
|
client3.start()
|
|
client3.leave_group(self.group_id).get()
|
|
|
|
# Only works for clients that have access to the groups they are part
|
|
# of, to ensure that after we got booted out by client3 that this
|
|
# client now no longer believes its part of the group.
|
|
if (hasattr(self._coord, '_joined_groups')
|
|
and (self._coord.run_watchers
|
|
== tooz.coordination.CoordinationDriverCachedRunWatchers.run_watchers)): # noqa
|
|
self.assertIn(self.group_id, self._coord._joined_groups)
|
|
self._coord.run_watchers()
|
|
self.assertNotIn(self.group_id, self._coord._joined_groups)
|
|
|
|
def test_watch_leave_group_non_existent(self):
|
|
self.assertRaises(tooz.coordination.GroupNotCreated,
|
|
self._coord.watch_leave_group,
|
|
self.group_id,
|
|
lambda: None)
|
|
self.assertEqual(0, len(self._coord._hooks_leave_group[self.group_id]))
|
|
|
|
def test_run_for_election(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.watch_elected_as_leader(self.group_id, self._set_event)
|
|
self._coord.run_watchers()
|
|
|
|
self.assertIsInstance(self.event,
|
|
tooz.coordination.LeaderElected)
|
|
self.assertEqual(self.member_id,
|
|
self.event.member_id)
|
|
self.assertEqual(self.group_id,
|
|
self.event.group_id)
|
|
|
|
def test_run_for_election_multiple_clients(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.watch_elected_as_leader(self.group_id, self._set_event)
|
|
self._coord.run_watchers()
|
|
|
|
member_id_test2 = tests.get_random_uuid()
|
|
client2 = tooz.coordination.get_coordinator(self.url,
|
|
member_id_test2)
|
|
client2.start()
|
|
client2.watch_elected_as_leader(self.group_id, self._set_event)
|
|
client2.run_watchers()
|
|
|
|
self.assertIsInstance(self.event,
|
|
tooz.coordination.LeaderElected)
|
|
self.assertEqual(self.member_id,
|
|
self.event.member_id)
|
|
self.assertEqual(self.group_id,
|
|
self.event.group_id)
|
|
self.assertEqual(self._coord.get_leader(self.group_id).get(),
|
|
self.member_id)
|
|
|
|
self.event = None
|
|
|
|
self._coord.stop()
|
|
client2.run_watchers()
|
|
|
|
self.assertIsInstance(self.event,
|
|
tooz.coordination.LeaderElected)
|
|
self.assertEqual(member_id_test2,
|
|
self.event.member_id)
|
|
self.assertEqual(self.group_id,
|
|
self.event.group_id)
|
|
self.assertEqual(client2.get_leader(self.group_id).get(),
|
|
member_id_test2)
|
|
|
|
# Restart the coord because tearDown stops it
|
|
self._coord.start()
|
|
|
|
def test_get_leader(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
|
|
leader = self._coord.get_leader(self.group_id).get()
|
|
self.assertIsNone(leader)
|
|
|
|
self._coord.join_group(self.group_id).get()
|
|
|
|
leader = self._coord.get_leader(self.group_id).get()
|
|
self.assertIsNone(leader)
|
|
|
|
# Let's get elected
|
|
self._coord.watch_elected_as_leader(self.group_id, self._set_event)
|
|
self._coord.run_watchers()
|
|
|
|
leader = self._coord.get_leader(self.group_id).get()
|
|
self.assertEqual(leader, self.member_id)
|
|
|
|
def test_run_for_election_multiple_clients_stand_down(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.watch_elected_as_leader(self.group_id, self._set_event)
|
|
self._coord.run_watchers()
|
|
|
|
member_id_test2 = tests.get_random_uuid()
|
|
client2 = tooz.coordination.get_coordinator(self.url,
|
|
member_id_test2)
|
|
client2.start()
|
|
client2.watch_elected_as_leader(self.group_id, self._set_event)
|
|
client2.run_watchers()
|
|
|
|
self.assertIsInstance(self.event,
|
|
tooz.coordination.LeaderElected)
|
|
self.assertEqual(self.member_id,
|
|
self.event.member_id)
|
|
self.assertEqual(self.group_id,
|
|
self.event.group_id)
|
|
|
|
self.event = None
|
|
|
|
self._coord.stand_down_group_leader(self.group_id)
|
|
client2.run_watchers()
|
|
|
|
self.assertIsInstance(self.event,
|
|
tooz.coordination.LeaderElected)
|
|
self.assertEqual(member_id_test2,
|
|
self.event.member_id)
|
|
self.assertEqual(self.group_id,
|
|
self.event.group_id)
|
|
|
|
self.event = None
|
|
|
|
client2.stand_down_group_leader(self.group_id)
|
|
self._coord.run_watchers()
|
|
|
|
self.assertIsInstance(self.event,
|
|
tooz.coordination.LeaderElected)
|
|
self.assertEqual(self.member_id,
|
|
self.event.member_id)
|
|
self.assertEqual(self.group_id,
|
|
self.event.group_id)
|
|
|
|
def test_unwatch_elected_as_leader(self):
|
|
# Create a group and add a elected_as_leader callback
|
|
self._coord.create_group(self.group_id).get()
|
|
self._coord.watch_elected_as_leader(self.group_id, self._set_event)
|
|
|
|
# Ensure exactly one leader election hook exists
|
|
self.assertEqual(1,
|
|
len(self._coord._hooks_elected_leader[self.group_id]))
|
|
|
|
# Unwatch, and ensure no leader election hooks exist
|
|
self._coord.unwatch_elected_as_leader(self.group_id, self._set_event)
|
|
self.assertEqual(0, len(self._coord._hooks_elected_leader))
|
|
|
|
def test_unwatch_elected_as_leader_callback_not_found(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self.assertRaises(tooz.coordination.WatchCallbackNotFound,
|
|
self._coord.unwatch_elected_as_leader,
|
|
self.group_id, lambda x: None)
|
|
|
|
def test_unwatch_join_group_callback_not_found(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self.assertRaises(tooz.coordination.WatchCallbackNotFound,
|
|
self._coord.unwatch_join_group,
|
|
self.group_id, lambda x: None)
|
|
|
|
def test_unwatch_leave_group(self):
|
|
# Create a group and add a leave_group callback
|
|
self._coord.create_group(self.group_id).get()
|
|
self.assertEqual(0, len(self._coord._hooks_leave_group))
|
|
self._coord.watch_leave_group(self.group_id, self._set_event)
|
|
|
|
# Ensure exactly one leave group hook exists
|
|
self.assertEqual(1, len(self._coord._hooks_leave_group[self.group_id]))
|
|
|
|
# Unwatch, and ensure no leave group hooks exist
|
|
self._coord.unwatch_leave_group(self.group_id, self._set_event)
|
|
self.assertEqual(0, len(self._coord._hooks_leave_group))
|
|
|
|
def test_unwatch_leave_group_callback_not_found(self):
|
|
self._coord.create_group(self.group_id).get()
|
|
self.assertRaises(tooz.coordination.WatchCallbackNotFound,
|
|
self._coord.unwatch_leave_group,
|
|
self.group_id, lambda x: None)
|
|
|
|
def test_get_lock(self):
|
|
lock = self._coord.get_lock(tests.get_random_uuid())
|
|
self.assertTrue(lock.acquire())
|
|
self.assertTrue(lock.release())
|
|
with lock:
|
|
pass
|
|
|
|
def test_get_lock_concurrency_locking_same_lock(self):
|
|
lock = self._coord.get_lock(tests.get_random_uuid())
|
|
|
|
graceful_ending = threading.Event()
|
|
|
|
def thread():
|
|
self.assertTrue(lock.acquire())
|
|
self.assertTrue(lock.release())
|
|
graceful_ending.set()
|
|
|
|
t = threading.Thread(target=thread)
|
|
t.daemon = True
|
|
with lock:
|
|
t.start()
|
|
# Ensure the thread try to get the lock
|
|
time.sleep(.1)
|
|
t.join()
|
|
graceful_ending.wait(.2)
|
|
self.assertTrue(graceful_ending.is_set())
|
|
|
|
def _do_test_get_lock_concurrency_locking_two_lock(self, executor,
|
|
use_same_coord):
|
|
name = tests.get_random_uuid()
|
|
lock1 = self._coord.get_lock(name)
|
|
with lock1:
|
|
with executor(max_workers=1) as e:
|
|
coord = self._coord if use_same_coord else None
|
|
f = e.submit(try_to_lock_job, name, coord, self.url,
|
|
tests.get_random_uuid())
|
|
self.assertFalse(f.result())
|
|
|
|
def test_get_lock_concurrency_locking_two_lock_process(self):
|
|
self._do_test_get_lock_concurrency_locking_two_lock(
|
|
futures.ProcessPoolExecutor, False)
|
|
|
|
def test_get_lock_concurrency_locking_two_lock_thread1(self):
|
|
self._do_test_get_lock_concurrency_locking_two_lock(
|
|
futures.ThreadPoolExecutor, False)
|
|
|
|
def test_get_lock_concurrency_locking_two_lock_thread2(self):
|
|
self._do_test_get_lock_concurrency_locking_two_lock(
|
|
futures.ThreadPoolExecutor, True)
|
|
|
|
def test_get_lock_concurrency_locking2(self):
|
|
# NOTE(sileht): some database based lock can have only
|
|
# one lock per connection, this test ensures acquiring a
|
|
# second lock doesn't release the first one.
|
|
lock1 = self._coord.get_lock(tests.get_random_uuid())
|
|
lock2 = self._coord.get_lock(tests.get_random_uuid())
|
|
|
|
graceful_ending = threading.Event()
|
|
thread_locked = threading.Event()
|
|
|
|
def thread():
|
|
with lock2:
|
|
try:
|
|
self.assertFalse(lock1.acquire(blocking=False))
|
|
except tooz.NotImplemented:
|
|
pass
|
|
thread_locked.set()
|
|
graceful_ending.set()
|
|
|
|
t = threading.Thread(target=thread)
|
|
t.daemon = True
|
|
|
|
with lock1:
|
|
t.start()
|
|
thread_locked.wait()
|
|
self.assertTrue(thread_locked.is_set())
|
|
t.join()
|
|
graceful_ending.wait()
|
|
self.assertTrue(graceful_ending.is_set())
|
|
|
|
def test_get_lock_twice_locked_twice(self):
|
|
name = tests.get_random_uuid()
|
|
lock1 = self._coord.get_lock(name)
|
|
lock2 = self._coord.get_lock(name)
|
|
with lock1:
|
|
self.assertFalse(lock2.acquire(blocking=False))
|
|
|
|
def test_get_lock_context_fails(self):
|
|
name = tests.get_random_uuid()
|
|
lock1 = self._coord.get_lock(name)
|
|
lock2 = self._coord.get_lock(name)
|
|
with mock.patch.object(lock2, 'acquire', return_value=False):
|
|
with lock1:
|
|
self.assertRaises(
|
|
tooz.coordination.LockAcquireFailed,
|
|
lock2.__enter__)
|
|
|
|
def test_get_lock_context_check_value(self):
|
|
name = tests.get_random_uuid()
|
|
lock = self._coord.get_lock(name)
|
|
with lock as returned_lock:
|
|
self.assertEqual(lock, returned_lock)
|
|
|
|
def test_lock_context_manager_acquire_no_argument(self):
|
|
name = tests.get_random_uuid()
|
|
lock1 = self._coord.get_lock(name)
|
|
lock2 = self._coord.get_lock(name)
|
|
with lock1():
|
|
self.assertFalse(lock2.acquire(blocking=False))
|
|
|
|
def test_lock_context_manager_acquire_argument_return_value(self):
|
|
name = tests.get_random_uuid()
|
|
blocking_value = 10.12
|
|
lock = self._coord.get_lock(name)
|
|
with lock(blocking_value) as returned_lock:
|
|
self.assertEqual(lock, returned_lock)
|
|
|
|
def test_lock_context_manager_acquire_argument_release_within(self):
|
|
name = tests.get_random_uuid()
|
|
blocking_value = 10.12
|
|
lock = self._coord.get_lock(name)
|
|
with lock(blocking_value) as returned_lock:
|
|
self.assertTrue(returned_lock.release())
|
|
|
|
def test_lock_context_manager_acquire_argument(self):
|
|
name = tests.get_random_uuid()
|
|
blocking_value = 10.12
|
|
lock = self._coord.get_lock(name)
|
|
with mock.patch.object(lock, 'acquire', wraps=True, autospec=True) as \
|
|
mock_acquire:
|
|
with lock(blocking_value):
|
|
mock_acquire.assert_called_once_with(blocking_value)
|
|
|
|
def test_lock_context_manager_acquire_argument_timeout(self):
|
|
name = tests.get_random_uuid()
|
|
lock1 = self._coord.get_lock(name)
|
|
lock2 = self._coord.get_lock(name)
|
|
with lock1:
|
|
try:
|
|
with lock2(False):
|
|
self.fail('Lock acquire should have failed')
|
|
except tooz.coordination.LockAcquireFailed:
|
|
pass
|
|
|
|
def test_get_lock_locked_twice(self):
|
|
name = tests.get_random_uuid()
|
|
lock = self._coord.get_lock(name)
|
|
with lock:
|
|
self.assertFalse(lock.acquire(blocking=False))
|
|
|
|
def test_get_multiple_locks_with_same_coord(self):
|
|
name = tests.get_random_uuid()
|
|
lock1 = self._coord.get_lock(name)
|
|
lock2 = self._coord.get_lock(name)
|
|
self.assertTrue(lock1.acquire())
|
|
self.assertFalse(lock2.acquire(blocking=False))
|
|
self.assertFalse(self._coord.get_lock(name).acquire(blocking=False))
|
|
self.assertTrue(lock1.release())
|
|
|
|
def test_ensure_acquire_release_return(self):
|
|
name = tests.get_random_uuid()
|
|
lock1 = self._coord.get_lock(name)
|
|
self.assertTrue(lock1.acquire())
|
|
self.assertTrue(lock1.release())
|
|
self.assertFalse(lock1.release())
|
|
|
|
def test_get_lock_multiple_coords(self):
|
|
member_id2 = tests.get_random_uuid()
|
|
client2 = tooz.coordination.get_coordinator(self.url,
|
|
member_id2)
|
|
client2.start()
|
|
|
|
lock_name = tests.get_random_uuid()
|
|
lock = self._coord.get_lock(lock_name)
|
|
self.assertTrue(lock.acquire())
|
|
|
|
lock2 = client2.get_lock(lock_name)
|
|
self.assertFalse(lock2.acquire(blocking=False))
|
|
self.assertTrue(lock.release())
|
|
self.assertTrue(lock2.acquire(blocking=True))
|
|
self.assertTrue(lock2.release())
|
|
|
|
def test_get_started_status(self):
|
|
self.assertTrue(self._coord.is_started)
|
|
self._coord.stop()
|
|
self.assertFalse(self._coord.is_started)
|
|
self._coord.start()
|
|
|
|
def do_test_name_property(self):
|
|
name = tests.get_random_uuid()
|
|
lock = self._coord.get_lock(name)
|
|
self.assertEqual(name, lock.name)
|
|
|
|
def test_acquire_twice_no_deadlock_releasing(self):
|
|
name = tests.get_random_uuid()
|
|
lock = self._coord.get_lock(name)
|
|
self.assertTrue(lock.acquire(blocking=False))
|
|
self.assertFalse(lock.acquire(blocking=False))
|
|
self.assertTrue(lock.release())
|
|
|
|
|
|
class ZakeTestAPI(TestAPI):
|
|
url = "zake://"
|
|
|
|
|
|
class IPCTestAPI(TestAPI):
|
|
url = "ipc://"
|
|
|
|
|
|
class FileTestAPI(TestAPI):
|
|
url = "file:///tmp"
|
|
|
|
|
|
class TestHook(testcase.TestCase):
|
|
def setUp(self):
|
|
super(TestHook, self).setUp()
|
|
self.hooks = tooz.coordination.Hooks()
|
|
self.triggered = False
|
|
|
|
def _trigger(self):
|
|
self.triggered = True
|
|
|
|
def test_register_hook(self):
|
|
self.assertEqual(self.hooks.run(), [])
|
|
self.assertFalse(self.triggered)
|
|
self.hooks.append(self._trigger)
|
|
self.assertEqual(self.hooks.run(), [None])
|
|
self.assertTrue(self.triggered)
|
|
|
|
def test_unregister_hook(self):
|
|
self.hooks.append(self._trigger)
|
|
self.assertEqual(self.hooks.run(), [None])
|
|
self.assertTrue(self.triggered)
|
|
self.triggered = False
|
|
self.hooks.remove(self._trigger)
|
|
self.assertEqual(self.hooks.run(), [])
|
|
self.assertFalse(self.triggered)
|