Handle when a group used to exist but no longer does

When a group used to exist (or for some reason still exists
in the groups list) we should avoid failing when getting
members of that group in the running of watches function and
should instead handle that case specially and trigger all
watches (to notify those watches that everyone was removed).

Change-Id: I5484857e6ea58f1cdec5f9f8fac69a3571e053ed

tooz/drivers/redis.py

@@ -464,6 +464,10 @@ class RedisDriver(coordination.CoordinationDriver):
                                               encoded_group,
                                               value_from_callable=True))
 
+    def _destroy_group(self, group_id):
+        """Should only be used in tests..."""
+        self._client.delete(self._encode_group_id(group_id))
+
     def get_groups(self):
 
         def _get_groups():
@@ -503,8 +507,12 @@ class RedisDriver(coordination.CoordinationDriver):
     def run_watchers(self, timeout=None):
         result = []
         for group_id in self.get_groups().get(timeout=timeout):
-            group_members = set(self.get_members(group_id)
-                                .get(timeout=timeout))
+            try:
+                group_members = self.get_members(group_id).get(timeout=timeout)
+            except coordination.GroupNotCreated:
+                group_members = set()
+            else:
+                group_members = set(group_members)
             old_group_members = self._group_members.get(group_id, set())
             for member_id in (group_members - old_group_members):
                 result.extend(

tooz/tests/test_coordination.py

@@ -314,6 +314,38 @@ class TestAPI(testscenarios.TestWithScenarios,
         self._coord.run_watchers()
         self.assertIsNone(self.event)
 
+    def test_watch_join_group_disappear(self):
+        if not hasattr(self._coord, '_destroy_group'):
+            self.skipTest("This test only works with coordinators"
+                          " that have the ability to destroy groups.")
+
+        self._coord.create_group(self.group_id).get()
+        self._coord.watch_join_group(self.group_id, self._set_event)
+        self._coord.watch_leave_group(self.group_id, self._set_event)
+
+        member_id_test2 = self._get_random_uuid()
+        client2 = tooz.coordination.get_coordinator(self.url,
+                                                    member_id_test2)
+        client2.start()
+        client2.join_group(self.group_id).get()
+
+        while True:
+            if self._coord.run_watchers():
+                break
+        self.assertIsInstance(self.event,
+                              tooz.coordination.MemberJoinedGroup)
+        self.event = None
+
+        # Force the group to disappear...
+        self._coord._destroy_group(self.group_id)
+
+        while True:
+            if self._coord.run_watchers():
+                break
+
+        self.assertIsInstance(self.event,
+                              tooz.coordination.MemberLeftGroup)
+
     def test_watch_join_group_non_existent(self):
         self.assertRaises(tooz.coordination.GroupNotCreated,
                           self._coord.watch_join_group,