Only allow one scheduler service in tests

There have been two recent issues [1][2] caused by starting multiple
instances of the same service in tests. This can cause races when the
services' (shared) state conflicts.

With this patch, the nexus of nova service starting,
nova.test.TestCase.start_service, is instrumented to keep track of how
many of each service we are running. If we try to run the scheduler
service more than once, we fail.

We could probably do the same thing for conductor, though that's less
important (for now) because conductor is stateless (for now).

[1] https://bugs.launchpad.net/nova/+bug/1844174
[2] https://review.opendev.org/#/c/681059/ (not a nova service, but same
class of problem)

Change-Id: I56d3cb17260dad8b88f03c0a7b9688efb3258d6f

nova/test.py

@@ -24,6 +24,7 @@ inline callbacks.
 import nova.monkey_patch  # noqa
 
 import abc
+import collections
 import copy
 import datetime
 import inspect
@@ -213,6 +214,9 @@ class TestCase(testtools.TestCase):
             os.environ.get('OS_TEST_TIMEOUT', 0),
             self.TIMEOUT_SCALING_FACTOR))
 
+        # How many of which service we've started. {$service-name: $count}
+        self._service_fixture_count = collections.defaultdict(int)
+
         self.useFixture(nova_fixtures.OpenStackSDKFixture())
 
         self.useFixture(fixtures.NestedTempfile())
@@ -434,6 +438,10 @@ class TestCase(testtools.TestCase):
             CONF.set_override(k, v, group)
 
     def start_service(self, name, host=None, **kwargs):
+        # Disallow starting multiple scheduler services
+        if name == 'scheduler' and self._service_fixture_count[name]:
+            raise TestingException("Duplicate start_service(%s)!" % name)
+
         cell = None
         # if the host is None then the CONF.host remains defaulted to
         # 'fake-mini' (originally done in ConfFixture)
@@ -459,6 +467,19 @@ class TestCase(testtools.TestCase):
         svc = self.useFixture(
             nova_fixtures.ServiceFixture(name, host, cell=cell, **kwargs))
 
+        # Keep track of how many instances of this service are running.
+        self._service_fixture_count[name] += 1
+        real_stop = svc.service.stop
+
+        # Make sure stopping the service decrements the active count, so that
+        # start,stop,start doesn't trigger the "Duplicate start_service"
+        # exception.
+        def patch_stop(*a, **k):
+            self._service_fixture_count[name] -= 1
+            return real_stop(*a, **k)
+        self.useFixture(fixtures.MockPatchObject(
+            svc.service, 'stop', patch_stop))
+
         return svc.service
 
     def restart_compute_service(self, compute, keep_hypervisor_state=True):