add test to ensure cloud is using minimum number of nodes

This ensures that the the cloud in question is using some minimum
number of nodes as specified by compute.min_compute_nodes. We then
attempt to schedule to all those nodes directly with scheduler hints,
and verify that the host_ids are all different on the booted nodes.

This kind of test is needed when doing multinode rolling upgrade
testing, because an extremely common failure is a change which makes
the upgraded controller incompatible with a non upgraded worker. The
worker will then stop communicating and effectively drop out of the
cluster. All the other tests will pass because they just route around
the damage. However we really do want to poke every node to make sure
they all still work.

Change-Id: I7d7230d100901179a1d1a0281aa8be990d00c02a

tempest/config.py

@@ -330,7 +330,13 @@ ComputeGroup = [
                help='Unallocated floating IP range, which will be used to '
                     'test the floating IP bulk feature for CRUD operation. '
                     'This block must not overlap an existing floating IP '
-                    'pool.')
+                    'pool.'),
+    cfg.IntOpt('min_compute_nodes',
+               default=1,
+               help=('The minimum number of compute nodes expected. This will '
+                     'be utilized by some multinode specific tests to ensure '
+                     'that requests match the expected size of the cluster '
+                     'you are testing with.'))
 ]
 
 compute_features_group = cfg.OptGroup(name='compute-feature-enabled',

tempest/scenario/test_server_multinode.py

@@ -0,0 +1,78 @@
+# Copyright 2012 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from oslo_log import log as logging
+
+from tempest import config
+from tempest import exceptions
+from tempest.scenario import manager
+from tempest import test
+
+CONF = config.CONF
+
+LOG = logging.getLogger(__name__)
+
+
+class TestServerMultinode(manager.ScenarioTest):
+
+    """
+    This is a set of tests specific to multinode testing.
+
+    """
+    credentials = ['primary', 'admin']
+
+    @test.idempotent_id('9cecbe35-b9d4-48da-a37e-7ce70aa43d30')
+    @test.attr(type='smoke')
+    @test.services('compute', 'network')
+    def test_schedule_to_all_nodes(self):
+        host_client = self.admin_manager.hosts_client
+        hosts = host_client.list_hosts()['hosts']
+        hosts = [x for x in hosts if x['service'] == 'compute']
+
+        # ensure we have at least as many compute hosts as we expect
+        if len(hosts) < CONF.compute.min_compute_nodes:
+            raise exceptions.InvalidConfiguration(
+                "Host list %s is shorter than min_compute_nodes. "
+                "Did a compute worker not boot correctly?" % hosts)
+
+        # create 1 compute for each node, up to the min_compute_nodes
+        # threshold (so that things don't get crazy if you have 1000
+        # compute nodes but set min to 3).
+        servers = []
+
+        for host in hosts[:CONF.compute.min_compute_nodes]:
+            create_kwargs = {
+                'scheduler_hints': {'force_hosts': host['host_name']}
+            }
+
+            # by getting to active state here, this means this has
+            # landed on the host in question.
+            inst = self.create_server(image=CONF.compute.image_ref,
+                                      flavor=CONF.compute.flavor_ref,
+                                      create_kwargs=create_kwargs)
+            server = self.servers_client.show_server(inst['id'])['server']
+            servers.append(server)
+
+        # make sure we really have the number of servers we think we should
+        self.assertEqual(
+            len(servers), CONF.compute.min_compute_nodes,
+            "Incorrect number of servers built %s" % servers)
+
+        # ensure that every server ended up on a different host
+        host_ids = [x['hostId'] for x in servers]
+        self.assertEqual(
+            len(set(host_ids)), len(servers),
+            "Incorrect number of distinct host_ids scheduled to %s" % servers)