Allow scheduler to run cell host discovery periodically

Since cellsv2 has required host mappings to be present, we have lost
our automatic host registration behavior. This is a conscious choice,
given that we do not want to have upcalls from the cells to the API
database. This adds a periodic task to have the scheduler do the
discovery routine, which for small deployments may be an acceptable
amount of background overhead in exchange for the automatic behavior.

There is also probably some amount of performance improvement that
can be added to the host discovery routine to make it less of an
issue. However, just generalizing the existing routine and letting
it run periodically gives us some coverage, given where we are in
the current cycle.

Related to blueprint cells-scheduler-integration

Change-Id: Iab5f172cdef35645bae56b9b384e032f3160e826
This commit is contained in:
Dan Smith 2017-01-30 08:15:02 -08:00
parent d308d1f70e
commit 9eb7bcea05
7 changed files with 178 additions and 48 deletions

View File

@ -85,6 +85,7 @@ from nova import objects
from nova.objects import aggregate as aggregate_obj
from nova.objects import build_request as build_request_obj
from nova.objects import flavor as flavor_obj
from nova.objects import host_mapping as host_mapping_obj
from nova.objects import instance as instance_obj
from nova.objects import instance_group as instance_group_obj
from nova.objects import keypair as keypair_obj
@ -1383,54 +1384,12 @@ class CellV2Commands(object):
each cell, or a single one if passed in, and map any hosts which are
not currently mapped. If a host is already mapped nothing will be done.
"""
def status_fn(msg):
if verbose:
print(msg)
ctxt = context.RequestContext()
# TODO(alaski): If this is not run on a host configured to use the API
# database most of the lookups below will fail and may not provide a
# great error message. Add a check which will raise a useful error
# message about running this from an API host.
if cell_uuid:
cell_mappings = [objects.CellMapping.get_by_uuid(ctxt, cell_uuid)]
else:
cell_mappings = objects.CellMappingList.get_all(ctxt)
if verbose:
print(_('Found %s cell mappings.') % len(cell_mappings))
for cell_mapping in cell_mappings:
if cell_mapping.is_cell0():
if verbose:
print(_('Skipping cell0 since it does not contain hosts.'))
continue
if verbose:
if 'name' in cell_mapping and cell_mapping.name:
print(_("Getting compute nodes from cell '%(name)s': "
"%(uuid)s") % {'name': cell_mapping.name,
'uuid': cell_mapping.uuid})
else:
print(_("Getting compute nodes from cell: %(uuid)s") %
{'uuid': cell_mapping.uuid})
with context.target_cell(ctxt, cell_mapping):
compute_nodes = objects.ComputeNodeList.get_all(ctxt)
if verbose:
print(_('Found %(num)s computes in cell: %(uuid)s') %
{'num': len(compute_nodes),
'uuid': cell_mapping.uuid})
for compute in compute_nodes:
if verbose:
print(_("Checking host mapping for compute host "
"'%(host)s': %(uuid)s") %
{'host': compute.host, 'uuid': compute.uuid})
try:
objects.HostMapping.get_by_host(ctxt, compute.host)
except exception.HostMappingNotFound:
if verbose:
print(_("Creating host mapping for compute host "
"'%(host)s': %(uuid)s") %
{'host': compute.host, 'uuid': compute.uuid})
host_mapping = objects.HostMapping(
ctxt, host=compute.host,
cell_mapping=cell_mapping)
host_mapping.create()
host_mapping_obj.discover_hosts(ctxt, cell_uuid, status_fn)
@action_description(
_("Add a new cell to nova API database. "

View File

@ -124,7 +124,25 @@ Possible values:
* A positive integer, where the integer corresponds to the max number of
attempts that can be made when scheduling an instance.
""")]
"""),
cfg.IntOpt("discover_hosts_in_cells_interval",
default=-1,
min=-1,
help="""
Periodic task interval.
This value controls how often (in seconds) the scheduler should attept
to discover new hosts that have been added to cells. If negative (the
default), no automatic discovery will occur.
Small deployments may want this periodic task enabled, as surveying the
cells for new hosts is likely to be lightweight enough to not cause undue
burdon to the scheduler. However, larger clouds (and those that are not
adding hosts regularly) will likely want to disable this automatic
behavior and instead use the `nova-manage cell_v2 discover_hosts` command
when hosts have been added to a cell.
"""),
]
filter_scheduler_group = cfg.OptGroup(name="filter_scheduler",
title="Filter scheduler options")

View File

@ -12,9 +12,11 @@
from sqlalchemy.orm import joinedload
from nova import context
from nova.db.sqlalchemy import api as db_api
from nova.db.sqlalchemy import api_models
from nova import exception
from nova.i18n import _
from nova.objects import base
from nova.objects import cell_mapping
from nova.objects import fields
@ -163,3 +165,55 @@ class HostMappingList(base.ObjectListBase, base.NovaObject):
def get_by_cell_id(cls, context, cell_id):
db_mappings = cls._get_by_cell_id_from_db(context, cell_id)
return base.obj_make_list(context, cls(), HostMapping, db_mappings)
def discover_hosts(ctxt, cell_uuid=None, status_fn=None):
# TODO(alaski): If this is not run on a host configured to use the API
# database most of the lookups below will fail and may not provide a
# great error message. Add a check which will raise a useful error
# message about running this from an API host.
from nova import objects
if not status_fn:
status_fn = lambda x: None
if cell_uuid:
cell_mappings = [objects.CellMapping.get_by_uuid(ctxt, cell_uuid)]
else:
cell_mappings = objects.CellMappingList.get_all(ctxt)
status_fn(_('Found %s cell mappings.') % len(cell_mappings))
host_mappings = []
for cm in cell_mappings:
if cm.is_cell0():
status_fn(_('Skipping cell0 since it does not contain hosts.'))
continue
if 'name' in cm and cm.name:
status_fn(_("Getting compute nodes from cell '%(name)s': "
"%(uuid)s") % {'name': cm.name,
'uuid': cm.uuid})
else:
status_fn(_("Getting compute nodes from cell: %(uuid)s") %
{'uuid': cm.uuid})
with context.target_cell(ctxt, cm):
compute_nodes = objects.ComputeNodeList.get_all(ctxt)
status_fn(_('Found %(num)s computes in cell: %(uuid)s') %
{'num': len(compute_nodes),
'uuid': cm.uuid})
for compute in compute_nodes:
status_fn(_("Checking host mapping for compute host "
"'%(host)s': %(uuid)s") %
{'host': compute.host, 'uuid': compute.uuid})
try:
objects.HostMapping.get_by_host(ctxt, compute.host)
except exception.HostMappingNotFound:
status_fn(_("Creating host mapping for compute host "
"'%(host)s': %(uuid)s") %
{'host': compute.host, 'uuid': compute.uuid})
host_mapping = objects.HostMapping(
ctxt, host=compute.host,
cell_mapping=cm)
host_mapping.create()
host_mappings.append(host_mapping)
return host_mappings

View File

@ -27,8 +27,10 @@ from stevedore import driver
import nova.conf
from nova import exception
from nova.i18n import _LI
from nova import manager
from nova import objects
from nova.objects import host_mapping as host_mapping_obj
from nova import quota
@ -60,6 +62,18 @@ class SchedulerManager(manager.Manager):
def _expire_reservations(self, context):
QUOTAS.expire(context)
@periodic_task.periodic_task(
spacing=CONF.scheduler.discover_hosts_in_cells_interval,
run_immediately=True)
def _discover_hosts_in_cells(self, context):
host_mappings = host_mapping_obj.discover_hosts(context)
if host_mappings:
LOG.info(_LI('Discovered %(count)i new hosts: %(hosts)s'),
{'count': len(host_mappings),
'hosts': ','.join(['%s:%s' % (hm.cell_mapping.name,
hm.host)
for hm in host_mappings])})
@periodic_task.periodic_task(spacing=CONF.scheduler.periodic_task_interval,
run_immediately=True)
def _run_periodic_tasks(self, context):

View File

@ -12,11 +12,14 @@
import mock
from nova import context
from nova import exception
from nova import objects
from nova.objects import host_mapping
from nova import test
from nova.tests.unit.objects import test_cell_mapping
from nova.tests.unit.objects import test_objects
from nova.tests import uuidsentinel as uuids
def get_db_mapping(mapped_cell=None, **updates):
@ -145,3 +148,62 @@ class TestHostMappingObject(test_objects._LocalTest,
class TestRemoteHostMappingObject(test_objects._RemoteTest,
_TestHostMappingObject):
pass
class TestHostMappingDiscovery(test.NoDBTestCase):
@mock.patch('nova.objects.CellMappingList.get_all')
@mock.patch('nova.objects.HostMapping.create')
@mock.patch('nova.objects.HostMapping.get_by_host')
@mock.patch('nova.objects.ComputeNodeList.get_all')
def test_discover_hosts_all(self, mock_cn_get, mock_hm_get, mock_hm_create,
mock_cm):
def _hm_get(context, host):
if host in ['a', 'b', 'c']:
return objects.HostMapping()
raise exception.HostMappingNotFound(name=host)
mock_hm_get.side_effect = _hm_get
mock_cn_get.side_effect = [[objects.ComputeNode(host='d',
uuid=uuids.cn1)],
[objects.ComputeNode(host='e',
uuid=uuids.cn2)]]
cell_mappings = [objects.CellMapping(name='foo',
uuid=uuids.cm1),
objects.CellMapping(name='bar',
uuid=uuids.cm2)]
mock_cm.return_value = cell_mappings
ctxt = context.get_admin_context()
hms = host_mapping.discover_hosts(ctxt)
self.assertEqual(2, len(hms))
self.assertTrue(mock_hm_create.called)
self.assertEqual(['d', 'e'],
[hm.host for hm in hms])
@mock.patch('nova.objects.CellMapping.get_by_uuid')
@mock.patch('nova.objects.HostMapping.create')
@mock.patch('nova.objects.HostMapping.get_by_host')
@mock.patch('nova.objects.ComputeNodeList.get_all')
def test_discover_hosts_one(self, mock_cn_get, mock_hm_get, mock_hm_create,
mock_cm):
def _hm_get(context, host):
if host in ['a', 'b', 'c']:
return objects.HostMapping()
raise exception.HostMappingNotFound(name=host)
mock_hm_get.side_effect = _hm_get
# NOTE(danms): Provide both side effects, but expect it to only
# be called once if we provide a cell
mock_cn_get.side_effect = [[objects.ComputeNode(host='d',
uuid=uuids.cn1)],
[objects.ComputeNode(host='e',
uuid=uuids.cn2)]]
mock_cm.return_value = objects.CellMapping(name='foo',
uuid=uuids.cm1)
ctxt = context.get_admin_context()
hms = host_mapping.discover_hosts(ctxt, uuids.cm1)
self.assertEqual(1, len(hms))
self.assertTrue(mock_hm_create.called)
self.assertEqual(['d'],
[hm.host for hm in hms])

View File

@ -160,6 +160,16 @@ class SchedulerManagerTestCase(test.NoDBTestCase):
mock.sentinel.host_name,
mock.sentinel.instance_uuids)
@mock.patch('nova.objects.host_mapping.discover_hosts')
def test_discover_hosts(self, mock_discover):
cm1 = objects.CellMapping(name='cell1')
cm2 = objects.CellMapping(name='cell2')
mock_discover.return_value = [objects.HostMapping(host='a',
cell_mapping=cm1),
objects.HostMapping(host='b',
cell_mapping=cm2)]
self.manager._discover_hosts_in_cells(mock.sentinel.context)
class SchedulerInitTestCase(test.NoDBTestCase):
"""Test case for base scheduler driver initiation."""

View File

@ -0,0 +1,13 @@
---
features:
- As new hosts are added to Nova, the `nova-manage cell_v2
discover_hosts` command must be run in order to map them into
their cell. For deployments with proper automation, this is a
trivial extra step in that process. However, for smaller or
non-automated deployments, there is a new configuration variable
for the scheduler process which will perform this discovery
periodically. By setting
`scheduler.discover_hosts_in_cells_interval` to a positive value,
the scheduler will handle this for you. Note that this process
involves listing all hosts in all cells, and is likely to be too
heavyweight for large deployments to run all the time.