Allow scheduler to run cell host discovery periodically
Since cellsv2 has required host mappings to be present, we have lost our automatic host registration behavior. This is a conscious choice, given that we do not want to have upcalls from the cells to the API database. This adds a periodic task to have the scheduler do the discovery routine, which for small deployments may be an acceptable amount of background overhead in exchange for the automatic behavior. There is also probably some amount of performance improvement that can be added to the host discovery routine to make it less of an issue. However, just generalizing the existing routine and letting it run periodically gives us some coverage, given where we are in the current cycle. Related to blueprint cells-scheduler-integration Change-Id: Iab5f172cdef35645bae56b9b384e032f3160e826
This commit is contained in:
Dan Smith
parent
d308d1f70e
commit
9eb7bcea05
@ -85,6 +85,7 @@ from nova import objects
|
||||
from nova.objects import aggregate as aggregate_obj
|
||||
from nova.objects import build_request as build_request_obj
|
||||
from nova.objects import flavor as flavor_obj
|
||||
from nova.objects import host_mapping as host_mapping_obj
|
||||
from nova.objects import instance as instance_obj
|
||||
from nova.objects import instance_group as instance_group_obj
|
||||
from nova.objects import keypair as keypair_obj
|
||||
@ -1383,54 +1384,12 @@ class CellV2Commands(object):
|
||||
each cell, or a single one if passed in, and map any hosts which are
|
||||
not currently mapped. If a host is already mapped nothing will be done.
|
||||
"""
|
||||
def status_fn(msg):
|
||||
if verbose:
|
||||
print(msg)
|
||||
|
||||
ctxt = context.RequestContext()
|
||||
|
||||
# TODO(alaski): If this is not run on a host configured to use the API
|
||||
# database most of the lookups below will fail and may not provide a
|
||||
# great error message. Add a check which will raise a useful error
|
||||
# message about running this from an API host.
|
||||
if cell_uuid:
|
||||
cell_mappings = [objects.CellMapping.get_by_uuid(ctxt, cell_uuid)]
|
||||
else:
|
||||
cell_mappings = objects.CellMappingList.get_all(ctxt)
|
||||
if verbose:
|
||||
print(_('Found %s cell mappings.') % len(cell_mappings))
|
||||
|
||||
for cell_mapping in cell_mappings:
|
||||
if cell_mapping.is_cell0():
|
||||
if verbose:
|
||||
print(_('Skipping cell0 since it does not contain hosts.'))
|
||||
continue
|
||||
if verbose:
|
||||
if 'name' in cell_mapping and cell_mapping.name:
|
||||
print(_("Getting compute nodes from cell '%(name)s': "
|
||||
"%(uuid)s") % {'name': cell_mapping.name,
|
||||
'uuid': cell_mapping.uuid})
|
||||
else:
|
||||
print(_("Getting compute nodes from cell: %(uuid)s") %
|
||||
{'uuid': cell_mapping.uuid})
|
||||
with context.target_cell(ctxt, cell_mapping):
|
||||
compute_nodes = objects.ComputeNodeList.get_all(ctxt)
|
||||
if verbose:
|
||||
print(_('Found %(num)s computes in cell: %(uuid)s') %
|
||||
{'num': len(compute_nodes),
|
||||
'uuid': cell_mapping.uuid})
|
||||
for compute in compute_nodes:
|
||||
if verbose:
|
||||
print(_("Checking host mapping for compute host "
|
||||
"'%(host)s': %(uuid)s") %
|
||||
{'host': compute.host, 'uuid': compute.uuid})
|
||||
try:
|
||||
objects.HostMapping.get_by_host(ctxt, compute.host)
|
||||
except exception.HostMappingNotFound:
|
||||
if verbose:
|
||||
print(_("Creating host mapping for compute host "
|
||||
"'%(host)s': %(uuid)s") %
|
||||
{'host': compute.host, 'uuid': compute.uuid})
|
||||
host_mapping = objects.HostMapping(
|
||||
ctxt, host=compute.host,
|
||||
cell_mapping=cell_mapping)
|
||||
host_mapping.create()
|
||||
host_mapping_obj.discover_hosts(ctxt, cell_uuid, status_fn)
|
||||
|
||||
@action_description(
|
||||
_("Add a new cell to nova API database. "
|
||||
|
||||
@ -124,7 +124,25 @@ Possible values:
|
||||
|
||||
* A positive integer, where the integer corresponds to the max number of
|
||||
attempts that can be made when scheduling an instance.
|
||||
""")]
|
||||
"""),
|
||||
cfg.IntOpt("discover_hosts_in_cells_interval",
|
||||
default=-1,
|
||||
min=-1,
|
||||
help="""
|
||||
Periodic task interval.
|
||||
|
||||
This value controls how often (in seconds) the scheduler should attept
|
||||
to discover new hosts that have been added to cells. If negative (the
|
||||
default), no automatic discovery will occur.
|
||||
|
||||
Small deployments may want this periodic task enabled, as surveying the
|
||||
cells for new hosts is likely to be lightweight enough to not cause undue
|
||||
burdon to the scheduler. However, larger clouds (and those that are not
|
||||
adding hosts regularly) will likely want to disable this automatic
|
||||
behavior and instead use the `nova-manage cell_v2 discover_hosts` command
|
||||
when hosts have been added to a cell.
|
||||
"""),
|
||||
]
|
||||
|
||||
filter_scheduler_group = cfg.OptGroup(name="filter_scheduler",
|
||||
title="Filter scheduler options")
|
||||
|
||||
@ -12,9 +12,11 @@
|
||||
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
from nova import context
|
||||
from nova.db.sqlalchemy import api as db_api
|
||||
from nova.db.sqlalchemy import api_models
|
||||
from nova import exception
|
||||
from nova.i18n import _
|
||||
from nova.objects import base
|
||||
from nova.objects import cell_mapping
|
||||
from nova.objects import fields
|
||||
@ -163,3 +165,55 @@ class HostMappingList(base.ObjectListBase, base.NovaObject):
|
||||
def get_by_cell_id(cls, context, cell_id):
|
||||
db_mappings = cls._get_by_cell_id_from_db(context, cell_id)
|
||||
return base.obj_make_list(context, cls(), HostMapping, db_mappings)
|
||||
|
||||
|
||||
def discover_hosts(ctxt, cell_uuid=None, status_fn=None):
|
||||
# TODO(alaski): If this is not run on a host configured to use the API
|
||||
# database most of the lookups below will fail and may not provide a
|
||||
# great error message. Add a check which will raise a useful error
|
||||
# message about running this from an API host.
|
||||
|
||||
from nova import objects
|
||||
|
||||
if not status_fn:
|
||||
status_fn = lambda x: None
|
||||
|
||||
if cell_uuid:
|
||||
cell_mappings = [objects.CellMapping.get_by_uuid(ctxt, cell_uuid)]
|
||||
else:
|
||||
cell_mappings = objects.CellMappingList.get_all(ctxt)
|
||||
status_fn(_('Found %s cell mappings.') % len(cell_mappings))
|
||||
|
||||
host_mappings = []
|
||||
for cm in cell_mappings:
|
||||
if cm.is_cell0():
|
||||
status_fn(_('Skipping cell0 since it does not contain hosts.'))
|
||||
continue
|
||||
if 'name' in cm and cm.name:
|
||||
status_fn(_("Getting compute nodes from cell '%(name)s': "
|
||||
"%(uuid)s") % {'name': cm.name,
|
||||
'uuid': cm.uuid})
|
||||
else:
|
||||
status_fn(_("Getting compute nodes from cell: %(uuid)s") %
|
||||
{'uuid': cm.uuid})
|
||||
with context.target_cell(ctxt, cm):
|
||||
compute_nodes = objects.ComputeNodeList.get_all(ctxt)
|
||||
status_fn(_('Found %(num)s computes in cell: %(uuid)s') %
|
||||
{'num': len(compute_nodes),
|
||||
'uuid': cm.uuid})
|
||||
for compute in compute_nodes:
|
||||
status_fn(_("Checking host mapping for compute host "
|
||||
"'%(host)s': %(uuid)s") %
|
||||
{'host': compute.host, 'uuid': compute.uuid})
|
||||
try:
|
||||
objects.HostMapping.get_by_host(ctxt, compute.host)
|
||||
except exception.HostMappingNotFound:
|
||||
status_fn(_("Creating host mapping for compute host "
|
||||
"'%(host)s': %(uuid)s") %
|
||||
{'host': compute.host, 'uuid': compute.uuid})
|
||||
host_mapping = objects.HostMapping(
|
||||
ctxt, host=compute.host,
|
||||
cell_mapping=cm)
|
||||
host_mapping.create()
|
||||
host_mappings.append(host_mapping)
|
||||
return host_mappings
|
||||
|
||||
@ -27,8 +27,10 @@ from stevedore import driver
|
||||
|
||||
import nova.conf
|
||||
from nova import exception
|
||||
from nova.i18n import _LI
|
||||
from nova import manager
|
||||
from nova import objects
|
||||
from nova.objects import host_mapping as host_mapping_obj
|
||||
from nova import quota
|
||||
|
||||
|
||||
@ -60,6 +62,18 @@ class SchedulerManager(manager.Manager):
|
||||
def _expire_reservations(self, context):
|
||||
QUOTAS.expire(context)
|
||||
|
||||
@periodic_task.periodic_task(
|
||||
spacing=CONF.scheduler.discover_hosts_in_cells_interval,
|
||||
run_immediately=True)
|
||||
def _discover_hosts_in_cells(self, context):
|
||||
host_mappings = host_mapping_obj.discover_hosts(context)
|
||||
if host_mappings:
|
||||
LOG.info(_LI('Discovered %(count)i new hosts: %(hosts)s'),
|
||||
{'count': len(host_mappings),
|
||||
'hosts': ','.join(['%s:%s' % (hm.cell_mapping.name,
|
||||
hm.host)
|
||||
for hm in host_mappings])})
|
||||
|
||||
@periodic_task.periodic_task(spacing=CONF.scheduler.periodic_task_interval,
|
||||
run_immediately=True)
|
||||
def _run_periodic_tasks(self, context):
|
||||
|
||||
@ -12,11 +12,14 @@
|
||||
|
||||
import mock
|
||||
|
||||
from nova import context
|
||||
from nova import exception
|
||||
from nova import objects
|
||||
from nova.objects import host_mapping
|
||||
from nova import test
|
||||
from nova.tests.unit.objects import test_cell_mapping
|
||||
from nova.tests.unit.objects import test_objects
|
||||
from nova.tests import uuidsentinel as uuids
|
||||
|
||||
|
||||
def get_db_mapping(mapped_cell=None, **updates):
|
||||
@ -145,3 +148,62 @@ class TestHostMappingObject(test_objects._LocalTest,
|
||||
class TestRemoteHostMappingObject(test_objects._RemoteTest,
|
||||
_TestHostMappingObject):
|
||||
pass
|
||||
|
||||
|
||||
class TestHostMappingDiscovery(test.NoDBTestCase):
|
||||
@mock.patch('nova.objects.CellMappingList.get_all')
|
||||
@mock.patch('nova.objects.HostMapping.create')
|
||||
@mock.patch('nova.objects.HostMapping.get_by_host')
|
||||
@mock.patch('nova.objects.ComputeNodeList.get_all')
|
||||
def test_discover_hosts_all(self, mock_cn_get, mock_hm_get, mock_hm_create,
|
||||
mock_cm):
|
||||
def _hm_get(context, host):
|
||||
if host in ['a', 'b', 'c']:
|
||||
return objects.HostMapping()
|
||||
raise exception.HostMappingNotFound(name=host)
|
||||
|
||||
mock_hm_get.side_effect = _hm_get
|
||||
mock_cn_get.side_effect = [[objects.ComputeNode(host='d',
|
||||
uuid=uuids.cn1)],
|
||||
[objects.ComputeNode(host='e',
|
||||
uuid=uuids.cn2)]]
|
||||
|
||||
cell_mappings = [objects.CellMapping(name='foo',
|
||||
uuid=uuids.cm1),
|
||||
objects.CellMapping(name='bar',
|
||||
uuid=uuids.cm2)]
|
||||
mock_cm.return_value = cell_mappings
|
||||
ctxt = context.get_admin_context()
|
||||
hms = host_mapping.discover_hosts(ctxt)
|
||||
self.assertEqual(2, len(hms))
|
||||
self.assertTrue(mock_hm_create.called)
|
||||
self.assertEqual(['d', 'e'],
|
||||
[hm.host for hm in hms])
|
||||
|
||||
@mock.patch('nova.objects.CellMapping.get_by_uuid')
|
||||
@mock.patch('nova.objects.HostMapping.create')
|
||||
@mock.patch('nova.objects.HostMapping.get_by_host')
|
||||
@mock.patch('nova.objects.ComputeNodeList.get_all')
|
||||
def test_discover_hosts_one(self, mock_cn_get, mock_hm_get, mock_hm_create,
|
||||
mock_cm):
|
||||
def _hm_get(context, host):
|
||||
if host in ['a', 'b', 'c']:
|
||||
return objects.HostMapping()
|
||||
raise exception.HostMappingNotFound(name=host)
|
||||
|
||||
mock_hm_get.side_effect = _hm_get
|
||||
# NOTE(danms): Provide both side effects, but expect it to only
|
||||
# be called once if we provide a cell
|
||||
mock_cn_get.side_effect = [[objects.ComputeNode(host='d',
|
||||
uuid=uuids.cn1)],
|
||||
[objects.ComputeNode(host='e',
|
||||
uuid=uuids.cn2)]]
|
||||
|
||||
mock_cm.return_value = objects.CellMapping(name='foo',
|
||||
uuid=uuids.cm1)
|
||||
ctxt = context.get_admin_context()
|
||||
hms = host_mapping.discover_hosts(ctxt, uuids.cm1)
|
||||
self.assertEqual(1, len(hms))
|
||||
self.assertTrue(mock_hm_create.called)
|
||||
self.assertEqual(['d'],
|
||||
[hm.host for hm in hms])
|
||||
|
||||
@ -160,6 +160,16 @@ class SchedulerManagerTestCase(test.NoDBTestCase):
|
||||
mock.sentinel.host_name,
|
||||
mock.sentinel.instance_uuids)
|
||||
|
||||
@mock.patch('nova.objects.host_mapping.discover_hosts')
|
||||
def test_discover_hosts(self, mock_discover):
|
||||
cm1 = objects.CellMapping(name='cell1')
|
||||
cm2 = objects.CellMapping(name='cell2')
|
||||
mock_discover.return_value = [objects.HostMapping(host='a',
|
||||
cell_mapping=cm1),
|
||||
objects.HostMapping(host='b',
|
||||
cell_mapping=cm2)]
|
||||
self.manager._discover_hosts_in_cells(mock.sentinel.context)
|
||||
|
||||
|
||||
class SchedulerInitTestCase(test.NoDBTestCase):
|
||||
"""Test case for base scheduler driver initiation."""
|
||||
|
||||
@ -0,0 +1,13 @@
|
||||
---
|
||||
features:
|
||||
- As new hosts are added to Nova, the `nova-manage cell_v2
|
||||
discover_hosts` command must be run in order to map them into
|
||||
their cell. For deployments with proper automation, this is a
|
||||
trivial extra step in that process. However, for smaller or
|
||||
non-automated deployments, there is a new configuration variable
|
||||
for the scheduler process which will perform this discovery
|
||||
periodically. By setting
|
||||
`scheduler.discover_hosts_in_cells_interval` to a positive value,
|
||||
the scheduler will handle this for you. Note that this process
|
||||
involves listing all hosts in all cells, and is likely to be too
|
||||
heavyweight for large deployments to run all the time.
|
||||
Loading…
Reference in New Issue
Block a user