x/gantt
Code Issues Proposed changes

Created the filters directory in nova/scheduler

Browse Source
This commit is contained in:
Ed Leafe 2011-08-12 13:58:26 -05:00
parent 198ed7b54e
commit bc295c6cb1
12 changed files with 391 additions and 644 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nova/scheduler/__init__.py
View File

@ -21,5 +21,7 @@
.. automodule:: nova.scheduler
:platform: Unix
:synopsis: Module that picks a compute node to run a VM instance.
.. moduleauthor:: Sandy Walsh <sandy.walsh@rackspace.com>
.. moduleauthor:: Ed Leafe <ed@leafe.com>
.. moduleauthor:: Chris Behrens <cbehrens@codestud.com>
"""

30
nova/scheduler/abstract_scheduler.py
View File

@ -269,18 +269,13 @@ class AbstractScheduler(driver.Scheduler):
# Get all available hosts.
all_hosts = self.zone_manager.service_states.iteritems()
print "-"*88
ss = self.zone_manager.service_states
print ss
print "KEYS", ss.keys()
print "-"*88
unfiltered_hosts = [(host, services[host])
unfiltered_hosts = [(host, services[topic])
for host, services in all_hosts
if topic in services[host]]
if topic in services]
# Filter local hosts based on requirements ...
filtered_hosts = self.filter_hosts(topic, request_spec, host_list)
filtered_hosts = self.filter_hosts(topic, request_spec,
unfiltered_hosts)
if not filtered_hosts:
LOG.warn(_("No hosts available"))
return []
@ -307,22 +302,19 @@ class AbstractScheduler(driver.Scheduler):
weighted_hosts.sort(key=operator.itemgetter('weight'))
return weighted_hosts
def basic_ram_filter(self, hostname, capabilities, request_spec):
"""Return whether or not we can schedule to this compute node.
Derived classes should override this and return True if the host
is acceptable for scheduling.
"""
instance_type = request_spec['instance_type']
requested_mem = instance_type['memory_mb'] * 1024 * 1024
return capabilities['host_memory_free'] >= requested_mem
def filter_hosts(self, topic, request_spec, host_list=None):
def filter_hosts(self, topic, request_spec, host_list):
"""Filter the full host list returned from the ZoneManager. By default,
this method only applies the basic_ram_filter(), meaning all hosts
with at least enough RAM for the requested instance are returned.
Override in subclasses to provide greater selectivity.
"""
def basic_ram_filter(hostname, capabilities, request_spec):
"""Only return hosts with sufficient available RAM."""
instance_type = request_spec['instance_type']
requested_mem = instance_type['memory_mb'] * 1024 * 1024
return capabilities['host_memory_free'] >= requested_mem
return [(host, services) for host, services in host_list
if basic_ram_filter(host, services, request_spec)]

312
nova/scheduler/base_scheduler.py
View File

@ -20,324 +20,22 @@ across zones. There are two expansion points to this class for:
2. Filtering Hosts based on required instance capabilities
"""
import operator
import json
import M2Crypto
from novaclient import v1_1 as novaclient
from novaclient import exceptions as novaclient_exceptions
from nova import crypto
from nova import db
from nova import exception
from nova import flags
from nova import log as logging
from nova import rpc
from nova.compute import api as compute_api
from nova.scheduler import api
from nova.scheduler import driver
from nova.scheduler import abstract_scheduler
from nova.scheduler import host_filter
FLAGS = flags.FLAGS
LOG = logging.getLogger('nova.scheduler.abstract_scheduler')
LOG = logging.getLogger('nova.scheduler.base_scheduler')
class InvalidBlob(exception.NovaException):
message = _("Ill-formed or incorrectly routed 'blob' data sent "
"to instance create request.")
class AbstractScheduler(driver.Scheduler):
class BaseScheduler(abstract_scheduler.AbstractScheduler):
"""Base class for creating Schedulers that can work across any nova
deployment, from simple designs to multiply-nested zones.
"""
def _call_zone_method(self, context, method, specs, zones):
"""Call novaclient zone method. Broken out for testing."""
return api.call_zone_method(context, method, specs=specs, zones=zones)
def _provision_resource_locally(self, context, build_plan_item,
request_spec, kwargs):
"""Create the requested resource in this Zone."""
host = build_plan_item['hostname']
base_options = request_spec['instance_properties']
image = request_spec['image']
# TODO(sandy): I guess someone needs to add block_device_mapping
# support at some point? Also, OS API has no concept of security
# groups.
instance = compute_api.API().create_db_entry_for_new_instance(context,
image, base_options, None, [])
instance_id = instance['id']
kwargs['instance_id'] = instance_id
rpc.cast(context,
db.queue_get_for(context, "compute", host),
{"method": "run_instance",
"args": kwargs})
LOG.debug(_("Provisioning locally via compute node %(host)s")
% locals())
def _decrypt_blob(self, blob):
"""Returns the decrypted blob or None if invalid. Broken out
for testing."""
decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key)
try:
json_entry = decryptor(blob)
return json.dumps(json_entry)
except M2Crypto.EVP.EVPError:
pass
return None
def _ask_child_zone_to_create_instance(self, context, zone_info,
request_spec, kwargs):
"""Once we have determined that the request should go to one
of our children, we need to fabricate a new POST /servers/
call with the same parameters that were passed into us.
Note that we have to reverse engineer from our args to get back the
image, flavor, ipgroup, etc. since the original call could have
come in from EC2 (which doesn't use these things)."""
instance_type = request_spec['instance_type']
instance_properties = request_spec['instance_properties']
name = instance_properties['display_name']
image_ref = instance_properties['image_ref']
meta = instance_properties['metadata']
flavor_id = instance_type['flavorid']
reservation_id = instance_properties['reservation_id']
files = kwargs['injected_files']
ipgroup = None # Not supported in OS API ... yet
child_zone = zone_info['child_zone']
child_blob = zone_info['child_blob']
zone = db.zone_get(context, child_zone)
url = zone.api_url
LOG.debug(_("Forwarding instance create call to child zone %(url)s"
". ReservationID=%(reservation_id)s")
% locals())
nova = None
try:
nova = novaclient.Client(zone.username, zone.password, None, url)
nova.authenticate()
except novaclient_exceptions.BadRequest, e:
raise exception.NotAuthorized(_("Bad credentials attempting "
"to talk to zone at %(url)s.") % locals())
nova.servers.create(name, image_ref, flavor_id, ipgroup, meta, files,
child_blob, reservation_id=reservation_id)
def _provision_resource_from_blob(self, context, build_plan_item,
instance_id, request_spec, kwargs):
"""Create the requested resource locally or in a child zone
based on what is stored in the zone blob info.
Attempt to decrypt the blob to see if this request is:
1. valid, and
2. intended for this zone or a child zone.
Note: If we have "blob" that means the request was passed
into us from a parent zone. If we have "child_blob" that
means we gathered the info from one of our children.
It's possible that, when we decrypt the 'blob' field, it
contains "child_blob" data. In which case we forward the
request."""
host_info = None
if "blob" in build_plan_item:
# Request was passed in from above. Is it for us?
host_info = self._decrypt_blob(build_plan_item['blob'])
elif "child_blob" in build_plan_item:
# Our immediate child zone provided this info ...
host_info = build_plan_item
if not host_info:
raise InvalidBlob()
# Valid data ... is it for us?
if 'child_zone' in host_info and 'child_blob' in host_info:
self._ask_child_zone_to_create_instance(context, host_info,
request_spec, kwargs)
else:
self._provision_resource_locally(context, host_info, request_spec,
kwargs)
def _provision_resource(self, context, build_plan_item, instance_id,
request_spec, kwargs):
"""Create the requested resource in this Zone or a child zone."""
if "hostname" in build_plan_item:
self._provision_resource_locally(context, build_plan_item,
request_spec, kwargs)
return
self._provision_resource_from_blob(context, build_plan_item,
instance_id, request_spec, kwargs)
def _adjust_child_weights(self, child_results, zones):
"""Apply the Scale and Offset values from the Zone definition
to adjust the weights returned from the child zones. Alters
child_results in place.
"""
for zone_id, result in child_results:
if not result:
continue
assert isinstance(zone_id, int)
for zone_rec in zones:
if zone_rec['id'] != zone_id:
continue
for item in result:
try:
offset = zone_rec['weight_offset']
scale = zone_rec['weight_scale']
raw_weight = item['weight']
cooked_weight = offset + scale * raw_weight
item['weight'] = cooked_weight
item['raw_weight'] = raw_weight
except KeyError:
LOG.exception(_("Bad child zone scaling values "
"for Zone: %(zone_id)s") % locals())
def schedule_run_instance(self, context, instance_id, request_spec,
*args, **kwargs):
"""This method is called from nova.compute.api to provision
an instance. However we need to look at the parameters being
passed in to see if this is a request to:
1. Create a Build Plan and then provision, or
2. Use the Build Plan information in the request parameters
to simply create the instance (either in this zone or
a child zone).
"""
# TODO(sandy): We'll have to look for richer specs at some point.
blob = request_spec.get('blob')
if blob:
self._provision_resource(context, request_spec, instance_id,
request_spec, kwargs)
return None
num_instances = request_spec.get('num_instances', 1)
LOG.debug(_("Attempting to build %(num_instances)d instance(s)") %
locals())
# Create build plan and provision ...
build_plan = self.select(context, request_spec)
if not build_plan:
raise driver.NoValidHost(_('No hosts were available'))
for num in xrange(num_instances):
if not build_plan:
break
build_plan_item = build_plan.pop(0)
self._provision_resource(context, build_plan_item, instance_id,
request_spec, kwargs)
# Returning None short-circuits the routing to Compute (since
# we've already done it here)
return None
def select(self, context, request_spec, *args, **kwargs):
"""Select returns a list of weights and zone/host information
corresponding to the best hosts to service the request. Any
child zone information has been encrypted so as not to reveal
anything about the children.
"""
return self._schedule(context, "compute", request_spec,
*args, **kwargs)
# TODO(sandy): We're only focused on compute instances right now,
# so we don't implement the default "schedule()" method required
# of Schedulers.
def schedule(self, context, topic, request_spec, *args, **kwargs):
"""The schedule() contract requires we return the one
best-suited host for this request.
"""
raise driver.NoValidHost(_('No hosts were available'))
def _schedule(self, context, topic, request_spec, *args, **kwargs):
"""Returns a list of hosts that meet the required specs,
ordered by their fitness.
"""
if topic != "compute":
raise NotImplementedError(_("Scheduler only understands"
" Compute nodes (for now)"))
num_instances = request_spec.get('num_instances', 1)
instance_type = request_spec['instance_type']
weighted = []
host_list = None
for i in xrange(num_instances):
# Filter local hosts based on requirements ...
#
# The first pass through here will pass 'None' as the
# host_list.. which tells the filter to build the full
# list of hosts.
# On a 2nd pass, the filter can modify the host_list with
# any updates it needs to make based on resources that
# may have been consumed from a previous build..
host_list = self.filter_hosts(topic, request_spec, host_list)
if not host_list:
LOG.warn(_("Filter returned no hosts after processing "
"%(i)d of %(num_instances)d instances") % locals())
break
# then weigh the selected hosts.
# weighted = [{weight=weight, hostname=hostname,
# capabilities=capabs}, ...]
weights = self.weigh_hosts(topic, request_spec, host_list)
weights.sort(key=operator.itemgetter('weight'))
best_weight = weights[0]
weighted.append(best_weight)
self.consume_resources(topic, best_weight['capabilities'],
instance_type)
# Next, tack on the best weights from the child zones ...
json_spec = json.dumps(request_spec)
all_zones = db.zone_get_all(context)
child_results = self._call_zone_method(context, "select",
specs=json_spec, zones=all_zones)
self._adjust_child_weights(child_results, all_zones)
for child_zone, result in child_results:
for weighting in result:
# Remember the child_zone so we can get back to
# it later if needed. This implicitly builds a zone
# path structure.
host_dict = {"weight": weighting["weight"],
"child_zone": child_zone,
"child_blob": weighting["blob"]}
weighted.append(host_dict)
weighted.sort(key=operator.itemgetter('weight'))
return weighted
def compute_filter(self, hostname, capabilities, request_spec):
"""Return whether or not we can schedule to this compute node.
Derived classes should override this and return True if the host
is acceptable for scheduling.
"""
instance_type = request_spec['instance_type']
requested_mem = instance_type['memory_mb'] * 1024 * 1024
return capabilities['host_memory_free'] >= requested_mem
def hold_filter_hosts(self, topic, request_spec, hosts=None):
def filter_hosts(self, topic, request_spec, hosts=None):
"""Filter the full host list (from the ZoneManager)"""
# NOTE(dabo): The logic used by the current _schedule() method
# is incorrect. Since this task is just to refactor the classes,
# I'm not fixing the logic now - that will be the next task.
# So for now this method is just renamed; afterwards this will
# become the filter_hosts() method, and the one below will
# be removed.
filter_name = request_spec.get('filter', None)
# Make sure that the requested filter is legitimate.
selected_filter = host_filter.choose_host_filter(filter_name)

18
nova/scheduler/filters/__init__.py Normal file
View File

@ -0,0 +1,18 @@
# Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from all_hosts_filter import AllHostsFilter
from instance_type_filter import InstanceTypeFilter
from json_filter import JsonFilter

87
nova/scheduler/filters/abstract_filter.py Normal file
View File

@ -0,0 +1,87 @@
# Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
The Host Filter classes are a way to ensure that only hosts that are
appropriate are considered when creating a new instance. Hosts that are
either incompatible or insufficient to accept a newly-requested instance
are removed by Host Filter classes from consideration. Those that pass
the filter are then passed on for weighting or other process for ordering.
Three filters are included: AllHosts, Flavor & JSON. AllHosts just
returns the full, unfiltered list of hosts. Flavor is a hard coded
matching mechanism based on flavor criteria and JSON is an ad-hoc
filter grammar.
Why JSON? The requests for instances may come in through the
REST interface from a user or a parent Zone.
Currently Flavors and/or InstanceTypes are used for
specifing the type of instance desired. Specific Nova users have
noted a need for a more expressive way of specifying instances.
Since we don't want to get into building full DSL this is a simple
form as an example of how this could be done. In reality, most
consumers will use the more rigid filters such as FlavorFilter.
"""
import json
from nova import exception
from nova import flags
from nova import log as logging
import nova.scheduler
LOG = logging.getLogger('nova.scheduler.host_filter')
FLAGS = flags.FLAGS
flags.DEFINE_string('default_host_filter',
'nova.scheduler.host_filter.AllHostsFilter',
'Which filter to use for filtering hosts')
class AbstractHostFilter(object):
"""Base class for host filters."""
def instance_type_to_filter(self, instance_type):
"""Convert instance_type into a filter for most common use-case."""
raise NotImplementedError()
def filter_hosts(self, zone_manager, query):
"""Return a list of hosts that fulfill the filter."""
raise NotImplementedError()
def _full_name(self):
"""module.classname of the filter."""
return "%s.%s" % (self.__module__, self.__class__.__name__)
def _get_filters():
from nova.scheduler import filters
return [itm for itm in dir(filters)
if issubclass(itm, AbstractHostFilter)]
def choose_host_filter(filter_name=None):
"""Since the caller may specify which filter to use we need
to have an authoritative list of what is permissible. This
function checks the filter name against a predefined set
of acceptable filters.
"""
if not filter_name:
filter_name = FLAGS.default_host_filter
for filter_class in _get_filters():
host_match = "%s.%s" % (filter_class.__module__, filter_class.__name__)
if host_match == filter_name:
return filter_class()
raise exception.SchedulerHostFilterNotFound(filter_name=filter_name)

31
nova/scheduler/filters/all_hosts_filter.py Normal file
View File

@ -0,0 +1,31 @@
# Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import nova.scheduler
class AllHostsFilter(nova.scheduler.host_filter.AbstractHostFilter):
"""NOP host filter. Returns all hosts in ZoneManager."""
def instance_type_to_filter(self, instance_type):
"""Return anything to prevent base-class from raising
exception.
"""
return (self._full_name(), instance_type)
def filter_hosts(self, zone_manager, query):
"""Return a list of hosts from ZoneManager list."""
return [(host, services)
for host, services in zone_manager.service_states.iteritems()]

86
nova/scheduler/filters/instance_type_filter.py Normal file
View File

@ -0,0 +1,86 @@
# Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from nova.scheduler import host_filter
class InstanceTypeFilter(host_filter.AbstractHostFilter):
"""HostFilter hard-coded to work with InstanceType records."""
def instance_type_to_filter(self, instance_type):
"""Use instance_type to filter hosts."""
return (self._full_name(), instance_type)
def _satisfies_extra_specs(self, capabilities, instance_type):
"""Check that the capabilities provided by the compute service
satisfy the extra specs associated with the instance type"""
if 'extra_specs' not in instance_type:
return True
# NOTE(lorinh): For now, we are just checking exact matching on the
# values. Later on, we want to handle numerical
# values so we can represent things like number of GPU cards
try:
for key, value in instance_type['extra_specs'].iteritems():
if capabilities[key] != value:
return False
except KeyError:
return False
return True
def filter_hosts(self, zone_manager, query):
"""Return a list of hosts that can create instance_type."""
instance_type = query
selected_hosts = []
for host, services in zone_manager.service_states.iteritems():
capabilities = services.get('compute', {})
if not capabilities:
continue
host_ram_mb = capabilities['host_memory_free']
disk_bytes = capabilities['disk_available']
spec_ram = instance_type['memory_mb']
spec_disk = instance_type['local_gb']
extra_specs = instance_type['extra_specs']
if ((host_ram_mb >= spec_ram) and (disk_bytes >= spec_disk) and
self._satisfies_extra_specs(capabilities, instance_type)):
selected_hosts.append((host, capabilities))
return selected_hosts
# host entries (currently) are like:
# {'host_name-description': 'Default install of XenServer',
# 'host_hostname': 'xs-mini',
# 'host_memory_total': 8244539392,
# 'host_memory_overhead': 184225792,
# 'host_memory_free': 3868327936,
# 'host_memory_free_computed': 3840843776,
# 'host_other_config': {},
# 'host_ip_address': '192.168.1.109',
# 'host_cpu_info': {},
# 'disk_available': 32954957824,
# 'disk_total': 50394562560,
# 'disk_used': 17439604736,
# 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f',
# 'host_name_label': 'xs-mini'}
# instance_type table has:
# name = Column(String(255), unique=True)
# memory_mb = Column(Integer)
# vcpus = Column(Integer)
# local_gb = Column(Integer)
# flavorid = Column(Integer, unique=True)
# swap = Column(Integer, nullable=False, default=0)
# rxtx_quota = Column(Integer, nullable=False, default=0)
# rxtx_cap = Column(Integer, nullable=False, default=0)

141
nova/scheduler/filters/json_filter.py Normal file
View File

@ -0,0 +1,141 @@
# Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import operator
from nova.scheduler import host_filter
class JsonFilter(host_filter.AbstractHostFilter):
"""Host Filter to allow simple JSON-based grammar for
selecting hosts.
"""
def _op_comp(self, args, op):
"""Returns True if the specified operator can successfully
compare the first item in the args with all the rest. Will
return False if only one item is in the list.
"""
if len(args) < 2:
return False
bad = [arg for arg in args[1:]
if not op(args[0], arg)]
return not bool(bad)
def _equals(self, args):
"""First term is == all the other terms."""
return self._op_comp(args, operator.eq)
def _less_than(self, args):
"""First term is < all the other terms."""
return self._op_comp(args, operator.lt)
def _greater_than(self, args):
"""First term is > all the other terms."""
return self._op_comp(args, operator.gt)
def _in(self, args):
"""First term is in set of remaining terms"""
return self._op_comp(args, operator.contains)
def _less_than_equal(self, args):
"""First term is <= all the other terms."""
return self._op_comp(args, operator.le)
def _greater_than_equal(self, args):
"""First term is >= all the other terms."""
return self._op_comp(args, operator.ge)
def _not(self, args):
"""Flip each of the arguments."""
return [not arg for arg in args]
def _or(self, args):
"""True if any arg is True."""
return any(args)
def _and(self, args):
"""True if all args are True."""
return all(args)
commands = {
'=': _equals,
'<': _less_than,
'>': _greater_than,
'in': _in,
'<=': _less_than_equal,
'>=': _greater_than_equal,
'not': _not,
'or': _or,
'and': _and,
}
def instance_type_to_filter(self, instance_type):
"""Convert instance_type into JSON filter object."""
required_ram = instance_type['memory_mb']
required_disk = instance_type['local_gb']
query = ['and',
['>=', '$compute.host_memory_free', required_ram],
['>=', '$compute.disk_available', required_disk]]
return (self._full_name(), json.dumps(query))
def _parse_string(self, string, host, services):
"""Strings prefixed with $ are capability lookups in the
form '$service.capability[.subcap*]'.
"""
if not string:
return None
if not string.startswith("$"):
return string
path = string[1:].split(".")
for item in path:
services = services.get(item, None)
if not services:
return None
return services
def _process_filter(self, zone_manager, query, host, services):
"""Recursively parse the query structure."""
if not query:
return True
cmd = query[0]
method = self.commands[cmd]
cooked_args = []
for arg in query[1:]:
if isinstance(arg, list):
arg = self._process_filter(zone_manager, arg, host, services)
elif isinstance(arg, basestring):
arg = self._parse_string(arg, host, services)
if arg is not None:
cooked_args.append(arg)
result = method(self, cooked_args)
return result
def filter_hosts(self, zone_manager, query):
"""Return a list of hosts that can fulfill the requirements
specified in the query.
"""
expanded = json.loads(query)
filtered_hosts = []
for host, services in zone_manager.service_states.iteritems():
result = self._process_filter(zone_manager, expanded, host,
services)
if isinstance(result, list):
# If any succeeded, include the host
result = any(result)
if result:
filtered_hosts.append((host, services))
return filtered_hosts

314
nova/scheduler/host_filter.py
View File

@ -1,314 +0,0 @@
# Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
The Host Filter classes are a way to ensure that only hosts that are
appropriate are considered when creating a new instance. Hosts that are
either incompatible or insufficient to accept a newly-requested instance
are removed by Host Filter classes from consideration. Those that pass
the filter are then passed on for weighting or other process for ordering.
Three filters are included: AllHosts, Flavor & JSON. AllHosts just
returns the full, unfiltered list of hosts. Flavor is a hard coded
matching mechanism based on flavor criteria and JSON is an ad-hoc
filter grammar.
Why JSON? The requests for instances may come in through the
REST interface from a user or a parent Zone.
Currently Flavors and/or InstanceTypes are used for
specifing the type of instance desired. Specific Nova users have
noted a need for a more expressive way of specifying instances.
Since we don't want to get into building full DSL this is a simple
form as an example of how this could be done. In reality, most
consumers will use the more rigid filters such as FlavorFilter.
"""
import json
from nova import exception
from nova import flags
from nova import log as logging
from nova import utils
LOG = logging.getLogger('nova.scheduler.host_filter')
FLAGS = flags.FLAGS
flags.DEFINE_string('default_host_filter',
'nova.scheduler.host_filter.AllHostsFilter',
'Which filter to use for filtering hosts.')
class HostFilter(object):
"""Base class for host filters."""
def instance_type_to_filter(self, instance_type):
"""Convert instance_type into a filter for most common use-case."""
raise NotImplementedError()
def filter_hosts(self, zone_manager, query):
"""Return a list of hosts that fulfill the filter."""
raise NotImplementedError()
def _full_name(self):
"""module.classname of the filter."""
return "%s.%s" % (self.__module__, self.__class__.__name__)
class AllHostsFilter(HostFilter):
""" NOP host filter. Returns all hosts in ZoneManager.
This essentially does what the old Scheduler+Chance used
to give us.
"""
def instance_type_to_filter(self, instance_type):
"""Return anything to prevent base-class from raising
exception."""
return (self._full_name(), instance_type)
def filter_hosts(self, zone_manager, query):
"""Return a list of hosts from ZoneManager list."""
return [(host, services)
for host, services in zone_manager.service_states.iteritems()]
class InstanceTypeFilter(HostFilter):
"""HostFilter hard-coded to work with InstanceType records."""
def instance_type_to_filter(self, instance_type):
"""Use instance_type to filter hosts."""
return (self._full_name(), instance_type)
def _satisfies_extra_specs(self, capabilities, instance_type):
"""Check that the capabilities provided by the compute service
satisfy the extra specs associated with the instance type"""
if 'extra_specs' not in instance_type:
return True
# Note(lorinh): For now, we are just checking exact matching on the
# values. Later on, we want to handle numerical
# values so we can represent things like number of GPU cards
try:
for key, value in instance_type['extra_specs'].iteritems():
if capabilities[key] != value:
return False
except KeyError:
return False
return True
def filter_hosts(self, zone_manager, query):
"""Return a list of hosts that can create instance_type."""
instance_type = query
selected_hosts = []
for host, services in zone_manager.service_states.iteritems():
capabilities = services.get('compute', {})
host_ram_mb = capabilities['host_memory_free']
disk_bytes = capabilities['disk_available']
spec_ram = instance_type['memory_mb']
spec_disk = instance_type['local_gb']
extra_specs = instance_type['extra_specs']
if ((host_ram_mb >= spec_ram) and (disk_bytes >= spec_disk) and
self._satisfies_extra_specs(capabilities, instance_type)):
selected_hosts.append((host, capabilities))
return selected_hosts
#host entries (currently) are like:
# {'host_name-description': 'Default install of XenServer',
# 'host_hostname': 'xs-mini',
# 'host_memory_total': 8244539392,
# 'host_memory_overhead': 184225792,
# 'host_memory_free': 3868327936,
# 'host_memory_free_computed': 3840843776,
# 'host_other_config': {},
# 'host_ip_address': '192.168.1.109',
# 'host_cpu_info': {},
# 'disk_available': 32954957824,
# 'disk_total': 50394562560,
# 'disk_used': 17439604736,
# 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f',
# 'host_name_label': 'xs-mini'}
# instance_type table has:
#name = Column(String(255), unique=True)
#memory_mb = Column(Integer)
#vcpus = Column(Integer)
#local_gb = Column(Integer)
#flavorid = Column(Integer, unique=True)
#swap = Column(Integer, nullable=False, default=0)
#rxtx_quota = Column(Integer, nullable=False, default=0)
#rxtx_cap = Column(Integer, nullable=False, default=0)
class JsonFilter(HostFilter):
"""Host Filter to allow simple JSON-based grammar for
selecting hosts.
"""
def _equals(self, args):
"""First term is == all the other terms."""
if len(args) < 2:
return False
lhs = args[0]
for rhs in args[1:]:
if lhs != rhs:
return False
return True
def _less_than(self, args):
"""First term is < all the other terms."""
if len(args) < 2:
return False
lhs = args[0]
for rhs in args[1:]:
if lhs >= rhs:
return False
return True
def _greater_than(self, args):
"""First term is > all the other terms."""
if len(args) < 2:
return False
lhs = args[0]
for rhs in args[1:]:
if lhs <= rhs:
return False
return True
def _in(self, args):
"""First term is in set of remaining terms"""
if len(args) < 2:
return False
return args[0] in args[1:]
def _less_than_equal(self, args):
"""First term is <= all the other terms."""
if len(args) < 2:
return False
lhs = args[0]
for rhs in args[1:]:
if lhs > rhs:
return False
return True
def _greater_than_equal(self, args):
"""First term is >= all the other terms."""
if len(args) < 2:
return False
lhs = args[0]
for rhs in args[1:]:
if lhs < rhs:
return False
return True
def _not(self, args):
"""Flip each of the arguments."""
if len(args) == 0:
return False
return [not arg for arg in args]
def _or(self, args):
"""True if any arg is True."""
return True in args
def _and(self, args):
"""True if all args are True."""
return False not in args
commands = {
'=': _equals,
'<': _less_than,
'>': _greater_than,
'in': _in,
'<=': _less_than_equal,
'>=': _greater_than_equal,
'not': _not,
'or': _or,
'and': _and,
}
def instance_type_to_filter(self, instance_type):
"""Convert instance_type into JSON filter object."""
required_ram = instance_type['memory_mb']
required_disk = instance_type['local_gb']
query = ['and',
['>=', '$compute.host_memory_free', required_ram],
['>=', '$compute.disk_available', required_disk]]
return (self._full_name(), json.dumps(query))
def _parse_string(self, string, host, services):
"""Strings prefixed with $ are capability lookups in the
form '$service.capability[.subcap*]'
"""
if not string:
return None
if string[0] != '$':
return string
path = string[1:].split('.')
for item in path:
services = services.get(item, None)
if not services:
return None
return services
def _process_filter(self, zone_manager, query, host, services):
"""Recursively parse the query structure."""
if len(query) == 0:
return True
cmd = query[0]
method = self.commands[cmd] # Let exception fly.
cooked_args = []
for arg in query[1:]:
if isinstance(arg, list):
arg = self._process_filter(zone_manager, arg, host, services)
elif isinstance(arg, basestring):
arg = self._parse_string(arg, host, services)
if arg != None:
cooked_args.append(arg)
result = method(self, cooked_args)
return result
def filter_hosts(self, zone_manager, query):
"""Return a list of hosts that can fulfill filter."""
expanded = json.loads(query)
hosts = []
for host, services in zone_manager.service_states.iteritems():
r = self._process_filter(zone_manager, expanded, host, services)
if isinstance(r, list):
r = True in r
if r:
hosts.append((host, services))
return hosts
FILTERS = [AllHostsFilter, InstanceTypeFilter, JsonFilter]
def choose_host_filter(filter_name=None):
"""Since the caller may specify which filter to use we need
to have an authoritative list of what is permissible. This
function checks the filter name against a predefined set
of acceptable filters.
"""
if not filter_name:
filter_name = FLAGS.default_host_filter
for filter_class in FILTERS:
host_match = "%s.%s" % (filter_class.__module__, filter_class.__name__)
if host_match == filter_name:
return filter_class()
raise exception.SchedulerHostFilterNotFound(filter_name=filter_name)

3
nova/tests/scheduler/test_abstract_scheduler.py
View File

@ -77,6 +77,9 @@ class FakeZoneManager(zone_manager.ZoneManager):
'host3': {
'compute': {'host_memory_free': 3221225472},
},
'host4': {
'compute': {'host_memory_free': 999999999},
},
}

4
nova/tests/scheduler/test_host_filter.py
View File

@ -20,7 +20,7 @@ import json
from nova import exception
from nova import test
from nova.scheduler import host_filter
from nova.scheduler import filters
class FakeZoneManager:
@ -55,7 +55,7 @@ class HostFilterTestCase(test.TestCase):
def setUp(self):
super(HostFilterTestCase, self).setUp()
default_host_filter = 'nova.scheduler.host_filter.AllHostsFilter'
default_host_filter = 'nova.scheduler.filteris.AllHostsFilter'
self.flags(default_host_filter=default_host_filter)
self.instance_type = dict(name='tiny',
memory_mb=50,

7
nova/tests/scheduler/test_least_cost_scheduler.py
View File

@ -122,11 +122,14 @@ class LeastCostSchedulerTestCase(test.TestCase):
self.flags(least_cost_scheduler_cost_functions=[
'nova.scheduler.least_cost.compute_fill_first_cost_fn'],
compute_fill_first_cost_fn_weight=1)
num = 1
instance_type = {'memory_mb': 1024}
request_spec = {'instance_type': instance_type}
hosts = self.sched.filter_hosts('compute', request_spec, None)
all_hosts = self.sched.zone_manager.service_states.iteritems()
all_hosts = [(host, services["compute"])
for host, services in all_hosts
if "compute" in services]
hosts = self.sched.filter_hosts('compute', request_spec, host_list)
expected = []
for idx, (hostname, caps) in enumerate(hosts):