x/python-ganttclient
Code Issues Proposed changes

scheduler + unittests

Browse Source
This commit is contained in:
Vishvananda Ishaya
2010-09-02 11:32:37 -07:00
parent 75a3dacd3c
commit 063af4dfcd
10 changed files with 193 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bin/nova-scheduler
View File

@@ -21,12 +21,12 @@
Twistd daemon for the nova scheduler nodes.
"""
from nova import service
from nova import twistd
from nova.scheduler import service
if __name__ == '__main__':
twistd.serve(__file__)
if __name__ == '__builtin__':
application = service.SchedulerService.create()
application = service.Service.create()

6
nova/endpoint/cloud.py
View File

@@ -488,9 +488,9 @@ class CloudController(object):
host = db.network_get_host(context, network_ref['id'])
if not host:
host = yield rpc.call(FLAGS.network_topic,
{"method": "set_network_host",
"args": {"context": None,
"project_id": context.project.id}})
{"method": "set_network_host",
"args": {"context": None,
"project_id": context.project.id}})
defer.returnValue(db.queue_get_for(context, FLAGS.network_topic, host))
@rbac.allow('projectmanager', 'sysadmin')

2
nova/flags.py
View File

@@ -220,5 +220,7 @@ DEFINE_string('network_manager', 'nova.network.manager.VlanManager',
'Manager for network')
DEFINE_string('volume_manager', 'nova.volume.manager.AOEManager',
'Manager for volume')
DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager',
'Manager for scheduler')

80
nova/scheduler/base.py
View File

@@ -1,80 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2010 Openstack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Scheduler base class that all Schedulers should inherit from
"""
import time
from nova import flags
from nova.datastore import Redis
FLAGS = flags.FLAGS
flags.DEFINE_integer('node_down_time',
60,
'seconds without heartbeat that determines a '
'compute node to be down')
class Scheduler(object):
"""
The base class that all Scheduler clases should inherit from
"""
@staticmethod
def compute_nodes():
"""
Return a list of compute nodes
"""
return [identifier.split(':')[0]
for identifier in Redis.instance().smembers("daemons")
if (identifier.split(':')[1] == "nova-compute")]
@staticmethod
def compute_node_is_up(node):
"""
Given a node name, return whether the node is considered 'up' by
if it's sent a heartbeat recently
"""
time_str = Redis.instance().hget('%s:%s:%s' %
('daemon', node, 'nova-compute'),
'updated_at')
if not time_str:
return False
# Would be a lot easier if we stored heartbeat time in epoch :)
# The 'str()' here is to get rid of a pylint error
time_str = str(time_str).replace('Z', 'UTC')
time_split = time.strptime(time_str, '%Y-%m-%dT%H:%M:%S%Z')
epoch_time = int(time.mktime(time_split)) - time.timezone
return (time.time() - epoch_time) < FLAGS.node_down_time
def compute_nodes_up(self):
"""
Return the list of compute nodes that are considered 'up'
"""
return [node for node in self.compute_nodes()
if self.compute_node_is_up(node)]
def pick_node(self, instance_id, **_kwargs):
"""You DEFINITELY want to define this in your subclass"""
raise NotImplementedError("Your subclass should define pick_node")

34
nova/scheduler/bestfit.py
View File

@@ -1,34 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2010 Openstack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Best Fit Scheduler
"""
from nova.scheduler.base import Scheduler
class BestFitScheduler(Scheduler):
"""
Implements Scheduler as a best-fit node selector
"""
def pick_node(self, instance_id, **_kwargs):
"""
Picks a node that is up and is a best fit for the new instance
"""
raise NotImplementedError("BestFitScheduler is not done yet")

76
nova/scheduler/service.py
View File

@@ -1,76 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2010 Openstack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Scheduler Service
"""
import logging
from twisted.internet import defer
from nova import exception
from nova import flags
from nova import rpc
from nova import service
from nova.scheduler import chance
from nova.scheduler import bestfit
FLAGS = flags.FLAGS
flags.DEFINE_string('scheduler_type',
'chance',
'the scheduler to use')
SCHEDULER_CLASSES = {'chance': chance.ChanceScheduler,
'bestfit': bestfit.BestFitScheduler}
class SchedulerService(service.Service):
"""
Manages the running instances.
"""
def __init__(self):
super(SchedulerService, self).__init__()
if (FLAGS.scheduler_type not in SCHEDULER_CLASSES):
raise exception.Error("Scheduler '%s' does not exist" %
FLAGS.scheduler_type)
self._scheduler_class = SCHEDULER_CLASSES[FLAGS.scheduler_type]
@staticmethod
def noop():
""" simple test of an AMQP message call """
return defer.succeed('PONG')
def pick_node(self, instance_id, **_kwargs):
"""
Return a node to use based on the selected Scheduler
"""
return self._scheduler_class().pick_node(instance_id, **_kwargs)
@exception.wrap_exception
def run_instance(self, instance_id, **_kwargs):
"""
Picks a node for a running VM and casts the run_instance request
"""
node = self.pick_node(instance_id, **_kwargs)
rpc.cast('%s.%s' % (FLAGS.compute_topic, node),
{"method": "run_instance",
"args": {"instance_id": instance_id}})
logging.debug("Casting to node %s for running instance %s",
node, instance_id)

81
nova/scheduler/simple.py Normal file
View File

@@ -0,0 +1,81 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2010 Openstack, LLC.
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Simple Scheduler
"""
from nova import db
from nova import flags
from nova.scheduler import driver
FLAGS = flags.FLAGS
flags.DEFINE_integer("max_instances", 16,
"maximum number of instances to allow per host")
flags.DEFINE_integer("max_volumes", 100,
"maximum number of volumes to allow per host")
flags.DEFINE_integer("max_networks", 1000,
"maximum number of networks to allow per host")
class SimpleScheduler(driver.Scheduler):
"""
Implements Naive Scheduler that tries to find least loaded host
"""
def pick_compute_host(self, context, instance_id, **_kwargs):
"""
Picks a host that is up and has the fewest running instances
"""
results = db.daemon_get_all_compute_sorted(context)
for result in results:
(daemon, instance_count) = result
if instance_count >= FLAGS.max_instances:
raise driver.NoValidHost("All hosts have too many instances")
if self.daemon_is_up(daemon):
return daemon['host']
raise driver.NoValidHost("No hosts found")
def pick_volume_host(self, context, volume_id, **_kwargs):
"""
Picks a host that is up and has the fewest volumes
"""
results = db.daemon_get_all_volume_sorted(context)
for result in results:
(daemon, instance_count) = result
if instance_count >= FLAGS.max_volumes:
raise driver.NoValidHost("All hosts have too many volumes")
if self.daemon_is_up(daemon):
return daemon['host']
raise driver.NoValidHost("No hosts found")
def pick_network_host(self, context, network_id, **_kwargs):
"""
Picks a host that is up and has the fewest networks
"""
results = db.daemon_get_all_network_sorted(context)
for result in results:
(daemon, instance_count) = result
if instance_count >= FLAGS.max_networks:
raise driver.NoValidHost("All hosts have too many networks")
if self.daemon_is_up(daemon):
return daemon['host']
raise driver.NoValidHost("No hosts found")

2
nova/tests/compute_unittest.py
View File

@@ -61,7 +61,7 @@ class ComputeTestCase(test.TrialTestCase):
inst['instance_type'] = 'm1.tiny'
inst['mac_address'] = utils.generate_mac()
inst['ami_launch_index'] = 0
return db.instance_create(None, inst)
return db.instance_create(self.context, inst)
@defer.inlineCallbacks
def test_run_terminate(self):

103
nova/tests/scheduler_unittest.py Normal file
View File

@@ -0,0 +1,103 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Tests For Scheduler
"""
import logging
from twisted.internet import defer
from nova import db
from nova import flags
from nova import service
from nova import test
from nova import utils
from nova.auth import manager as auth_manager
from nova.scheduler import manager
FLAGS = flags.FLAGS
class SchedulerTestCase(test.TrialTestCase):
"""Test case for scheduler"""
def setUp(self): # pylint: disable-msg=C0103
super(SchedulerTestCase, self).setUp()
self.flags(connection_type='fake',
scheduler_driver='nova.scheduler.simple.SimpleScheduler')
self.scheduler = manager.SchedulerManager()
self.context = None
self.manager = auth_manager.AuthManager()
self.user = self.manager.create_user('fake', 'fake', 'fake')
self.project = self.manager.create_project('fake', 'fake', 'fake')
self.context = None
def tearDown(self): # pylint: disable-msg=C0103
self.manager.delete_user(self.user)
self.manager.delete_project(self.project)
def _create_instance(self):
"""Create a test instance"""
inst = {}
inst['image_id'] = 'ami-test'
inst['reservation_id'] = 'r-fakeres'
inst['launch_time'] = '10'
inst['user_id'] = self.user.id
inst['project_id'] = self.project.id
inst['instance_type'] = 'm1.tiny'
inst['mac_address'] = utils.generate_mac()
inst['ami_launch_index'] = 0
return db.instance_create(self.context, inst)
def test_hosts_are_up(self):
# NOTE(vish): constructing service without create method
# because we are going to use it without queue
service1 = service.Service('host1',
'nova-compute',
'compute',
FLAGS.compute_manager)
service2 = service.Service('host2',
'nova-compute',
'compute',
FLAGS.compute_manager)
service1.report_state()
service2.report_state()
hosts = self.scheduler.driver.hosts_up(self.context, 'compute')
self.assertEqual(len(hosts), 2)
def test_least_busy_host_gets_instance(self):
# NOTE(vish): constructing service without create method
# because we are going to use it without queue
service1 = service.Service('host1',
'nova-compute',
'compute',
FLAGS.compute_manager)
service2 = service.Service('host2',
'nova-compute',
'compute',
FLAGS.compute_manager)
service1.report_state()
service2.report_state()
instance_id = self._create_instance()
FLAGS.host = 'host1'
service1.run_instance(self.context,
instance_id)
print type(self.scheduler.driver)
host = self.scheduler.driver.pick_compute_host(self.context,
instance_id)
self.assertEqual(host, 'host2')

1
run_tests.py
View File

@@ -60,6 +60,7 @@ from nova.tests.network_unittest import *
from nova.tests.objectstore_unittest import *
from nova.tests.process_unittest import *
from nova.tests.rpc_unittest import *
from nova.tests.scheduler_unittest import *
from nova.tests.service_unittest import *
from nova.tests.validator_unittest import *
from nova.tests.volume_unittest import *