OpenStack Block Storage (Cinder)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

645 lines
27 KiB

# Copyright (c) 2010 OpenStack Foundation
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Scheduler Service
"""
import collections
from datetime import datetime
import functools
import eventlet
from oslo_config import cfg
from oslo_log import log as logging
import oslo_messaging as messaging
from oslo_service import periodic_task
from oslo_utils import excutils
from oslo_utils import importutils
from oslo_utils import timeutils
from oslo_utils import versionutils
from cinder.backup import rpcapi as backup_rpcapi
from cinder import context
from cinder import db
from cinder import exception
from cinder import flow_utils
from cinder.i18n import _
from cinder import manager
from cinder.message import api as mess_api
from cinder.message import message_field
from cinder import objects
from cinder.objects import fields
from cinder import quota
from cinder import rpc
from cinder.scheduler.flows import create_volume
from cinder.scheduler import rpcapi as scheduler_rpcapi
from cinder.volume import rpcapi as volume_rpcapi
from cinder.volume import volume_utils as vol_utils
scheduler_manager_opts = [
cfg.StrOpt('scheduler_driver',
default='cinder.scheduler.filter_scheduler.'
'FilterScheduler',
help='Default scheduler driver to use'),
cfg.IntOpt('scheduler_driver_init_wait_time',
default=60,
min=1,
help='Maximum time in seconds to wait for the driver to '
'report as ready'),
]
CONF = cfg.CONF
CONF.register_opts(scheduler_manager_opts)
QUOTAS = quota.QUOTAS
LOG = logging.getLogger(__name__)
def append_operation_type(name=None):
def _decorator(schedule_function):
@functools.wraps(schedule_function)
def inject_operation_decorator(*args, **kwargs):
request_spec = kwargs.get('request_spec', None)
request_spec_list = kwargs.get('request_spec_list', None)
if request_spec:
request_spec['operation'] = name or schedule_function.__name__
if request_spec_list:
for rs in request_spec_list:
rs['operation'] = name or schedule_function.__name__
return schedule_function(*args, **kwargs)
return inject_operation_decorator
return _decorator
class SchedulerManager(manager.CleanableManager, manager.Manager):
"""Chooses a host to create volumes."""
RPC_API_VERSION = scheduler_rpcapi.SchedulerAPI.RPC_API_VERSION
target = messaging.Target(version=RPC_API_VERSION)
def __init__(self, scheduler_driver=None, service_name=None,
*args, **kwargs):
if not scheduler_driver:
scheduler_driver = CONF.scheduler_driver
self.driver = importutils.import_object(scheduler_driver)
super(SchedulerManager, self).__init__(*args, **kwargs)
self._startup_delay = True
self.backup_api = backup_rpcapi.BackupAPI()
self.volume_api = volume_rpcapi.VolumeAPI()
self.sch_api = scheduler_rpcapi.SchedulerAPI()
self.message_api = mess_api.API()
self.rpc_api_version = versionutils.convert_version_to_int(
self.RPC_API_VERSION)
def init_host_with_rpc(self):
ctxt = context.get_admin_context()
self.request_service_capabilities(ctxt)
for __ in range(CONF.scheduler_driver_init_wait_time):
if self.driver.is_first_receive():
break
eventlet.sleep(1)
self._startup_delay = False
def reset(self):
super(SchedulerManager, self).reset()
self.volume_api = volume_rpcapi.VolumeAPI()
self.sch_api = scheduler_rpcapi.SchedulerAPI()
self.driver.reset()
@periodic_task.periodic_task(spacing=CONF.message_reap_interval,
run_immediately=True)
def _clean_expired_messages(self, context):
self.message_api.cleanup_expired_messages(context)
@periodic_task.periodic_task(spacing=CONF.reservation_clean_interval,
run_immediately=True)
def _clean_expired_reservation(self, context):
QUOTAS.expire(context)
def update_service_capabilities(self, context, service_name=None,
host=None, capabilities=None,
cluster_name=None, timestamp=None,
**kwargs):
"""Process a capability update from a service node."""
if capabilities is None:
capabilities = {}
# If we received the timestamp we have to deserialize it
elif timestamp:
timestamp = datetime.strptime(timestamp,
timeutils.PERFECT_TIME_FORMAT)
self.driver.update_service_capabilities(service_name,
host,
capabilities,
cluster_name,
timestamp)
def notify_service_capabilities(self, context, service_name,
capabilities, host=None, backend=None,
timestamp=None):
"""Process a capability update from a service node."""
# TODO(geguileo): On v4 remove host field.
if capabilities is None:
capabilities = {}
# If we received the timestamp we have to deserialize it
elif timestamp:
timestamp = datetime.strptime(timestamp,
timeutils.PERFECT_TIME_FORMAT)
backend = backend or host
self.driver.notify_service_capabilities(service_name,
backend,
capabilities,
timestamp)
def _wait_for_scheduler(self):
# NOTE(dulek): We're waiting for scheduler to announce that it's ready
# or CONF.scheduler_driver_init_wait_time seconds from service startup
# has passed.
while self._startup_delay and not self.driver.is_ready():
eventlet.sleep(1)
@append_operation_type()
def create_group(self, context, group, group_spec=None,
group_filter_properties=None, request_spec_list=None,
filter_properties_list=None):
self._wait_for_scheduler()
try:
self.driver.schedule_create_group(
context, group,
group_spec,
request_spec_list,
group_filter_properties,
filter_properties_list)
except exception.NoValidBackend:
LOG.error("Could not find a backend for group "
"%(group_id)s.",
{'group_id': group.id})
group.status = fields.GroupStatus.ERROR
group.save()
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception("Failed to create generic group "
"%(group_id)s.",
{'group_id': group.id})
group.status = fields.GroupStatus.ERROR
group.save()
@objects.Volume.set_workers
@append_operation_type()
def create_volume(self, context, volume, snapshot_id=None, image_id=None,
request_spec=None, filter_properties=None,
backup_id=None):
self._wait_for_scheduler()
try:
flow_engine = create_volume.get_flow(context,
self.driver,
request_spec,
filter_properties,
volume,
snapshot_id,
image_id,
backup_id)
except Exception:
msg = _("Failed to create scheduler manager volume flow")
LOG.exception(msg)
raise exception.CinderException(msg)
with flow_utils.DynamicLogListener(flow_engine, logger=LOG):
flow_engine.run()
@append_operation_type()
def create_snapshot(self, ctxt, volume, snapshot, backend,
request_spec=None, filter_properties=None):
"""Create snapshot for a volume.
The main purpose of this method is to check if target
backend (of volume and snapshot) has sufficient capacity
to host to-be-created snapshot.
"""
self._wait_for_scheduler()
try:
tgt_backend = self.driver.backend_passes_filters(
ctxt, backend, request_spec, filter_properties)
tgt_backend.consume_from_volume(
{'size': request_spec['volume_properties']['size']})
except exception.NoValidBackend as ex:
self._set_snapshot_state_and_notify('create_snapshot',
snapshot,
fields.SnapshotStatus.ERROR,
ctxt, ex, request_spec)
else:
volume_rpcapi.VolumeAPI().create_snapshot(ctxt, volume,
snapshot)
def _do_cleanup(self, ctxt, vo_resource):
# We can only receive cleanup requests for volumes, but we check anyway
# We need to cleanup the volume status for cases where the scheduler
# died while scheduling the volume creation.
if (isinstance(vo_resource, objects.Volume) and
vo_resource.status == 'creating'):
vo_resource.status = 'error'
vo_resource.save()
def request_service_capabilities(self, context):
volume_rpcapi.VolumeAPI().publish_service_capabilities(context)
try:
self.backup_api.publish_service_capabilities(context)
except exception.ServiceTooOld as e:
# cinder-backup has publish_service_capabilities starting Stein
# release only.
msg = ("Failed to notify about cinder-backup service "
"capabilities for host %(host)s. This is normal "
"during a live upgrade. Error: %(e)s")
LOG.warning(msg, {'host': self.host, 'e': e})
@append_operation_type()
def migrate_volume(self, context, volume, backend, force_copy,
request_spec, filter_properties):
"""Ensure that the backend exists and can accept the volume."""
self._wait_for_scheduler()
def _migrate_volume_set_error(self, context, ex, request_spec):
if volume.status == 'maintenance':
previous_status = (
volume.previous_status or 'maintenance')
volume_state = {'volume_state': {'migration_status': 'error',
'status': previous_status}}
else:
volume_state = {'volume_state': {'migration_status': 'error'}}
self._set_volume_state_and_notify('migrate_volume_to_host',
volume_state,
context, ex, request_spec)
try:
tgt_backend = self.driver.backend_passes_filters(context, backend,
request_spec,
filter_properties)
except exception.NoValidBackend as ex:
_migrate_volume_set_error(self, context, ex, request_spec)
except Exception as ex:
with excutils.save_and_reraise_exception():
_migrate_volume_set_error(self, context, ex, request_spec)
else:
volume_rpcapi.VolumeAPI().migrate_volume(context, volume,
tgt_backend,
force_copy)
# FIXME(geguileo): Remove this in v4.0 of RPC API.
def migrate_volume_to_host(self, context, volume, host, force_host_copy,
request_spec, filter_properties=None):
return self.migrate_volume(context, volume, host, force_host_copy,
request_spec, filter_properties)
@append_operation_type(name='retype_volume')
def retype(self, context, volume, request_spec, filter_properties=None):
"""Schedule the modification of a volume's type.
:param context: the request context
:param volume: the volume object to retype
:param request_spec: parameters for this retype request
:param filter_properties: parameters to filter by
"""
self._wait_for_scheduler()
def _retype_volume_set_error(self, context, ex, request_spec,
volume_ref, reservations, msg=None):
if reservations:
QUOTAS.rollback(context, reservations)
previous_status = (
volume_ref.previous_status or volume_ref.status)
volume_state = {'volume_state': {'status': previous_status}}
self._set_volume_state_and_notify('retype', volume_state,
context, ex, request_spec, msg)
reservations = request_spec.get('quota_reservations')
old_reservations = request_spec.get('old_reservations', None)
new_type = request_spec.get('volume_type')
if new_type is None:
msg = _('New volume type not specified in request_spec.')
ex = exception.ParameterNotFound(param='volume_type')
_retype_volume_set_error(self, context, ex, request_spec,
volume, reservations, msg)
# Default migration policy is 'never'
migration_policy = request_spec.get('migration_policy')
if not migration_policy:
migration_policy = 'never'
try:
tgt_backend = self.driver.find_retype_backend(context,
request_spec,
filter_properties,
migration_policy)
except Exception as ex:
# Not having a valid host is an expected exception, so we don't
# reraise on it.
reraise = not isinstance(ex, exception.NoValidBackend)
with excutils.save_and_reraise_exception(reraise=reraise):
_retype_volume_set_error(self, context, ex, request_spec,
volume, reservations)
else:
volume_rpcapi.VolumeAPI().retype(context, volume,
new_type['id'], tgt_backend,
migration_policy,
reservations,
old_reservations)
@append_operation_type()
def manage_existing(self, context, volume, request_spec,
filter_properties=None):
"""Ensure that the host exists and can accept the volume."""
self._wait_for_scheduler()
def _manage_existing_set_error(self, context, ex, request_spec):
volume_state = {'volume_state': {'status': 'error_managing'}}
self._set_volume_state_and_notify('manage_existing', volume_state,
context, ex, request_spec)
try:
backend = self.driver.backend_passes_filters(
context, volume.service_topic_queue, request_spec,
filter_properties)
# At the API we didn't have the pool info, so the volume DB entry
# was created without it, now we add it.
volume.host = backend.host
volume.cluster_name = backend.cluster_name
volume.save()
except exception.NoValidBackend as ex:
_manage_existing_set_error(self, context, ex, request_spec)
except Exception as ex:
with excutils.save_and_reraise_exception():
_manage_existing_set_error(self, context, ex, request_spec)
else:
volume_rpcapi.VolumeAPI().manage_existing(context, volume,
request_spec.get('ref'))
@append_operation_type()
def manage_existing_snapshot(self, context, volume, snapshot, ref,
request_spec, filter_properties=None):
"""Ensure that the host exists and can accept the snapshot."""
self._wait_for_scheduler()
try:
backend = self.driver.backend_passes_filters(
context, volume.service_topic_queue, request_spec,
filter_properties)
backend.consume_from_volume({'size': volume.size})
except exception.NoValidBackend as ex:
self._set_snapshot_state_and_notify('manage_existing_snapshot',
snapshot,
fields.SnapshotStatus.ERROR,
context, ex, request_spec)
else:
volume_rpcapi.VolumeAPI().manage_existing_snapshot(
context, snapshot, ref,
volume.service_topic_queue)
def get_pools(self, context, filters=None):
"""Get active pools from scheduler's cache.
NOTE(dulek): There's no self._wait_for_scheduler() because get_pools is
an RPC call (is blocking for the c-api). Also this is admin-only API
extension so it won't hurt the user much to retry the request manually.
"""
return self.driver.get_pools(context, filters)
@append_operation_type(name='create_group')
def validate_host_capacity(self, context, backend, request_spec,
filter_properties):
try:
backend_state = self.driver.backend_passes_filters(
context,
backend,
request_spec, filter_properties)
backend_state.consume_from_volume(
{'size': request_spec['volume_properties']['size']})
except exception.NoValidBackend:
LOG.error("Desired host %(host)s does not have enough "
"capacity.", {'host': backend})
return False
return True
@append_operation_type()
def extend_volume(self, context, volume, new_size, reservations,
request_spec=None, filter_properties=None):
def _extend_volume_set_error(self, context, ex, request_spec):
volume_state = {'volume_state': {'status': volume.previous_status,
'previous_status': None}}
self._set_volume_state_and_notify('extend_volume', volume_state,
context, ex, request_spec)
if not filter_properties:
filter_properties = {}
filter_properties['new_size'] = new_size
try:
backend_state = self.driver.backend_passes_filters(
context,
volume.service_topic_queue,
request_spec, filter_properties)
backend_state.consume_from_volume(
{'size': new_size - volume.size})
volume_rpcapi.VolumeAPI().extend_volume(context, volume, new_size,
reservations)
except exception.NoValidBackend as ex:
QUOTAS.rollback(context, reservations,
project_id=volume.project_id)
_extend_volume_set_error(self, context, ex, request_spec)
self.message_api.create(
context,
message_field.Action.EXTEND_VOLUME,
resource_uuid=volume.id,
exception=ex)
def _set_volume_state_and_notify(self, method, updates, context, ex,
request_spec, msg=None):
# TODO(harlowja): move into a task that just does this later.
if not msg:
msg = ("Failed to schedule_%(method)s: %(ex)s" %
{'method': method, 'ex': ex})
LOG.error(msg)
volume_state = updates['volume_state']
properties = request_spec.get('volume_properties', {})
volume_id = request_spec.get('volume_id', None)
if volume_id:
db.volume_update(context, volume_id, volume_state)
if volume_state.get('status') == 'error_managing':
volume_state['status'] = 'error'
payload = dict(request_spec=request_spec,
volume_properties=properties,
volume_id=volume_id,
state=volume_state,
method=method,
reason=ex)
rpc.get_notifier("scheduler").error(context,
'scheduler.' + method,
payload)
def _set_snapshot_state_and_notify(self, method, snapshot, state,
context, ex, request_spec,
msg=None):
if not msg:
msg = ("Failed to schedule_%(method)s: %(ex)s" %
{'method': method, 'ex': ex})
LOG.error(msg)
model_update = dict(status=state)
snapshot.update(model_update)
snapshot.save()
payload = dict(request_spec=request_spec,
snapshot_id=snapshot.id,
state=state,
method=method,
reason=ex)
rpc.get_notifier("scheduler").error(context,
'scheduler.' + method,
payload)
@property
def upgrading_cloud(self):
min_version_str = self.sch_api.determine_rpc_version_cap()
min_version = versionutils.convert_version_to_int(min_version_str)
return min_version < self.rpc_api_version
def _cleanup_destination(self, clusters, service):
"""Determines the RPC method, destination service and name.
The name is only used for logging, and it is the topic queue.
"""
# For the scheduler we don't have a specific destination, as any
# scheduler will do and we know we are up, since we are running this
# code.
if service.binary == 'cinder-scheduler':
cleanup_rpc = self.sch_api.do_cleanup
dest = None
dest_name = service.host
else:
cleanup_rpc = self.volume_api.do_cleanup
# For clustered volume services we try to get info from the cache.
if service.is_clustered:
# Get cluster info from cache
dest = clusters[service.binary].get(service.cluster_name)
# Cache miss forces us to get the cluster from the DB via OVO
if not dest:
dest = service.cluster
clusters[service.binary][service.cluster_name] = dest
dest_name = dest.name
# Non clustered volume services
else:
dest = service
dest_name = service.host
return cleanup_rpc, dest, dest_name
def work_cleanup(self, context, cleanup_request):
"""Process request from API to do cleanup on services.
Here we retrieve from the DB which services we want to clean up based
on the request from the user.
Then send individual cleanup requests to each of the services that are
up, and we finally return a tuple with services that we have sent a
cleanup request and those that were not up and we couldn't send it.
"""
if self.upgrading_cloud:
raise exception.UnavailableDuringUpgrade(action='workers cleanup')
LOG.info('Workers cleanup request started.')
filters = dict(service_id=cleanup_request.service_id,
cluster_name=cleanup_request.cluster_name,
host=cleanup_request.host,
binary=cleanup_request.binary,
is_up=cleanup_request.is_up,
disabled=cleanup_request.disabled)
# Get the list of all the services that match the request
services = objects.ServiceList.get_all(context, filters)
until = cleanup_request.until or timeutils.utcnow()
requested = []
not_requested = []
# To reduce DB queries we'll cache the clusters data
clusters = collections.defaultdict(dict)
for service in services:
cleanup_request.cluster_name = service.cluster_name
cleanup_request.service_id = service.id
cleanup_request.host = service.host
cleanup_request.binary = service.binary
cleanup_request.until = until
cleanup_rpc, dest, dest_name = self._cleanup_destination(clusters,
service)
# If it's a scheduler or the service is up, send the request.
if not dest or dest.is_up:
LOG.info('Sending cleanup for %(binary)s %(dest_name)s.',
{'binary': service.binary,
'dest_name': dest_name})
cleanup_rpc(context, cleanup_request)
requested.append(service)
# We don't send cleanup requests when there are no services alive
# to do the cleanup.
else:
LOG.info('No service available to cleanup %(binary)s '
'%(dest_name)s.',
{'binary': service.binary,
'dest_name': dest_name})
not_requested.append(service)
LOG.info('Cleanup requests completed.')
return requested, not_requested
def create_backup(self, context, backup):
volume_id = backup.volume_id
volume = self.db.volume_get(context, volume_id)
try:
host = self.driver.get_backup_host(volume)
backup.host = host
backup.save()
self.backup_api.create_backup(context, backup)
except exception.ServiceNotFound:
self.db.volume_update(context, volume_id,
{'status': volume['previous_status'],
'previous_status': volume['status']})
msg = "Service not found for creating backup."
LOG.error(msg)
vol_utils.update_backup_error(backup, msg)