# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """Functionality related to allocations.""" import random from ironic_lib import metrics_utils from oslo_config import cfg from oslo_log import log from oslo_utils import excutils import retrying from ironic.common import exception from ironic.common.i18n import _ from ironic.common import states from ironic.conductor import task_manager from ironic import objects CONF = cfg.CONF LOG = log.getLogger(__name__) METRICS = metrics_utils.get_metrics_logger(__name__) def do_allocate(context, allocation): """Process the allocation. This call runs in a separate thread on a conductor. It finds suitable nodes for the allocation and reserves one of them. This call does not raise exceptions since it's designed to work asynchronously. :param context: an admin context :param allocation: an allocation object """ try: nodes = _candidate_nodes(context, allocation) _allocate_node(context, allocation, nodes) except exception.AllocationFailed as exc: LOG.error(str(exc)) _allocation_failed(allocation, exc) except Exception as exc: LOG.exception("Unexpected exception during processing of " "allocation %s", allocation.uuid) reason = _("Unexpected exception during allocation: %s") % exc _allocation_failed(allocation, reason) def verify_node_for_deallocation(node, allocation): """Verify that allocation can be removed for the node. :param node: a node object :param allocation: an allocation object associated with the node """ if node.maintenance: # Allocations can always be removed in the maintenance mode. return if (node.target_provision_state and node.provision_state not in states.UPDATE_ALLOWED_STATES): msg = (_("Cannot remove allocation %(uuid)s for node %(node)s, " "because the node is in state %(state)s where updates are " "not allowed (and maintenance mode is off)") % {'node': node.uuid, 'uuid': allocation.uuid, 'state': node.provision_state}) raise exception.InvalidState(msg) if node.provision_state == states.ACTIVE: msg = (_("Cannot remove allocation %(uuid)s for node %(node)s, " "because the node is active (and maintenance mode is off)") % {'node': node.uuid, 'uuid': allocation.uuid}) raise exception.InvalidState(msg) def _allocation_failed(allocation, reason): """Failure handler for the allocation.""" try: allocation.node_id = None allocation.state = states.ERROR allocation.last_error = str(reason) allocation.save() except exception.AllocationNotFound as exc: LOG.debug('Not saving a failed allocation: %s', exc) except Exception: LOG.exception('Could not save the failed allocation %s', allocation.uuid) def _traits_match(traits, node): return {t.trait for t in node.traits.objects}.issuperset(traits) def _candidate_nodes(context, allocation): """Get a list of candidate nodes for the allocation.""" filters = {'resource_class': allocation.resource_class, 'provision_state': states.AVAILABLE, 'associated': False, 'with_power_state': True, 'maintenance': False} if allocation.candidate_nodes: # NOTE(dtantsur): we assume that candidate_nodes were converted to # UUIDs on the API level. filters['uuid_in'] = allocation.candidate_nodes if allocation.owner: filters['project'] = allocation.owner nodes = objects.Node.list(context, filters=filters) if not nodes: if allocation.candidate_nodes: error = _("none of the requested nodes are available and match " "the resource class %s") % allocation.resource_class else: error = _("no available nodes match the resource class %s") % ( allocation.resource_class) raise exception.AllocationFailed(uuid=allocation.uuid, error=error) # TODO(dtantsur): database-level filtering? if allocation.traits: traits = set(allocation.traits) nodes = [n for n in nodes if _traits_match(traits, n)] if not nodes: error = (_("no suitable nodes have the requested traits %s") % ', '.join(traits)) raise exception.AllocationFailed(uuid=allocation.uuid, error=error) # NOTE(dtantsur): make sure that parallel allocations do not try the nodes # in the same order. random.shuffle(nodes) LOG.debug('%(count)d nodes are candidates for allocation %(uuid)s', {'count': len(nodes), 'uuid': allocation.uuid}) return nodes def _verify_node(node, allocation): """Check that the node still satisfiest the request.""" if node.maintenance: LOG.debug('Node %s is now in maintenance, skipping', node.uuid) return False if node.instance_uuid: LOG.debug('Node %(node)s is already associated with instance ' '%(inst)s, skipping', {'node': node.uuid, 'inst': node.instance_uuid}) return False if node.provision_state != states.AVAILABLE: LOG.debug('Node %s is no longer available, skipping', node.uuid) return False if node.resource_class != allocation.resource_class: LOG.debug('Resource class of node %(node)s no longer ' 'matches requested resource class %(rsc)s for ' 'allocation %(uuid)s, skipping', {'node': node.uuid, 'rsc': allocation.resource_class, 'uuid': allocation.uuid}) return False if allocation.traits and not _traits_match(set(allocation.traits), node): LOG.debug('List of traits of node %(node)s no longer ' 'matches requested traits %(traits)s for ' 'allocation %(uuid)s, skipping', {'node': node.uuid, 'traits': allocation.traits, 'uuid': allocation.uuid}) return False return True # NOTE(dtantsur): instead of trying to allocate each node # node_locked_retry_attempt times, we try to allocate *any* node the same # number of times. This avoids getting stuck on a node reserved e.g. for power # sync periodic task. @retrying.retry( retry_on_exception=lambda e: isinstance(e, exception.AllocationFailed), stop_max_attempt_number=CONF.conductor.node_locked_retry_attempts, wait_fixed=CONF.conductor.node_locked_retry_interval * 1000) def _allocate_node(context, allocation, nodes): """Go through the list of nodes and try to allocate one of them.""" retry_nodes = [] for node in nodes: try: # NOTE(dtantsur): retries are done for all nodes above, so disable # per-node retry. Also disable loading the driver, since the # current conductor may not have the requried hardware type or # interfaces (it's picked at random). with task_manager.acquire(context, node.uuid, shared=False, retry=False, load_driver=False, purpose='allocating') as task: # NOTE(dtantsur): double-check the node details, since they # could have changed before we acquired the lock. if not _verify_node(task.node, allocation): continue allocation.node_id = task.node.id allocation.state = states.ACTIVE # NOTE(dtantsur): the node.instance_uuid and allocation_id are # updated inside of the save() call within the same # transaction to avoid races. NodeAssociated can be raised if # another process allocates this node first. allocation.save() LOG.info('Node %(node)s has been successfully reserved for ' 'allocation %(uuid)s', {'node': node.uuid, 'uuid': allocation.uuid}) return allocation except exception.NodeLocked: LOG.debug('Node %s is currently locked, moving to the next one', node.uuid) retry_nodes.append(node) except exception.NodeAssociated: LOG.debug('Node %s is already associated, moving to the next one', node.uuid) # NOTE(dtantsur): rewrite the passed list to only contain the nodes that # are worth retrying. Do not include nodes that are no longer suitable. nodes[:] = retry_nodes if nodes: error = _('could not reserve any of %d suitable nodes') % len(nodes) else: error = _('all nodes were filtered out during reservation') raise exception.AllocationFailed(uuid=allocation.uuid, error=error) def backfill_allocation(context, allocation, node_id): """Assign the previously allocated node to the node allocation. This is not the actual allocation process, but merely backfilling of allocation_uuid for a previously allocated node. :param context: an admin context :param allocation: an allocation object associated with the node :param node_id: An ID of the node. :raises: AllocationFailed if the node does not match the allocation :raises: NodeAssociated if the node is already associated with another instance or allocation. :raises: InstanceAssociated if the allocation's UUID is already used on another node as instance_uuid. :raises: NodeNotFound if the node with the provided ID cannot be found. """ try: _do_backfill_allocation(context, allocation, node_id) except (exception.AllocationFailed, exception.InstanceAssociated, exception.NodeAssociated, exception.NodeNotFound) as exc: with excutils.save_and_reraise_exception(): LOG.error(str(exc)) _allocation_failed(allocation, exc) except Exception as exc: with excutils.save_and_reraise_exception(): LOG.exception("Unexpected exception during backfilling of " "allocation %s", allocation.uuid) reason = _("Unexpected exception during allocation: %s") % exc _allocation_failed(allocation, reason) def _do_backfill_allocation(context, allocation, node_id): with task_manager.acquire(context, node_id, purpose='allocation backfilling') as task: node = task.node errors = [] # NOTE(dtantsur): this feature is not designed to bypass the allocation # mechanism, but to backfill allocations for active nodes, hence this # check. if node.provision_state != states.ACTIVE: errors.append(_('Node must be in the "active" state, but the ' 'current state is "%s"') % node.provision_state) # NOTE(dtantsur): double-check that the node is still suitable. if (allocation.resource_class and node.resource_class != allocation.resource_class): errors.append(_('Resource class %(curr)s does not match ' 'the requested resource class %(rsc)s') % {'curr': node.resource_class, 'rsc': allocation.resource_class}) if (allocation.traits and not _traits_match(set(allocation.traits), node)): errors.append(_('List of traits %(curr)s does not match ' 'the requested traits %(traits)s') % {'curr': node.traits, 'traits': allocation.traits}) if (allocation.candidate_nodes and node.uuid not in allocation.candidate_nodes): errors.append(_('Candidate nodes must be empty or contain the ' 'target node, but got %s') % allocation.candidate_nodes) if errors: error = _('Cannot backfill an allocation for node %(node)s: ' '%(errors)s') % {'node': node.uuid, 'errors': '; '.join(errors)} raise exception.AllocationFailed(uuid=allocation.uuid, error=error) allocation.node_id = task.node.id allocation.state = states.ACTIVE # NOTE(dtantsur): the node.instance_uuid and allocation_id are # updated inside of the save() call within the same # transaction to avoid races. NodeAssociated can be raised if # another process allocates this node first. allocation.save() LOG.info('Node %(node)s has been successfully reserved for ' 'allocation %(uuid)s', {'node': node.uuid, 'uuid': allocation.uuid})