# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import functools import re from oslo_log import log as logging from oslo_serialization import jsonutils from oslo_utils import versionutils from oslo_versionedobjects import exception as ovoo_exc import six from sqlalchemy.sql import null from nova.db.sqlalchemy import api as db from nova.db.sqlalchemy import api_models from nova import exception from nova.i18n import _LE from nova import objects from nova.objects import base from nova.objects import fields LOG = logging.getLogger(__name__) @base.NovaObjectRegistry.register class BuildRequest(base.NovaObject): # Version 1.0: Initial version # Version 1.1: Added block_device_mappings # Version 1.2: Added save() method VERSION = '1.2' fields = { 'id': fields.IntegerField(), 'instance_uuid': fields.UUIDField(), 'project_id': fields.StringField(), 'instance': fields.ObjectField('Instance'), 'block_device_mappings': fields.ObjectField('BlockDeviceMappingList'), # NOTE(alaski): Normally these would come from the NovaPersistentObject # mixin but they're being set explicitly because we only need # created_at/updated_at. There is no soft delete for this object. 'created_at': fields.DateTimeField(nullable=True), 'updated_at': fields.DateTimeField(nullable=True), } def obj_make_compatible(self, primitive, target_version): super(BuildRequest, self).obj_make_compatible(primitive, target_version) target_version = versionutils.convert_version_to_tuple(target_version) if target_version < (1, 1) and 'block_device_mappings' in primitive: del primitive['block_device_mappings'] def _load_instance(self, db_instance): # NOTE(alaski): Be very careful with instance loading because it # changes more than most objects. try: self.instance = objects.Instance.obj_from_primitive( jsonutils.loads(db_instance)) except TypeError: LOG.debug('Failed to load instance from BuildRequest with uuid ' '%s because it is None', self.instance_uuid) raise exception.BuildRequestNotFound(uuid=self.instance_uuid) except ovoo_exc.IncompatibleObjectVersion as exc: # This should only happen if proper service upgrade strategies are # not followed. Log the exception and raise BuildRequestNotFound. # If the instance can't be loaded this object is useless and may # as well not exist. LOG.debug('Could not deserialize instance store in BuildRequest ' 'with uuid %(instance_uuid)s. Found version %(version)s ' 'which is not supported here.', dict(instance_uuid=self.instance_uuid, version=exc.objver)) LOG.exception(_LE('Could not deserialize instance in ' 'BuildRequest')) raise exception.BuildRequestNotFound(uuid=self.instance_uuid) # NOTE(sbauza): The instance primitive should already have the deleted # field being set, so when hydrating it back here, we should get the # right value but in case we don't have it, let's suppose that the # instance is not deleted, which is the default value for that field. self.instance.obj_set_defaults('deleted') # NOTE(alaski): Set some fields on instance that are needed by the api, # not lazy-loadable, and don't change. self.instance.disable_terminate = False self.instance.terminated_at = None self.instance.host = None self.instance.node = None self.instance.launched_at = None self.instance.launched_on = None self.instance.cell_name = None # The fields above are not set until the instance is in a cell at # which point this BuildRequest will be gone. locked_by could # potentially be set by an update so it should not be overwritten. if not self.instance.obj_attr_is_set('locked_by'): self.instance.locked_by = None # created_at/updated_at are not on the serialized instance because it # was never persisted. self.instance.created_at = self.created_at self.instance.updated_at = self.updated_at self.instance.tags = objects.TagList([]) def _load_block_device_mappings(self, db_bdms): # 'db_bdms' is a serialized BlockDeviceMappingList object. If it's None # we're in a mixed version nova-api scenario and can't retrieve the # actual list. Set it to an empty list here which will cause a # temporary API inconsistency that will be resolved as soon as the # instance is scheduled and on a compute. if db_bdms is None: LOG.debug('Failed to load block_device_mappings from BuildRequest ' 'for instance %s because it is None', self.instance_uuid) self.block_device_mappings = objects.BlockDeviceMappingList() return self.block_device_mappings = ( objects.BlockDeviceMappingList.obj_from_primitive( jsonutils.loads(db_bdms))) @staticmethod def _from_db_object(context, req, db_req): # Set this up front so that it can be pulled for error messages or # logging at any point. req.instance_uuid = db_req['instance_uuid'] for key in req.fields: if key == 'instance': continue elif isinstance(req.fields[key], fields.ObjectField): try: getattr(req, '_load_%s' % key)(db_req[key]) except AttributeError: LOG.exception(_LE('No load handler for %s'), key) else: setattr(req, key, db_req[key]) # Load instance last because other fields on req may be referenced req._load_instance(db_req['instance']) req.obj_reset_changes(recursive=True) req._context = context return req @staticmethod @db.api_context_manager.reader def _get_by_instance_uuid_from_db(context, instance_uuid): db_req = context.session.query(api_models.BuildRequest).filter_by( instance_uuid=instance_uuid).first() if not db_req: raise exception.BuildRequestNotFound(uuid=instance_uuid) return db_req @base.remotable_classmethod def get_by_instance_uuid(cls, context, instance_uuid): db_req = cls._get_by_instance_uuid_from_db(context, instance_uuid) return cls._from_db_object(context, cls(), db_req) @staticmethod @db.api_context_manager.writer def _create_in_db(context, updates): db_req = api_models.BuildRequest() db_req.update(updates) db_req.save(context.session) return db_req def _get_update_primitives(self): updates = self.obj_get_changes() for key, value in updates.items(): if isinstance(self.fields[key], fields.ObjectField): updates[key] = jsonutils.dumps(value.obj_to_primitive()) return updates @base.remotable def create(self): if self.obj_attr_is_set('id'): raise exception.ObjectActionError(action='create', reason='already created') if not self.obj_attr_is_set('instance_uuid'): # We can't guarantee this is not null in the db so check here raise exception.ObjectActionError(action='create', reason='instance_uuid must be set') updates = self._get_update_primitives() db_req = self._create_in_db(self._context, updates) self._from_db_object(self._context, self, db_req) @staticmethod @db.api_context_manager.writer def _destroy_in_db(context, instance_uuid): result = context.session.query(api_models.BuildRequest).filter_by( instance_uuid=instance_uuid).delete() if not result: raise exception.BuildRequestNotFound(uuid=instance_uuid) @base.remotable def destroy(self): self._destroy_in_db(self._context, self.instance_uuid) @db.api_context_manager.writer def _save_in_db(self, context, req_id, updates): db_req = context.session.query( api_models.BuildRequest).filter_by(id=req_id).first() if not db_req: raise exception.BuildRequestNotFound(uuid=self.instance_uuid) db_req.update(updates) context.session.add(db_req) return db_req @base.remotable def save(self): updates = self._get_update_primitives() db_req = self._save_in_db(self._context, self.id, updates) self._from_db_object(self._context, self, db_req) def get_new_instance(self, context): # NOTE(danms): This is a hack to make sure that the returned # instance has all dirty fields. There are probably better # ways to do this, but they kinda involve o.vo internals # so this is okay for the moment. instance = objects.Instance(context) for field in self.instance.obj_fields: # NOTE(danms): Don't copy the defaulted tags field # as instance.create() won't handle it properly. if field == 'tags': continue if self.instance.obj_attr_is_set(field): setattr(instance, field, getattr(self.instance, field)) return instance @base.NovaObjectRegistry.register class BuildRequestList(base.ObjectListBase, base.NovaObject): # Version 1.0: Initial version VERSION = '1.0' fields = { 'objects': fields.ListOfObjectsField('BuildRequest'), } @staticmethod @db.api_context_manager.reader def _get_all_from_db(context): query = context.session.query(api_models.BuildRequest) if not context.is_admin: query = query.filter_by(project_id=context.project_id) db_reqs = query.all() return db_reqs @base.remotable_classmethod def get_all(cls, context): db_build_reqs = cls._get_all_from_db(context) return base.obj_make_list(context, cls(context), objects.BuildRequest, db_build_reqs) @staticmethod def _pass_exact_filters(instance, filters): for filter_key, filter_val in filters.items(): if filter_key in ('metadata', 'system_metadata'): if isinstance(filter_val, list): for item in filter_val: for k, v in item.items(): if (k not in instance.metadata or v != instance.metadata[k]): return False else: for k, v in filter_val.items(): if (k not in instance.metadata or v != instance.metadata[k]): return False elif isinstance(filter_val, (list, tuple, set, frozenset)): if not filter_val: # Special value to indicate that nothing will match. return None if instance.get(filter_key, None) not in filter_val: return False else: if instance.get(filter_key, None) != filter_val: return False return True @staticmethod def _pass_regex_filters(instance, filters): for filter_name, filter_val in filters.items(): try: instance_attr = getattr(instance, filter_name) except AttributeError: continue # Sometimes the REGEX filter value is not a string if not isinstance(filter_val, six.string_types): filter_val = str(filter_val) filter_re = re.compile(filter_val) if instance_attr and not filter_re.search(str(instance_attr)): return False return True @staticmethod def _sort_build_requests(build_req_list, sort_keys, sort_dirs): # build_req_list is a [] of build_reqs sort_keys.reverse() sort_dirs.reverse() def sort_attr(sort_key, build_req): if sort_key == 'id': # 'id' is not set on the instance yet. Use the BuildRequest # 'id' instead. return build_req.id return getattr(build_req.instance, sort_key) for sort_key, sort_dir in zip(sort_keys, sort_dirs): reverse = False if sort_dir.lower().startswith('asc') else True build_req_list.sort(key=functools.partial(sort_attr, sort_key), reverse=reverse) return build_req_list @base.remotable_classmethod def get_by_filters(cls, context, filters, limit=None, marker=None, sort_keys=None, sort_dirs=None): if limit == 0: return cls(context, objects=[]) # 'deleted' records can not be returned from here since build_requests # are not soft deleted. if filters.get('deleted', False): return cls(context, objects=[]) # 'cleaned' records won't exist as they would need to be deleted. if filters.get('cleaned', False): return cls(context, objects=[]) # Because the build_requests table stores an instance as a serialized # versioned object it is not feasible to do the filtering and sorting # in the database. Just get all potentially relevant records and # process them here. It should be noted that build requests are short # lived so there should not be a lot of results to deal with. build_requests = cls.get_all(context) # Fortunately some filters do not apply here. # 'tags' can not be applied at boot time so will not be set for an # instance here. # 'changes-since' works off of the updated_at field which has not yet # been set at the point in the boot process where build_request still # exists. So it can be ignored. # 'deleted' and 'cleaned' are handled above. sort_keys, sort_dirs = db.process_sort_params(sort_keys, sort_dirs, default_dir='desc') # For other filters that don't match this, we will do regexp matching # Taken from db/sqlalchemy/api.py exact_match_filter_names = ['project_id', 'user_id', 'image_ref', 'vm_state', 'instance_type_id', 'uuid', 'metadata', 'host', 'task_state', 'system_metadata'] exact_filters = {} regex_filters = {} for key, value in filters.items(): if key in exact_match_filter_names: exact_filters[key] = value else: regex_filters[key] = value # As much as possible this copies the logic from db/sqlalchemy/api.py # instance_get_all_by_filters_sort. The main difference is that method # builds a sql query and this filters in python. filtered_build_reqs = [] for build_req in build_requests: instance = build_req.instance filter_result = cls._pass_exact_filters(instance, exact_filters) if filter_result is None: # The filter condition is such that nothing will match. # Bail early. return cls(context, objects=[]) if filter_result is False: continue if not cls._pass_regex_filters(instance, regex_filters): continue filtered_build_reqs.append(build_req) if (((len(filtered_build_reqs) < 2) or (not sort_keys)) and not marker): # No need to sort return cls(context, objects=filtered_build_reqs) sorted_build_reqs = cls._sort_build_requests(filtered_build_reqs, sort_keys, sort_dirs) marker_index = 0 if marker: for i, build_req in enumerate(sorted_build_reqs): if build_req.instance.uuid == marker: marker_index = i break else: raise exception.MarkerNotFound(marker=marker) len_build_reqs = len(sorted_build_reqs) limit_index = len_build_reqs if limit: limit_index = marker_index + limit if limit_index > len_build_reqs: limit_index = len_build_reqs return cls(context, objects=sorted_build_reqs[marker_index:limit_index]) @db.api_context_manager.reader def _get_build_requests_with_no_instance_uuid(context, limit): """Returns up to $limit build_requests where instance_uuid is null""" # build_requests don't use the SoftDeleteMixin so we don't have to filter # on the deleted column. return context.session.query(api_models.BuildRequest).\ filter_by(instance_uuid=null()).\ limit(limit).\ all() @db.api_context_manager.writer def _destroy_in_db(context, id): return context.session.query(api_models.BuildRequest).filter_by( id=id).delete() def delete_build_requests_with_no_instance_uuid(context, count): """Online data migration which cleans up failed build requests from Mitaka build_requests were initially a mirror of instances and had similar fields to satisfy listing/showing instances while they were building. In Mitaka if an instance failed to build we'd delete the instance but didn't delete the associated BuildRequest. In the Newton release we changed the schema on the build_requests table to just store a serialized Instance object and added an instance_uuid field which is expected to not be None as seen how it's used in _from_db_object. However, failed build requests created before that schema migration won't have the instance_uuid set and fail to load as an object when calling BuildRequestList.get_all(). So we need to perform a cleanup routine here where we search for build requests which do not have the instance_uuid field set and delete them. :param context: The auth context used to query the database. :type context: nova.context.RequestContext :param count: The max number of build requests to delete. :type count: int :returns: 2-item tuple of (number of orphaned build requests read from DB, number deleted) """ orphaned_build_requests = ( _get_build_requests_with_no_instance_uuid(context, count)) done = 0 for orphan_buildreq in orphaned_build_requests: result = _destroy_in_db(context, orphan_buildreq.id) if result: done += 1 return len(orphaned_build_requests), done