openstack/nova
Code Issues Proposed changes

Add persistence to the RequestSpec object

Browse Source 
In order to use the request spec for an instance boot during later
scheduling operations, like resize/migrate, it needs to be persisted.
This adds a db migration to store the request spec and methods to the
object to create/save/get the request spec.

The RequestSpec object is also important for cells v2 so that boot
requests can have their information persisted before the creation of an
instance record in the database.

Change-Id: I94cd50ad29e10eb320d1d78f3081db6519815ff9
Implements: bp persist-request-spec
This commit is contained in:
Andrew Laski 2015-08-12 15:30:12 -04:00 committed by Sylvain Bauza
parent ea725fd6ba
commit 0152047a77
9 changed files with 399 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
nova/db/sqlalchemy/api_migrations/migrate_repo/versions/004_add_request_spec.py Normal file
View File

@ -0,0 +1,46 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from migrate import UniqueConstraint
from sqlalchemy import Column
from sqlalchemy import DateTime
from sqlalchemy import Index
from sqlalchemy import Integer
from sqlalchemy import MetaData
from sqlalchemy import String
from sqlalchemy import Table
from sqlalchemy import Text
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
request_specs = Table('request_specs', meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('instance_uuid', String(36), nullable=False),
Column('spec', Text, nullable=False),
UniqueConstraint('instance_uuid',
name='uniq_request_specs0instance_uuid'),
mysql_engine='InnoDB',
mysql_charset='utf8'
)
# NOTE(mriedem): DB2 creates an index when a unique constraint is created
# so trying to add a second index on the host column will fail with error
# SQL0605W, so omit the index in the case of DB2.
if migrate_engine.name != 'ibm_db_sa':
Index('request_spec_instance_uuid_idx', request_specs.c.instance_uuid)
request_specs.create(checkfirst=True)

15
nova/db/sqlalchemy/api_models.py
View File

@ -69,3 +69,18 @@ class HostMapping(API_BASE):
cell_id = Column(Integer, ForeignKey('cell_mappings.id'),
nullable=False)
host = Column(String(255), nullable=False)
class RequestSpec(API_BASE):
"""Represents the information passed to the scheduler."""
__tablename__ = 'request_specs'
__table_args__ = (
Index('request_spec_instance_uuid_idx', 'instance_uuid'),
schema.UniqueConstraint('instance_uuid',
name='uniq_request_specs0instance_uuid'),
)
id = Column(Integer, primary_key=True)
instance_uuid = Column(String(36), nullable=False)
spec = Column(Text, nullable=False)

4
nova/exception.py
View File

@ -1966,3 +1966,7 @@ class HostMappingNotFound(Invalid):
class RealtimeConfigurationInvalid(Invalid):
msg_fmt = _("Cannot set realtime policy in a non dedicated "
"cpu pinning policy")
class RequestSpecNotFound(NotFound):
msg_fmt = _("RequestSpec not found for instance %(instance_uuid)s")

89
nova/objects/request_spec.py
View File

@ -12,14 +12,21 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from oslo_serialization import jsonutils
import six
from nova.db.sqlalchemy import api as db
from nova.db.sqlalchemy import api_models
from nova import exception
from nova import objects
from nova.objects import base
from nova.objects import fields
from nova.objects import instance as obj_instance
from nova.virt import hardware
LOG = logging.getLogger(__name__)
@base.NovaObjectRegistry.register
class RequestSpec(base.NovaObject):
@ -28,7 +35,8 @@ class RequestSpec(base.NovaObject):
# Version 1.2: SchedulerRetries version 1.1
# Version 1.3: InstanceGroup version 1.10
# Version 1.4: ImageMeta version 1.7
VERSION = '1.4'
# Version 1.5: Added get_by_instance_uuid(), create(), save()
VERSION = '1.5'
fields = {
'id': fields.IntegerField(),
@ -311,6 +319,85 @@ class RequestSpec(base.NovaObject):
hint) for hint in self.scheduler_hints}
return filt_props
@staticmethod
def _from_db_object(context, spec, db_spec):
spec = spec.obj_from_primitive(jsonutils.loads(db_spec['spec']))
spec._context = context
spec.obj_reset_changes()
return spec
@staticmethod
def _get_by_instance_uuid_from_db(context, instance_uuid):
session = db.get_api_session()
with session.begin():
db_spec = session.query(api_models.RequestSpec).filter_by(
instance_uuid=instance_uuid).first()
if not db_spec:
raise exception.RequestSpecNotFound(
instance_uuid=instance_uuid)
return db_spec
@base.remotable_classmethod
def get_by_instance_uuid(cls, context, instance_uuid):
db_spec = cls._get_by_instance_uuid_from_db(context, instance_uuid)
return cls._from_db_object(context, cls(), db_spec)
@staticmethod
@db.api_context_manager.writer
def _create_in_db(context, updates):
db_spec = api_models.RequestSpec()
db_spec.update(updates)
db_spec.save(context.session)
return db_spec
def _get_update_primitives(self):
"""Serialize object to match the db model.
We store copies of embedded objects rather than
references to these objects because we want a snapshot of the request
at this point. If the references changed or were deleted we would
not be able to reschedule this instance under the same conditions as
it was originally scheduled with.
"""
updates = self.obj_get_changes()
# NOTE(alaski): The db schema is the full serialized object in a
# 'spec' column. If anything has changed we rewrite the full thing.
if updates:
db_updates = {'spec': jsonutils.dumps(self.obj_to_primitive())}
if 'instance_uuid' in updates:
db_updates['instance_uuid'] = updates['instance_uuid']
return db_updates
@base.remotable
def create(self):
if self.obj_attr_is_set('id'):
raise exception.ObjectActionError(action='create',
reason='already created')
updates = self._get_update_primitives()
db_spec = self._create_in_db(self._context, updates)
self._from_db_object(self._context, self, db_spec)
@staticmethod
@db.api_context_manager.writer
def _save_in_db(context, instance_uuid, updates):
# FIXME(sbauza): Provide a classmethod when oslo.db bug #1520195 is
# fixed and released
db_spec = RequestSpec._get_by_instance_uuid_from_db(context,
instance_uuid)
db_spec.update(updates)
db_spec.save(context.session)
return db_spec
@base.remotable
def save(self):
updates = self._get_update_primitives()
db_spec = self._save_in_db(self._context, self.instance_uuid, updates)
self._from_db_object(self._context, self, db_spec)
self.obj_reset_changes()
@base.NovaObjectRegistry.register
class SchedulerRetries(base.NovaObject):

11
nova/tests/functional/db/api/test_migrations.py
View File

@ -188,6 +188,17 @@ class NovaAPIMigrationsWalk(test_migrations.WalkVersionsMixin):
self.assertEqual(['id'], fk['referred_columns'])
self.assertEqual(['cell_id'], fk['constrained_columns'])
def _check_004(self, engine, data):
columns = ['created_at', 'updated_at', 'id', 'instance_uuid', 'spec']
for column in columns:
self.assertColumnExists(engine, 'request_specs', column)
self.assertUniqueConstraintExists(engine, 'request_specs',
['instance_uuid'])
if engine.name != 'ibm_db_sa':
self.assertIndexExists(engine, 'request_specs',
'request_spec_instance_uuid_idx')
class TestNovaAPIMigrationsWalkSQLite(NovaAPIMigrationsWalk,
test_base.DbTestCase,

63
nova/tests/functional/db/test_request_spec.py Normal file
View File

@ -0,0 +1,63 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from nova import context
from nova import exception
from nova.objects import base as obj_base
from nova.objects import request_spec
from nova import test
from nova.tests import fixtures
from nova.tests.unit import fake_request_spec
class RequestSpecTestCase(test.NoDBTestCase):
def setUp(self):
super(RequestSpecTestCase, self).setUp()
self.useFixture(fixtures.Database(database='api'))
self.context = context.RequestContext('fake-user', 'fake-project')
self.spec_obj = request_spec.RequestSpec()
self.instance_uuid = None
def _create_spec(self):
args = fake_request_spec.fake_db_spec()
args.pop('id', None)
self.instance_uuid = args['instance_uuid']
spec = request_spec.RequestSpec._from_db_object(self.context,
self.spec_obj,
self.spec_obj._create_in_db(self.context, args))
return spec
def test_get_by_instance_uuid_not_found(self):
self.assertRaises(exception.RequestSpecNotFound,
self.spec_obj._get_by_instance_uuid_from_db, self.context,
self.instance_uuid)
def test_get_by_uuid(self):
spec = self._create_spec()
db_spec = self.spec_obj.get_by_instance_uuid(self.context,
self.instance_uuid)
self.assertTrue(obj_base.obj_equal_prims(spec, db_spec))
def test_save_in_db(self):
spec = self._create_spec()
old_az = spec.availability_zone
spec.availability_zone = '%s-new' % old_az
spec.save()
db_spec = self.spec_obj.get_by_instance_uuid(self.context,
spec.instance_uuid)
self.assertTrue(obj_base.obj_equal_prims(spec, db_spec))
self.assertNotEqual(old_az, db_spec.availability_zone)
def test_double_create(self):
spec = self._create_spec()
self.assertRaises(exception.ObjectActionError, spec.create)

90
nova/tests/unit/fake_request_spec.py Normal file
View File

@ -0,0 +1,90 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_serialization import jsonutils
from oslo_utils import uuidutils
from nova import context
from nova import objects
from nova.tests.unit import fake_flavor
INSTANCE_NUMA_TOPOLOGY = objects.InstanceNUMATopology(
cells=[objects.InstanceNUMACell(id=0, cpuset=set([1, 2]), memory=512),
objects.InstanceNUMACell(id=1, cpuset=set([3, 4]), memory=512)])
INSTANCE_NUMA_TOPOLOGY.obj_reset_changes(recursive=True)
IMAGE_META = objects.ImageMeta.from_dict(
{'status': 'active',
'container_format': 'bare',
'min_ram': 0,
'updated_at': '2014-12-12T11:16:36.000000',
'min_disk': '0',
'owner': '2d8b9502858c406ebee60f0849486222',
'protected': 'yes',
'properties': {
'os_type': 'Linux',
'hw_video_model': 'vga',
'hw_video_ram': '512',
'hw_qemu_guest_agent': 'yes',
'hw_scsi_model': 'virtio-scsi',
},
'size': 213581824,
'name': 'f16-x86_64-openstack-sda',
'checksum': '755122332caeb9f661d5c978adb8b45f',
'created_at': '2014-12-10T16:23:14.000000',
'disk_format': 'qcow2',
'id': 'c8b1790e-a07d-4971-b137-44f2432936cd',
}
)
IMAGE_META.obj_reset_changes(recursive=True)
PCI_REQUESTS = objects.InstancePCIRequests(
requests=[objects.InstancePCIRequest(count=1),
objects.InstancePCIRequest(count=2)])
PCI_REQUESTS.obj_reset_changes(recursive=True)
def fake_db_spec():
req_obj = fake_spec_obj()
db_request_spec = {
'id': 1,
'instance_uuid': req_obj.instance_uuid,
'spec': jsonutils.dumps(req_obj.obj_to_primitive()),
}
return db_request_spec
def fake_spec_obj(remove_id=False):
ctxt = context.RequestContext('fake', 'fake')
req_obj = objects.RequestSpec(ctxt)
if not remove_id:
req_obj.id = 42
req_obj.instance_uuid = uuidutils.generate_uuid()
req_obj.image = IMAGE_META
req_obj.numa_topology = INSTANCE_NUMA_TOPOLOGY
req_obj.pci_requests = PCI_REQUESTS
req_obj.flavor = fake_flavor.fake_flavor_obj(ctxt)
req_obj.retry = objects.SchedulerRetries()
req_obj.limits = objects.SchedulerLimits()
req_obj.instance_group = objects.InstanceGroup()
req_obj.project_id = 'fake'
req_obj.num_instances = 1
req_obj.availability_zone = None
req_obj.ignore_hosts = ['host2', 'host4']
req_obj.force_hosts = ['host1', 'host3']
req_obj.force_nodes = ['node1', 'node2']
req_obj.scheduler_hints = {'hint': ['over-there']}
# This should never be a changed field
req_obj.obj_reset_changes(['id'])
return req_obj

2
nova/tests/unit/objects/test_objects.py
View File

@ -1253,7 +1253,7 @@ object_data = {
'PciDevicePoolList': '1.1-15ecf022a68ddbb8c2a6739cfc9f8f5e',
'Quotas': '1.2-1fe4cd50593aaf5d36a6dc5ab3f98fb3',
'QuotasNoOp': '1.2-e041ddeb7dc8188ca71706f78aad41c1',
'RequestSpec': '1.4-6922fe208b5d1186bdd825513f677921',
'RequestSpec': '1.5-576a249869c161e17b7cd6d55f9d85f3',
'S3ImageMapping': '1.0-7dd7366a890d82660ed121de9092276e',
'SchedulerLimits': '1.0-249c4bd8e62a9b327b7026b7f19cc641',
'SchedulerRetries': '1.1-3c9c8b16143ebbb6ad7030e999d14cc0',

81
nova/tests/unit/objects/test_request_spec.py
View File

@ -13,11 +13,15 @@
# under the License.
import mock
from oslo_serialization import jsonutils
from oslo_utils import uuidutils
from nova import context
from nova import exception
from nova import objects
from nova.objects import base
from nova.objects import request_spec
from nova.tests.unit import fake_request_spec
from nova.tests.unit.objects import test_objects
@ -392,6 +396,83 @@ class _TestRequestSpecObject(object):
spec = objects.RequestSpec()
self.assertEqual({}, spec.to_legacy_filter_properties_dict())
@mock.patch.object(request_spec.RequestSpec,
'_get_by_instance_uuid_from_db')
def test_get_by_instance_uuid(self, get_by_uuid):
fake_spec = fake_request_spec.fake_db_spec()
get_by_uuid.return_value = fake_spec
req_obj = request_spec.RequestSpec.get_by_instance_uuid(self.context,
fake_spec['instance_uuid'])
self.assertEqual(1, req_obj.num_instances)
self.assertEqual(['host2', 'host4'], req_obj.ignore_hosts)
self.assertEqual('fake', req_obj.project_id)
self.assertEqual({'hint': ['over-there']}, req_obj.scheduler_hints)
self.assertEqual(['host1', 'host3'], req_obj.force_hosts)
self.assertIsNone(req_obj.availability_zone)
self.assertEqual(['node1', 'node2'], req_obj.force_nodes)
self.assertIsInstance(req_obj.image, objects.ImageMeta)
self.assertIsInstance(req_obj.numa_topology,
objects.InstanceNUMATopology)
self.assertIsInstance(req_obj.pci_requests,
objects.InstancePCIRequests)
self.assertIsInstance(req_obj.flavor, objects.Flavor)
self.assertIsInstance(req_obj.retry, objects.SchedulerRetries)
self.assertIsInstance(req_obj.limits, objects.SchedulerLimits)
self.assertIsInstance(req_obj.instance_group, objects.InstanceGroup)
def _check_update_primitive(self, req_obj, changes):
self.assertEqual(req_obj.instance_uuid, changes['instance_uuid'])
serialized_obj = objects.RequestSpec.obj_from_primitive(
jsonutils.loads(changes['spec']))
# primitive fields
for field in ['instance_uuid', 'num_instances', 'ignore_hosts',
'project_id', 'scheduler_hints', 'force_hosts',
'availability_zone', 'force_nodes']:
self.assertEqual(getattr(req_obj, field),
getattr(serialized_obj, field))
# object fields
for field in ['image', 'numa_topology', 'pci_requests', 'flavor',
'retry', 'limits', 'instance_group']:
self.assertDictEqual(
getattr(req_obj, field).obj_to_primitive(),
getattr(serialized_obj, field).obj_to_primitive())
def test_create(self):
req_obj = fake_request_spec.fake_spec_obj(remove_id=True)
def _test_create_args(self2, context, changes):
self._check_update_primitive(req_obj, changes)
# DB creation would have set an id
changes['id'] = 42
return changes
with mock.patch.object(request_spec.RequestSpec, '_create_in_db',
_test_create_args):
req_obj.create()
def test_create_id_set(self):
req_obj = request_spec.RequestSpec(self.context)
req_obj.id = 3
self.assertRaises(exception.ObjectActionError, req_obj.create)
def test_save(self):
req_obj = fake_request_spec.fake_spec_obj()
def _test_save_args(self2, context, instance_uuid, changes):
self._check_update_primitive(req_obj, changes)
# DB creation would have set an id
changes['id'] = 42
return changes
with mock.patch.object(request_spec.RequestSpec, '_save_in_db',
_test_save_args):
req_obj.save()
class TestRequestSpecObject(test_objects._LocalTest,
_TestRequestSpecObject):