From 0ce4dff41f8d38edf790a301ec8e7040b279d65a Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Thu, 27 Apr 2017 19:44:51 -0400 Subject: [PATCH] Add ComputeNode.mapped field This is an integer that defaults to zero. This can be used by API services to efficiently filter out records that have already been mapped by a HostMapping. This is an integer instead of a boolean in case we later need to do some other sort of mapping of these records and need to distinguish yet another level of mapped-ness. Note this also removes some tests from TestNewtonBlocker which can no longer work with the new ComputeNode model since they attempt to use it to create compute nodes at a schema older than needed for this new field. The point of the test was to verify, in Newton, that the blocker migration caught the thing we needed to catch. It doesn't need to be in master anymore and certainly is not worth the acrobatics that would be required to keep it working. (Note that we haven't even had such tests for many of our blocker migrations) Related to blueprint discover-hosts-faster Change-Id: I902d75efb0bbe177680d7211c23235f42497e3fe --- .../versions/360_add_compute_node_mapped.py | 24 ++++++++++ nova/db/sqlalchemy/models.py | 1 + nova/objects/compute_node.py | 10 +++- nova/tests/unit/compute/test_compute.py | 1 + nova/tests/unit/db/test_db_api.py | 2 + nova/tests/unit/db/test_migrations.py | 4 ++ .../unit/db/test_sqlalchemy_migration.py | 48 ------------------- nova/tests/unit/objects/test_compute_node.py | 13 +++++ nova/tests/unit/objects/test_objects.py | 2 +- 9 files changed, 55 insertions(+), 50 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/360_add_compute_node_mapped.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/360_add_compute_node_mapped.py b/nova/db/sqlalchemy/migrate_repo/versions/360_add_compute_node_mapped.py new file mode 100644 index 000000000000..d5b2acaea658 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/360_add_compute_node_mapped.py @@ -0,0 +1,24 @@ +# Copyright 2017 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import MetaData, Table, Column, Integer + + +def upgrade(migrate_engine): + meta = MetaData(bind=migrate_engine) + + for prefix in ('', 'shadow_'): + compute_nodes = Table('%scompute_nodes' % prefix, meta, autoload=True) + mapped = Column('mapped', Integer, default=0, nullable=True) + if not hasattr(compute_nodes.c, 'mapped'): + compute_nodes.create_column(mapped) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 0e8894267dc6..766aa0bad5cc 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -179,6 +179,7 @@ class ComputeNode(BASE, NovaBase, models.SoftDeleteMixin): ram_allocation_ratio = Column(Float, nullable=True) cpu_allocation_ratio = Column(Float, nullable=True) disk_allocation_ratio = Column(Float, nullable=True) + mapped = Column(Integer, nullable=True, default=0) class Certificate(BASE, NovaBase, models.SoftDeleteMixin): diff --git a/nova/objects/compute_node.py b/nova/objects/compute_node.py index 6f83b39a76d4..8be7d6bc2090 100644 --- a/nova/objects/compute_node.py +++ b/nova/objects/compute_node.py @@ -48,7 +48,8 @@ class ComputeNode(base.NovaPersistentObject, base.NovaObject): # Version 1.14: Added cpu_allocation_ratio and ram_allocation_ratio # Version 1.15: Added uuid # Version 1.16: Added disk_allocation_ratio - VERSION = '1.16' + # Version 1.17: Added mapped + VERSION = '1.17' fields = { 'id': fields.IntegerField(read_only=True), @@ -89,11 +90,15 @@ class ComputeNode(base.NovaPersistentObject, base.NovaObject): 'cpu_allocation_ratio': fields.FloatField(), 'ram_allocation_ratio': fields.FloatField(), 'disk_allocation_ratio': fields.FloatField(), + 'mapped': fields.IntegerField(), } def obj_make_compatible(self, primitive, target_version): super(ComputeNode, self).obj_make_compatible(primitive, target_version) target_version = versionutils.convert_version_to_tuple(target_version) + if target_version < (1, 17): + if 'mapped' in primitive: + del primitive['mapped'] if target_version < (1, 16): if 'disk_allocation_ratio' in primitive: del primitive['disk_allocation_ratio'] @@ -199,6 +204,9 @@ class ComputeNode(base.NovaPersistentObject, base.NovaObject): if value == 0.0 and key == 'disk_allocation_ratio': # It's not specified either on the controller value = 1.0 + elif key == 'mapped': + value = 0 if value is None else value + setattr(compute, key, value) stats = db_compute['stats'] diff --git a/nova/tests/unit/compute/test_compute.py b/nova/tests/unit/compute/test_compute.py index 2b12090e947d..8cd31e2c61ab 100644 --- a/nova/tests/unit/compute/test_compute.py +++ b/nova/tests/unit/compute/test_compute.py @@ -176,6 +176,7 @@ class BaseTestCase(test.TestCase): 'memory_mb': 131072, 'current_workload': 0, 'vcpus': 16, + 'mapped': 1, 'cpu_info': 'ppc64,powervm,3940', 'running_vms': 0, 'free_disk_gb': 259, diff --git a/nova/tests/unit/db/test_db_api.py b/nova/tests/unit/db/test_db_api.py index 46f4358f1c61..2391a6775402 100644 --- a/nova/tests/unit/db/test_db_api.py +++ b/nova/tests/unit/db/test_db_api.py @@ -7801,6 +7801,7 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin): supported_instances='', pci_stats='', metrics='', + mapped=0, extra_resources='', cpu_allocation_ratio=16.0, ram_allocation_ratio=1.5, @@ -7850,6 +7851,7 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin): supported_instances='', pci_stats='', metrics='', + mapped=0, extra_resources='', cpu_allocation_ratio=16.0, ram_allocation_ratio=1.5, diff --git a/nova/tests/unit/db/test_migrations.py b/nova/tests/unit/db/test_migrations.py index 5580df3419bd..3887a8d286e5 100644 --- a/nova/tests/unit/db/test_migrations.py +++ b/nova/tests/unit/db/test_migrations.py @@ -952,6 +952,10 @@ class NovaMigrationsCheckers(test_migrations.ModelsMigrationsSync, self.assertIndexMembers(engine, 'services', 'services_uuid_idx', ['uuid']) + def _check_360(self, engine, data): + self.assertColumnExists(engine, 'compute_nodes', 'mapped') + self.assertColumnExists(engine, 'shadow_compute_nodes', 'mapped') + class TestNovaMigrationsSQLite(NovaMigrationsCheckers, test_base.DbTestCase, diff --git a/nova/tests/unit/db/test_sqlalchemy_migration.py b/nova/tests/unit/db/test_sqlalchemy_migration.py index e53f8d08705d..f5dbab723cba 100644 --- a/nova/tests/unit/db/test_sqlalchemy_migration.py +++ b/nova/tests/unit/db/test_sqlalchemy_migration.py @@ -239,29 +239,6 @@ class TestNewtonCheck(test.TestCase): '330_enforce_mitaka_online_migrations') self.engine = db_api.get_engine() - def test_all_migrated(self): - cn = objects.ComputeNode(context=self.context, - vcpus=1, memory_mb=512, local_gb=10, - vcpus_used=0, memory_mb_used=256, - local_gb_used=5, hypervisor_type='HyperDanVM', - hypervisor_version='34', cpu_info='foo') - cn.create() - objects.Aggregate(context=self.context, - name='foo').create() - objects.PciDevice.create(self.context, {}) - self.migration.upgrade(self.engine) - - def test_cn_not_migrated(self): - cn = objects.ComputeNode(context=self.context, - vcpus=1, memory_mb=512, local_gb=10, - vcpus_used=0, memory_mb_used=256, - local_gb_used=5, hypervisor_type='HyperDanVM', - hypervisor_version='34', cpu_info='foo') - cn.create() - db_api.compute_node_update(self.context, cn.id, {'uuid': None}) - self.assertRaises(exception.ValidationError, - self.migration.upgrade, self.engine) - def test_aggregate_not_migrated(self): agg = db_api.aggregate_create(self.context, {"name": "foobar"}) db_api.aggregate_update(self.context, agg.id, {'uuid': None}) @@ -308,31 +285,6 @@ class TestNewtonCheck(test.TestCase): # blocker should not block on type-PCI devices self.migration.upgrade(self.engine) - def test_deleted_not_migrated(self): - cn_values = dict(vcpus=1, memory_mb=512, local_gb=10, - vcpus_used=0, memory_mb_used=256, - local_gb_used=5, hypervisor_type='HyperDanVM', - hypervisor_version='34', cpu_info='foo') - cn = db_api.compute_node_create(self.context, cn_values) - agg_values = dict(name='foo') - agg = db_api.aggregate_create(self.context, agg_values) - pd = db_api.pci_device_update(self.context, 1, 'foo:bar', - {'parent_addr': None, - 'compute_node_id': 1, - 'address': 'foo:bar', - 'vendor_id': '123', - 'product_id': '456', - 'dev_type': 'foo', - 'label': 'foobar', - 'status': 'whatisthis?'}) - db_api.compute_node_delete(self.context, cn['id']) - db_api.aggregate_delete(self.context, agg['id']) - db_api.pci_device_destroy(self.context, pd['compute_node_id'], - pd['address']) - - # blocker should not block on soft-deleted records - self.migration.upgrade(self.engine) - class TestOcataCheck(test.TestCase): def setUp(self): diff --git a/nova/tests/unit/objects/test_compute_node.py b/nova/tests/unit/objects/test_compute_node.py index 95373d382a09..07f20a196a34 100644 --- a/nova/tests/unit/objects/test_compute_node.py +++ b/nova/tests/unit/objects/test_compute_node.py @@ -90,6 +90,7 @@ fake_compute_node = { 'cpu_allocation_ratio': 16.0, 'ram_allocation_ratio': 1.5, 'disk_allocation_ratio': 1.0, + 'mapped': 0, } # FIXME(sbauza) : For compatibility checking, to be removed once we are sure # that all computes are running latest DB version with host field in it. @@ -163,6 +164,18 @@ class _TestComputeNodeObject(object): self.assertNotIn('uuid', compute.obj_what_changed()) get_mock.assert_called_once_with(self.context, 123) + @mock.patch.object(db, 'compute_node_get') + def test_get_without_mapped(self, get_mock): + fake_node = copy.copy(fake_compute_node) + fake_node['mapped'] = None + get_mock.return_value = fake_node + compute = compute_node.ComputeNode.get_by_id(self.context, 123) + self.compare_obj(compute, fake_compute_node, + subs=self.subs(), + comparators=self.comparators()) + self.assertIn('mapped', compute) + self.assertEqual(0, compute.mapped) + @mock.patch.object(objects.Service, 'get_by_id') @mock.patch.object(db, 'compute_node_get') def test_get_by_id_with_host_field_not_in_db(self, mock_cn_get, diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index 870e14289611..9ea4cb62b7e8 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1072,7 +1072,7 @@ object_data = { 'BuildRequestList': '1.0-cd95608eccb89fbc702c8b52f38ec738', 'CellMapping': '1.0-7f1a7e85a22bbb7559fc730ab658b9bd', 'CellMappingList': '1.0-4ee0d9efdfd681fed822da88376e04d2', - 'ComputeNode': '1.16-2436e5b836fa0306a3c4e6d9e5ddacec', + 'ComputeNode': '1.17-abc222ef5a707dcd3e3d32b48197c9e7', 'ComputeNodeList': '1.16-40258d802a6ed045690a127a2088544b', 'DNSDomain': '1.0-7b0b2dab778454b6a7b6c66afe163a1a', 'DNSDomainList': '1.0-4ee0d9efdfd681fed822da88376e04d2',