diff --git a/doc/source/cli/nova-manage.rst b/doc/source/cli/nova-manage.rst index 149e3b990128..85750a4e3720 100644 --- a/doc/source/cli/nova-manage.rst +++ b/doc/source/cli/nova-manage.rst @@ -77,6 +77,32 @@ Nova Database call. If not specified, migration will occur in batches of 50 until fully complete. +``nova-manage db ironic_flavor_migration [--all] [--host] [--node] [--resource_class]`` + + Perform the ironic flavor migration process against the database + while services are offline. This is `not recommended` for most + people. The ironic compute driver will do this online and as + necessary if run normally. This routine is provided only for + advanced users that may be skipping the 16.0.0 Pike release, never + able to run services normally at the Pike level. Since this utility + is for use when all services (including ironic) are down, you must + pass the resource class set on your node(s) with the + ``--resource_class`` parameter. + + To migrate a specific host and node, provide the hostname and node uuid with + ``--host $hostname --node $uuid``. To migrate all instances on nodes managed + by a single host, provide only ``--host``. To iterate over all nodes in the + system in a single pass, use ``--all``. Note that this process is not lightweight, + so it should not be run frequently without cause, although it is not harmful + to do so. If you have multiple cellsv2 cells, you should run this once per cell + with the corresponding cell config for each (i.e. this does not iterate cells + automatically). + + Note that this is not recommended unless you need to run this + specific data migration offline, and it should be used with care as + the work done is non-trivial. Running smaller and more targeted batches (such as + specific nodes) is recommended. + Nova API Database ~~~~~~~~~~~~~~~~~ diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py index e5dbf05074c8..304f1b614cb7 100644 --- a/nova/cmd/manage.py +++ b/nova/cmd/manage.py @@ -100,6 +100,7 @@ from nova import quota from nova import rpc from nova import utils from nova import version +from nova.virt import ironic CONF = nova.conf.CONF @@ -896,6 +897,102 @@ Error: %s""") % six.text_type(e)) return ran and 1 or 0 + @args('--resource_class', metavar='', required=True, + help='Ironic node class to set on instances') + @args('--host', metavar='', required=False, + help='Compute service name to migrate nodes on') + @args('--node', metavar='', required=False, + help='Ironic node UUID to migrate (all on the host if omitted)') + @args('--all', action='store_true', default=False, dest='all_hosts', + help='Run migrations for all ironic hosts and nodes') + @args('--verbose', action='store_true', default=False, + help='Print information about migrations being performed') + def ironic_flavor_migration(self, resource_class, host=None, node=None, + all_hosts=False, verbose=False): + """Migrate flavor information for ironic instances. + + This will manually push the instance flavor migration required + for ironic-hosted instances in Pike. The best way to accomplish + this migration is to run your ironic computes normally in Pike. + However, if you need to push the migration manually, then use + this. + + This is idempotent, but not trivial to start/stop/resume. It is + recommended that you do this with care and not from a script + assuming it is trivial. + + Running with --all may generate a large amount of DB traffic + all at once. Running at least one host at a time is recommended + for batching. + + Return values: + + 0: All work is completed (or none is needed) + 1: Specified host and/or node is not found, or no ironic nodes present + 2: Internal accounting error shows more than one instance per node + 3: Invalid combination of required arguments + """ + if not resource_class: + # Note that if --resource_class is not specified on the command + # line it will actually result in a return code of 2, but we + # leave 3 here for testing purposes. + print(_('A resource_class is required for all modes of operation')) + return 3 + + ctx = context.get_admin_context() + + if all_hosts: + if host or node: + print(_('--all with --host and/or --node does not make sense')) + return 3 + cns = objects.ComputeNodeList.get_by_hypervisor_type(ctx, 'ironic') + elif host and node: + try: + cn = objects.ComputeNode.get_by_host_and_nodename(ctx, host, + node) + cns = [cn] + except exception.ComputeHostNotFound: + cns = [] + elif host: + try: + cns = objects.ComputeNodeList.get_all_by_host(ctx, host) + except exception.ComputeHostNotFound: + cns = [] + else: + print(_('Either --all, --host, or --host and --node are required')) + return 3 + + if len(cns) == 0: + print(_('No ironic compute nodes found that match criteria')) + return 1 + + # Check that we at least got one ironic compute and we can pretty + # safely assume the rest are + if cns[0].hypervisor_type != 'ironic': + print(_('Compute node(s) specified is not of type ironic')) + return 1 + + for cn in cns: + # NOTE(danms): The instance.node is the + # ComputeNode.hypervisor_hostname, which in the case of ironic is + # the node uuid. Since only one instance can be on a node in + # ironic, do another sanity check here to make sure we look legit. + inst = objects.InstanceList.get_by_filters( + ctx, {'node': cn.hypervisor_hostname, + 'deleted': False}) + if len(inst) > 1: + print(_('Ironic node %s has multiple instances? ' + 'Something is wrong.') % cn.hypervisor_hostname) + return 2 + elif len(inst) == 1: + result = ironic.IronicDriver._pike_flavor_migration_for_node( + ctx, resource_class, inst[0].uuid) + if result and verbose: + print(_('Migrated instance %(uuid)s on node %(node)s') % { + 'uuid': inst[0].uuid, + 'node': cn.hypervisor_hostname}) + return 0 + class ApiDbCommands(object): """Class for managing the api database.""" diff --git a/nova/tests/functional/test_nova_manage.py b/nova/tests/functional/test_nova_manage.py new file mode 100644 index 000000000000..47fca2b1a123 --- /dev/null +++ b/nova/tests/functional/test_nova_manage.py @@ -0,0 +1,266 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova.cmd import manage +from nova import context +from nova import objects +from nova import test + + +class NovaManageDBIronicTest(test.TestCase): + def setUp(self): + super(NovaManageDBIronicTest, self).setUp() + self.commands = manage.DbCommands() + self.context = context.RequestContext('fake-user', 'fake-project') + + self.service1 = objects.Service(context=self.context, + host='fake-host1', + binary='nova-compute', + topic='fake-host1', + report_count=1, + disabled=False, + disabled_reason=None, + availability_zone='nova', + forced_down=False) + self.service1.create() + + self.service2 = objects.Service(context=self.context, + host='fake-host2', + binary='nova-compute', + topic='fake-host2', + report_count=1, + disabled=False, + disabled_reason=None, + availability_zone='nova', + forced_down=False) + self.service2.create() + + self.service3 = objects.Service(context=self.context, + host='fake-host3', + binary='nova-compute', + topic='fake-host3', + report_count=1, + disabled=False, + disabled_reason=None, + availability_zone='nova', + forced_down=False) + self.service3.create() + + self.cn1 = objects.ComputeNode(context=self.context, + service_id=self.service1.id, + host='fake-host1', + hypervisor_type='ironic', + vcpus=1, + memory_mb=1024, + local_gb=10, + vcpus_used=1, + memory_mb_used=1024, + local_gb_used=10, + hypervisor_version=0, + hypervisor_hostname='fake-node1', + cpu_info='{}') + self.cn1.create() + + self.cn2 = objects.ComputeNode(context=self.context, + service_id=self.service1.id, + host='fake-host1', + hypervisor_type='ironic', + vcpus=1, + memory_mb=1024, + local_gb=10, + vcpus_used=1, + memory_mb_used=1024, + local_gb_used=10, + hypervisor_version=0, + hypervisor_hostname='fake-node2', + cpu_info='{}') + self.cn2.create() + + self.cn3 = objects.ComputeNode(context=self.context, + service_id=self.service2.id, + host='fake-host2', + hypervisor_type='ironic', + vcpus=1, + memory_mb=1024, + local_gb=10, + vcpus_used=1, + memory_mb_used=1024, + local_gb_used=10, + hypervisor_version=0, + hypervisor_hostname='fake-node3', + cpu_info='{}') + self.cn3.create() + + self.cn4 = objects.ComputeNode(context=self.context, + service_id=self.service3.id, + host='fake-host3', + hypervisor_type='libvirt', + vcpus=1, + memory_mb=1024, + local_gb=10, + vcpus_used=1, + memory_mb_used=1024, + local_gb_used=10, + hypervisor_version=0, + hypervisor_hostname='fake-node4', + cpu_info='{}') + self.cn4.create() + + self.cn5 = objects.ComputeNode(context=self.context, + service_id=self.service2.id, + host='fake-host2', + hypervisor_type='ironic', + vcpus=1, + memory_mb=1024, + local_gb=10, + vcpus_used=1, + memory_mb_used=1024, + local_gb_used=10, + hypervisor_version=0, + hypervisor_hostname='fake-node5', + cpu_info='{}') + self.cn5.create() + + self.insts = [] + for cn in (self.cn1, self.cn2, self.cn3, self.cn4, self.cn4, self.cn5): + flavor = objects.Flavor(extra_specs={}) + inst = objects.Instance(context=self.context, + user_id=self.context.user_id, + project_id=self.context.project_id, + flavor=flavor, + node=cn.hypervisor_hostname) + inst.create() + self.insts.append(inst) + + self.ironic_insts = [i for i in self.insts + if i.node != self.cn4.hypervisor_hostname] + self.virt_insts = [i for i in self.insts + if i.node == self.cn4.hypervisor_hostname] + + def test_ironic_flavor_migration_by_host_and_node(self): + ret = self.commands.ironic_flavor_migration('test', 'fake-host1', + 'fake-node2', False, False) + self.assertEqual(0, ret) + k = 'resources:CUSTOM_TEST' + + for inst in self.ironic_insts: + inst.refresh() + if inst.node == 'fake-node2': + self.assertIn(k, inst.flavor.extra_specs) + self.assertEqual('1', inst.flavor.extra_specs[k]) + else: + self.assertNotIn(k, inst.flavor.extra_specs) + + for inst in self.virt_insts: + inst.refresh() + self.assertNotIn(k, inst.flavor.extra_specs) + + def test_ironic_flavor_migration_by_host(self): + ret = self.commands.ironic_flavor_migration('test', 'fake-host1', None, + False, False) + self.assertEqual(0, ret) + k = 'resources:CUSTOM_TEST' + + for inst in self.ironic_insts: + inst.refresh() + if inst.node in ('fake-node1', 'fake-node2'): + self.assertIn(k, inst.flavor.extra_specs) + self.assertEqual('1', inst.flavor.extra_specs[k]) + else: + self.assertNotIn(k, inst.flavor.extra_specs) + + for inst in self.virt_insts: + inst.refresh() + self.assertNotIn(k, inst.flavor.extra_specs) + + def test_ironic_flavor_migration_by_host_not_ironic(self): + ret = self.commands.ironic_flavor_migration('test', 'fake-host3', None, + False, False) + self.assertEqual(1, ret) + k = 'resources:CUSTOM_TEST' + + for inst in self.ironic_insts: + inst.refresh() + self.assertNotIn(k, inst.flavor.extra_specs) + + for inst in self.virt_insts: + inst.refresh() + self.assertNotIn(k, inst.flavor.extra_specs) + + def test_ironic_flavor_migration_all_hosts(self): + ret = self.commands.ironic_flavor_migration('test', None, None, + True, False) + self.assertEqual(0, ret) + k = 'resources:CUSTOM_TEST' + + for inst in self.ironic_insts: + inst.refresh() + self.assertIn(k, inst.flavor.extra_specs) + self.assertEqual('1', inst.flavor.extra_specs[k]) + + for inst in self.virt_insts: + inst.refresh() + self.assertNotIn(k, inst.flavor.extra_specs) + + def test_ironic_flavor_migration_invalid(self): + # No host or node and not "all" + ret = self.commands.ironic_flavor_migration('test', None, None, + False, False) + self.assertEqual(3, ret) + + # No host, only node + ret = self.commands.ironic_flavor_migration('test', None, 'fake-node', + False, False) + self.assertEqual(3, ret) + + # Asked for all but provided a node + ret = self.commands.ironic_flavor_migration('test', None, 'fake-node', + True, False) + self.assertEqual(3, ret) + + # Asked for all but provided a host + ret = self.commands.ironic_flavor_migration('test', 'fake-host', None, + True, False) + self.assertEqual(3, ret) + + # Asked for all but provided a host and node + ret = self.commands.ironic_flavor_migration('test', 'fake-host', + 'fake-node', True, False) + self.assertEqual(3, ret) + + # Did not provide a resource_class + ret = self.commands.ironic_flavor_migration(None, 'fake-host', + 'fake-node', False, False) + self.assertEqual(3, ret) + + def test_ironic_flavor_migration_no_match(self): + ret = self.commands.ironic_flavor_migration('test', 'fake-nonexist', + None, False, False) + self.assertEqual(1, ret) + + ret = self.commands.ironic_flavor_migration('test', 'fake-nonexist', + 'fake-node', False, False) + self.assertEqual(1, ret) + + def test_ironic_two_instances(self): + # NOTE(danms): This shouldn't be possible, but simulate it like + # someone hacked the database, which should also cover any other + # way this could happen. + + # Since we created two instances on cn4 in setUp() we can convert that + # to an ironic host and cause the two-instances-on-one-ironic paradox + # to happen. + self.cn4.hypervisor_type = 'ironic' + self.cn4.save() + ret = self.commands.ironic_flavor_migration('test', 'fake-host3', + 'fake-node4', False, False) + self.assertEqual(2, ret) diff --git a/releasenotes/notes/ironic_offline_flavor_migration-4845307799f0e24e.yaml b/releasenotes/notes/ironic_offline_flavor_migration-4845307799f0e24e.yaml new file mode 100644 index 000000000000..9aeb67185e57 --- /dev/null +++ b/releasenotes/notes/ironic_offline_flavor_migration-4845307799f0e24e.yaml @@ -0,0 +1,9 @@ +--- +other: + - | + The ironic driver will automatically migrate instance flavors for resource classes + at runtime. If you are not able to run the compute and ironic services at pike because + you are automating an upgrade past this release, you can use the + ``nova-manage db ironic_flavor_migration`` to push the migration manually. This is only + for advanced users taking on the risk of automating the process of upgrading through + pike and is not recommended for normal users. \ No newline at end of file