First implementation of bp/live-migration-resource-calc

Fix based on revewer's comment

upgraded the migration version
nova/db/sqlalchemy/migrate_repo/versions/069_block_migration.py

rebase on master

Change-Id: Ia762f8dec761c3d595bc6fcd39f127f6d92306d2

Authors

@@ -88,6 +88,7 @@ Justin Shepherd <jshepher@rackspace.com>
 Kei Masumoto <masumotok@nttdata.co.jp>
 Keisuke Tagami <tagami.keisuke@lab.ntt.co.jp>
 masumoto<masumotok@nttdata.co.jp>
+masukotm<masukotm@nttdata.co.jp>
 Ken Pepple <ken.pepple@gmail.com>
 Kevin Bringard <kbringard@attinteractive.com>
 Kevin L. Mitchell <kevin.mitchell@rackspace.com>
@@ -138,6 +139,7 @@ Stephanie Reese <reese.sm@gmail.com>
 Thierry Carrez <thierry@openstack.org>
 Tim Simpson <tim.simpson@rackspace.com>
 Todd Willey <todd@ansolabs.com>
+Tomoya Masuko<masukotm@nttdata.co.jp>
 Trey Morris <trey.morris@rackspace.com>
 Troy Toman <troy.toman@rackspace.com>
 Tushar Patil <tushar.vitthal.patil@gmail.com>

bin/nova-manage

@@ -980,7 +980,8 @@ class VmCommands(object):
                 instance['availability_zone'],
                 instance['launch_index'])
 
-    def _migration(self, ec2_id, dest, block_migration=False):
+    def _migration(self, ec2_id, dest, block_migration=False,
+                                       disk_over_commit=False):
         """Migrates a running instance to a new machine.
          :param ec2_id: instance id which comes from euca-describe-instance.
          :param dest: destination host name.
@@ -1007,7 +1008,8 @@ class VmCommands(object):
                   "args": {"instance_id": instance_id,
                            "dest": dest,
                            "topic": FLAGS.compute_topic,
-                           "block_migration": block_migration}})
+                           "block_migration": block_migration,
+                           "disk_over_commit": disk_over_commit}})
 
         print _('Migration of %s initiated.'
                'Check its progress using euca-describe-instances.') % ec2_id
@@ -1023,10 +1025,13 @@ class VmCommands(object):
     @args('--ec2_id', dest='ec2_id', metavar='<ec2 id>', help='EC2 ID')
     @args('--dest', dest='dest', metavar='<Destanation>',
                            help='destanation node')
-    def block_migration(self, ec2_id, dest):
+    @args('--disk_over_commit', dest='disk_over_commit',
+                           metavar='<overcommit flag>',
+                           help='Allow overcommit (default Flase)')
+    def block_migration(self, ec2_id, dest, disk_over_commit=False):
         """Migrates a running instance to a new machine with storage data."""
 
-        self._migration(ec2_id, dest, True)
+        self._migration(ec2_id, dest, True, disk_over_commit)
 
 
 class ServiceCommands(object):
@@ -1091,8 +1096,11 @@ class ServiceCommands(object):
 
     @args('--host', dest='host', metavar='<host>', help='Host')
     def describe_resource(self, host):
-        """Describes cpu/memory/hdd info for host."""
+        """Describes cpu/memory/hdd info for host.
 
+        :param host: hostname.
+
+        """
         result = rpc.call(context.get_admin_context(),
                      FLAGS.scheduler_topic,
                      {"method": "show_host_resources",
@@ -1102,49 +1110,66 @@ class ServiceCommands(object):
             print _('An unexpected error has occurred.')
             print _('[Result]'), result
         else:
-            cpu = result['resource']['vcpus']
+            # Printing a total and used_now
-            mem = result['resource']['memory_mb']
+            # (NOTE)The host name width 16 characters
-            hdd = result['resource']['local_gb']
+            print '%(a)-25s%(b)16s%(c)8s%(d)8s%(e)8s' % {"a": _('HOST'),
-            cpu_u = result['resource']['vcpus_used']
+                                                         "b": _('PROJECT'),
-            mem_u = result['resource']['memory_mb_used']
+                                                         "c": _('cpu'),
-            hdd_u = result['resource']['local_gb_used']
+                                                         "d": _('mem(mb)'),
+                                                         "e": _('hdd')}
+            print '%(a)-16s(total)%(b)26s%(c)8s%(d)8s' %\
+                              {"a": host,
+                               "b": result['resource']['vcpus'],
+                               "c": result['resource']['memory_mb'],
+                               "d": result['resource']['local_gb']}
 
+            print '%(a)-16s(used_now)%(b)23s%(c)8s%(d)8s' %\
+                              {"a": host,
+                               "b": result['resource']['vcpus_used'],
+                               "c": result['resource']['memory_mb_used'],
+                               "d": result['resource']['local_gb_used']}
+
+            # Printing a used_max
             cpu_sum = 0
             mem_sum = 0
             hdd_sum = 0
-            print 'HOST\t\t\tPROJECT\t\tcpu\tmem(mb)\tdisk(gb)'
+            ctxt = context.get_admin_context()
-            print '%s(total)\t\t\t%s\t%s\t%s' % (host, cpu, mem, hdd)
+            instance_refs = db.instance_get_all_by_host(ctxt, host)
-            print '%s(used_now)\t\t\t%s\t%s\t%s' % (host, cpu_u, mem_u, hdd_u)
+
-            for p_id, val in result['usage'].items():
+            project_ids = [i['project_id'] for i in instance_refs]
+            project_ids = list(set(project_ids))
+            usage = dict()
+            for project_id in project_ids:
+                vcpus = [i['vcpus'] for i in instance_refs \
+                    if i['project_id'] == project_id]
+
+                mem = [i['memory_mb']  for i in instance_refs \
+                    if i['project_id'] == project_id]
+
+                disk = [i['local_gb']  for i in instance_refs \
+                    if i['project_id'] == project_id]
+
+                usage[project_id] = {
+                                 'vcpus': reduce(lambda x, y: x + y, vcpus),
+                                 'memory_mb': reduce(lambda x, y: x + y, mem),
+                                 'local_gb': reduce(lambda x, y: x + y, disk)}
+
+            for p_id, val in usage.items():
                 cpu_sum += val['vcpus']
                 mem_sum += val['memory_mb']
                 hdd_sum += val['local_gb']
-            print '%s(used_max)\t\t\t%s\t%s\t%s' % (host, cpu_sum,
+            print '%(a)-16s(used_max)%(b)23s%(c)8s%(d)8s' % {"a": host,
-                                                    mem_sum, hdd_sum)
+                                                             "b": cpu_sum,
+                                                             "c": mem_sum,
+                                                             "d": hdd_sum}
 
-            for p_id, val in result['usage'].items():
+            for p_id, val in usage.items():
-                print '%s\t\t%s\t\t%s\t%s\t%s' % (host,
+                print '%(a)-25s%(b)16s%(c)8s%(d)8s%(e)8s' %\
-                                                  p_id,
+                                                     {"a": host,
-                                                  val['vcpus'],
+                                                      "b": p_id,
-                                                  val['memory_mb'],
+                                                      "c": val['vcpus'],
-                                                  val['local_gb'])
+                                                      "d": val['memory_mb'],
-
+                                                      "e": val['local_gb']}
-    @args('--host', dest='host', metavar='<host>', help='Host')
-    def update_resource(self, host):
-        """Updates available vcpu/memory/disk info for host."""
-
-        ctxt = context.get_admin_context()
-        service_refs = db.service_get_all_by_host(ctxt, host)
-        if len(service_refs) <= 0:
-            raise exception.Invalid(_('%s does not exist.') % host)
-
-        service_refs = [s for s in service_refs if s['topic'] == 'compute']
-        if len(service_refs) <= 0:
-            raise exception.Invalid(_('%s is not compute node.') % host)
-
-        rpc.call(ctxt,
-                 db.queue_get_for(ctxt, FLAGS.compute_topic, host),
-                 {"method": "update_available_resource"})
 
 
 class HostCommands(object):

nova/scheduler/multi.py

@@ -20,7 +20,6 @@
 """
 Scheduler that allows routing some calls to one driver and others to another.
 """
-
 from nova import flags
 from nova import utils
 from nova.scheduler import driver
@@ -39,6 +38,7 @@ flags.DEFINE_string('volume_scheduler_driver',
 _METHOD_MAP = {'run_instance': 'compute',
                'start_instance': 'compute',
                'prep_resize': 'compute',
+               'live_migration': 'compute',
                'create_volume': 'volume',
                'create_volumes': 'volume'}
 

nova/tests/fake_libvirt_utils.py

@@ -33,7 +33,7 @@ def get_disk_size(path):
     return disk_sizes.get(path, 1024 * 1024 * 20)
 
 
-def get_backing_file(path):
+def get_disk_backing_file(path):
     return disk_backing_files.get(path, None)
 
 

nova/tests/test_compute.py

@@ -1194,6 +1194,10 @@ class ComputeTestCase(BaseTestCase):
         self.mox.StubOutWithMock(rpc, 'call')
         rpc.call(c, FLAGS.volume_topic, {"method": "check_for_export",
                                          "args": {'instance_id': instance_id}})
+
+        self.mox.StubOutWithMock(self.compute.driver, 'get_instance_disk_info')
+        self.compute.driver.get_instance_disk_info(inst_ref.name)
+
         rpc.call(c, topic, {"method": "pre_live_migration",
                             "args": {'instance_id': instance_id,
                                      'block_migration': True,

nova/tests/test_instance_types.py

@@ -144,7 +144,7 @@ class InstanceTypeTestCase(test.TestCase):
         instance_types.create(name, 256, 1, 120, 'flavor1')
         self.assertRaises(exception.ApiError,
                           instance_types.create,
-                          name, 256, 1, 120, 'flavor2')
+                          name, "256", 1, 120, 'flavor2')
 
     def test_duplicate_flavorids_fail(self):
         """Ensures that flavorid duplicates raise ApiError"""

nova/tests/test_libvirt.py

@@ -757,67 +757,6 @@ class LibvirtConnTestCase(test.TestCase):
             self.assertEquals(conn.uri, testuri)
         db.instance_destroy(user_context, instance_ref['id'])
 
-    def test_update_available_resource_works_correctly(self):
-        """Confirm compute_node table is updated successfully."""
-        self.flags(instances_path='.')
-
-        # Prepare mocks
-        def getVersion():
-            return 12003
-
-        def getType():
-            return 'qemu'
-
-        def listDomainsID():
-            return []
-
-        service_ref = self.create_service(host='dummy')
-        self.create_fake_libvirt_mock(getVersion=getVersion,
-                                      getType=getType,
-                                      listDomainsID=listDomainsID)
-        self.mox.StubOutWithMock(connection.LibvirtConnection,
-                                 'get_cpu_info')
-        connection.LibvirtConnection.get_cpu_info().AndReturn('cpuinfo')
-
-        # Start test
-        self.mox.ReplayAll()
-        conn = connection.LibvirtConnection(False)
-        conn.update_available_resource(self.context, 'dummy')
-        service_ref = db.service_get(self.context, service_ref['id'])
-        compute_node = service_ref['compute_node'][0]
-
-        if sys.platform.upper() == 'LINUX2':
-            self.assertTrue(compute_node['vcpus'] >= 0)
-            self.assertTrue(compute_node['memory_mb'] > 0)
-            self.assertTrue(compute_node['local_gb'] > 0)
-            self.assertTrue(compute_node['vcpus_used'] == 0)
-            self.assertTrue(compute_node['memory_mb_used'] > 0)
-            self.assertTrue(compute_node['local_gb_used'] > 0)
-            self.assertTrue(len(compute_node['hypervisor_type']) > 0)
-            self.assertTrue(compute_node['hypervisor_version'] > 0)
-        else:
-            self.assertTrue(compute_node['vcpus'] >= 0)
-            self.assertTrue(compute_node['memory_mb'] == 0)
-            self.assertTrue(compute_node['local_gb'] > 0)
-            self.assertTrue(compute_node['vcpus_used'] == 0)
-            self.assertTrue(compute_node['memory_mb_used'] == 0)
-            self.assertTrue(compute_node['local_gb_used'] > 0)
-            self.assertTrue(len(compute_node['hypervisor_type']) > 0)
-            self.assertTrue(compute_node['hypervisor_version'] > 0)
-
-        db.service_destroy(self.context, service_ref['id'])
-
-    def test_update_resource_info_no_compute_record_found(self):
-        """Raise exception if no recorde found on services table."""
-        self.flags(instances_path='.')
-        self.create_fake_libvirt_mock()
-
-        self.mox.ReplayAll()
-        conn = connection.LibvirtConnection(False)
-        self.assertRaises(exception.ComputeServiceUnavailable,
-                          conn.update_available_resource,
-                          self.context, 'dummy')
-
     @test.skip_if(missing_libvirt(), "Test requires libvirt")
     def test_ensure_filtering_rules_for_instance_timeout(self):
         """ensure_filtering_fules_for_instance() finishes with timeout."""
@@ -950,7 +889,7 @@ class LibvirtConnTestCase(test.TestCase):
 
         # Test data
         instance_ref = db.instance_create(self.context, self.test_instance)
-        dummyjson = ('[{"path": "%s/disk", "local_gb": "10G",'
+        dummyjson = ('[{"path": "%s/disk", "disk_size": "10737418240",'
                      ' "type": "raw", "backing_file": ""}]')
 
         # Preparing mocks
@@ -984,6 +923,13 @@ class LibvirtConnTestCase(test.TestCase):
                     "<target dev='vdb' bus='virtio'/></disk>"
                     "</devices></domain>")
 
+        ret = ("image: /test/disk\n"
+               "file format: raw\n"
+               "virtual size: 20G (21474836480 bytes)\n"
+               "disk size: 3.1G\n"
+               "cluster_size: 2097152\n"
+               "backing file: /test/dummy (actual path: /backing/file)\n")
+
         # Preparing mocks
         vdmock = self.mox.CreateMock(libvirt.virDomain)
         self.mox.StubOutWithMock(vdmock, "XMLDesc")
@@ -998,18 +944,27 @@ class LibvirtConnTestCase(test.TestCase):
         fake_libvirt_utils.disk_sizes['/test/disk'] = 10 * GB
         fake_libvirt_utils.disk_sizes['/test/disk.local'] = 20 * GB
         fake_libvirt_utils.disk_backing_files['/test/disk.local'] = 'file'
+
+        self.mox.StubOutWithMock(os.path, "getsize")
+        os.path.getsize('/test/disk').AndReturn((10737418240))
+
+        self.mox.StubOutWithMock(utils, "execute")
+        utils.execute('qemu-img', 'info', '/test/disk.local').\
+            AndReturn((ret, ''))
+
+        os.path.getsize('/test/disk.local').AndReturn((21474836480))
+
         self.mox.ReplayAll()
         conn = connection.LibvirtConnection(False)
-        info = conn.get_instance_disk_info(self.context, instance_ref)
+        info = conn.get_instance_disk_info(instance_ref.name)
         info = utils.loads(info)
-
         self.assertEquals(info[0]['type'], 'raw')
-        self.assertEquals(info[1]['type'], 'qcow2')
         self.assertEquals(info[0]['path'], '/test/disk')
-        self.assertEquals(info[1]['path'], '/test/disk.local')
+        self.assertEquals(info[0]['disk_size'], 10737418240)
-        self.assertEquals(info[0]['local_gb'], '10G')
-        self.assertEquals(info[1]['local_gb'], '20G')
         self.assertEquals(info[0]['backing_file'], "")
+        self.assertEquals(info[1]['type'], 'qcow2')
+        self.assertEquals(info[1]['path'], '/test/disk.local')
+        self.assertEquals(info[1]['virt_disk_size'], 21474836480)
         self.assertEquals(info[1]['backing_file'], "file")
 
         db.instance_destroy(self.context, instance_ref['id'])
@@ -1188,6 +1143,9 @@ class HostStateTestCase(test.TestCase):
         def get_hypervisor_version(self):
             return 13091
 
+        def get_disk_available_least(self):
+            return 13091
+
     def test_update_status(self):
         self.mox.StubOutWithMock(connection, 'get_connection')
         connection.get_connection(True).AndReturn(self.FakeConnection())

nova/tests/test_virt_drivers.py

@@ -323,11 +323,6 @@ class _VirtDriverTestCase(test.TestCase):
         instance_ref, network_info = self._get_running_instance()
         self.connection.refresh_provider_fw_rules()
 
-    @catch_notimplementederror
-    def test_update_available_resource(self):
-        self.compute = self.start_service('compute', host='dummy')
-        self.connection.update_available_resource(self.ctxt, 'dummy')
-
     @catch_notimplementederror
     def test_compare_cpu(self):
         cpu_info = '''{ "topology": {