make compute get the new images properly, fix a bunch of tests, and provide conversion commands

bin/nova-manage

@@ -55,6 +55,8 @@
 
 import datetime
 import gettext
+import glob
+import json
 import os
 import re
 import sys
@@ -81,7 +83,7 @@ from nova import log as logging
 from nova import quota
 from nova import rpc
 from nova import utils
-from nova.api.ec2.ec2utils import ec2_id_to_id
+from nova.api.ec2 import ec2utils
 from nova.auth import manager
 from nova.cloudpipe import pipelib
 from nova.compute import instance_types
@@ -104,7 +106,7 @@ def param2id(object_id):
     args: [object_id], e.g. 'vol-0000000a' or 'volume-0000000a' or '10'
     """
     if '-' in object_id:
-        return ec2_id_to_id(object_id)
+        return ec2utils.ec2_id_to_id(object_id)
     else:
         return int(object_id)
 
@@ -735,6 +737,130 @@ class InstanceTypeCommands(object):
             self._print_instance_types(name, inst_types)
 
 
+class ImageCommands(object):
+    """Methods for dealing with a cloud in an odd state"""
+
+    def __init__(self, *args, **kwargs):
+        self.image_service = utils.import_object(FLAGS.image_service)
+
+    def _register(self, image_type, path, owner, name=None,
+                 is_public='T', architecture='x86_64',
+                 kernel_id=None, ramdisk_id=None):
+        meta = {'type': image_type,
+                'is_public': True,
+                'name': name,
+                'properties': {'image_state': 'available',
+                               'owner': owner,
+                               'architecture': architecture,
+                               'image_location': 'local',
+                               'is_public': (is_public == 'T')}}
+        if kernel_id:
+            meta['properties']['kernel_id'] = int(kernel_id)
+        if ramdisk_id:
+            meta['properties']['ramdisk_id'] = int(ramdisk_id)
+        elevated = context.get_admin_context()
+        try:
+            with open(path) as ifile:
+                image = self.image_service.create(elevated, meta, ifile)
+            new = image['id']
+            print _("Image registered to %(new)s (%(new)08x).") % locals()
+            return new
+        except Exception as exc:
+            print _("Failed to register %(path)s: %(exc)s") % locals()
+
+    def register_all(self, image, kernel, ramdisk, owner, name=None,
+                 is_public='T', architecture='x86_64'):
+        """Uploads an image, kernel, and ramdisk into the image_service
+        arguments: image kernel ramdisk owner [name] [is_public='T']
+                   [architecture='x86_64']"""
+        kernel_id = self._register('kernel', kernel, owner, None,
+                                   is_public, architecture)
+        ramdisk_id = self._register('ramdisk', ramdisk, owner, None,
+                                    is_public, architecture)
+        self._register('machine', image, owner, name, is_public,
+                       architecture, kernel_id, ramdisk_id)
+
+    def image_register(self, path, owner, name=None, is_public='T',
+              architecture='x86_64', kernel_id=None, ramdisk_id=None):
+        """Uploads an image into the image_service
+        arguments: path owner [name] [is_public='T'] [architecture='x86_64']
+                   [kernel_id] [ramdisk_id]"""
+        self._register('machine', path, owner, name, is_public,
+                       architecture, kernel_id, ramdisk_id)
+
+    def kernel_register(self, path, owner, name=None, is_public='T',
+               architecture='x86_64'):
+        """Uploads a kernel into the image_service
+        arguments: path owner [name] [is_public='T'] [architecture='x86_64']
+        """
+        self._register('kernel', path, owner, name, is_public, architecture)
+
+    def ramdisk_register(self, path, owner, name=None, is_public='T',
+                architecture='x86_64'):
+        """Uploads a ramdisk into the image_service
+        arguments: path owner [name] [is_public='T'] [architecture='x86_64']
+        """
+        self._register('ramdisk', path, owner, name, is_public, architecture)
+
+    def _lookup(self, old_image_id):
+        try:
+            internal_id = ec2utils.ec2_id_to_id(old_image_id)
+            image = self.image_service.show(context, internal_id)
+        except exception.NotFound:
+            image = self.image_service.show_by_name(context, old_image_id)
+        return image['id']
+
+    def _old_to_new(self, old):
+        new = {'type': old['type'],
+               'is_public': True,
+               'name': old['imageId'],
+               'properties': {'image_state': old['imageState'],
+                              'owner': old['imageOwnerId'],
+                              'architecture': old['architecture'],
+                              'image_location': old['imageLocation'],
+                              'is_public': old['isPublic']}}
+        if old.get('kernelId'):
+            new['properties']['kernel_id'] = self._lookup(old['kernelId'])
+        if old.get('ramdiskId'):
+            new['properties']['ramdisk_id'] = self._lookup(old['ramdiskId'])
+        return new
+
+    def _convert_images(self, images):
+        elevated = context.get_admin_context()
+        for image_path, image_metadata in images.iteritems():
+            meta = self._old_to_new(image_metadata)
+            old = meta['name']
+            try:
+                with open(image_path) as ifile:
+                    image = self.image_service.create(elevated, meta, ifile)
+                new = image['id']
+                print _("Image %(old)s converted to " \
+                        "%(new)s (%(new)08x).") % locals()
+            except Exception as exc:
+                print _("Failed to convert %(old)s: %(exc)s") % locals()
+
+
+    def convert(self, directory):
+        """Uploads old objectstore images in directory to new service
+        arguments: directory"""
+        machine_images = {}
+        other_images = {}
+        for fn in glob.glob("%s/*/info.json" % directory):
+            try:
+                image_path = os.path.join(fn.rpartition('/')[0], 'image')
+                with open(fn) as metadata_file:
+                    image_metadata = json.load(metadata_file)
+                if image_metadata['type'] == 'machine':
+                    machine_images[image_path] = image_metadata
+                else:
+                    other_images[image_path] = image_metadata
+            except Exception as exc:
+                print _("Failed to load %(fn)s.") % locals()
+        # NOTE(vish): do kernels and ramdisks first so images
+        self._convert_images(other_images)
+        self._convert_images(machine_images)
+
+
 CATEGORIES = [
     ('user', UserCommands),
     ('project', ProjectCommands),
@@ -749,6 +875,7 @@ CATEGORIES = [
     ('db', DbCommands),
     ('volume', VolumeCommands),
     ('instance_type', InstanceTypeCommands),
+    ('image', ImageCommands),
     ('flavor', InstanceTypeCommands)]
 
 

nova/tests/test_cloud.py

@@ -78,10 +78,11 @@ class CloudTestCase(test.TestCase):
                                               project=self.project)
         host = self.network.get_network_host(self.context.elevated())
 
-        def fake_image_show(meh, context, id):
+        def fake_show(meh, context, id):
-            return dict(kernelId=1, ramdiskId=1)
+            return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}}
 
-        self.stubs.Set(local.LocalImageService, 'show', fake_image_show)
+        self.stubs.Set(local.LocalImageService, 'show', fake_show)
+        self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show)
 
     def tearDown(self):
         network_ref = db.project_get_network(self.context,
@@ -195,8 +196,10 @@ class CloudTestCase(test.TestCase):
     def test_describe_instances(self):
         """Makes sure describe_instances works and filters results."""
         inst1 = db.instance_create(self.context, {'reservation_id': 'a',
+                                                  'image_id': 1,
                                                   'host': 'host1'})
         inst2 = db.instance_create(self.context, {'reservation_id': 'a',
+                                                  'image_id': 1,
                                                   'host': 'host2'})
         comp1 = db.service_create(self.context, {'host': 'host1',
                                                  'availability_zone': 'zone1',
@@ -222,11 +225,9 @@ class CloudTestCase(test.TestCase):
         db.service_destroy(self.context, comp2['id'])
 
     def test_console_output(self):
-        image_id = FLAGS.default_image
-        print image_id
         instance_type = FLAGS.default_instance_type
         max_count = 1
-        kwargs = {'image_id': image_id,
+        kwargs = {'image_id': 'ami-1',
                   'instance_type': instance_type,
                   'max_count': max_count}
         rv = self.cloud.run_instances(self.context, **kwargs)
@@ -242,8 +243,7 @@ class CloudTestCase(test.TestCase):
         greenthread.sleep(0.3)
 
     def test_ajax_console(self):
-        image_id = FLAGS.default_image
+        kwargs = {'image_id': 'ami-1'}
-        kwargs = {'image_id': image_id}
         rv = self.cloud.run_instances(self.context, **kwargs)
         instance_id = rv['instancesSet'][0]['instanceId']
         greenthread.sleep(0.3)

nova/tests/test_compute.py

@@ -47,16 +47,16 @@ class ComputeTestCase(test.TestCase):
                    network_manager='nova.network.manager.FlatManager')
         self.compute = utils.import_object(FLAGS.compute_manager)
         self.compute_api = compute.API()
-
-        def fake_image_show(meh, context, id):
-            return dict(kernelId=1, ramdiskId=1)
-
-        self.stubs.Set(local.LocalImageService, 'show', fake_image_show)
         self.manager = manager.AuthManager()
         self.user = self.manager.create_user('fake', 'fake', 'fake')
         self.project = self.manager.create_project('fake', 'fake', 'fake')
         self.context = context.RequestContext('fake', 'fake', False)
 
+        def fake_show(meh, context, id):
+            return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}}
+
+        self.stubs.Set(local.LocalImageService, 'show', fake_show)
+
     def tearDown(self):
         self.manager.delete_user(self.user)
         self.manager.delete_project(self.project)
@@ -65,7 +65,7 @@ class ComputeTestCase(test.TestCase):
     def _create_instance(self):
         """Create a test instance"""
         inst = {}
-        inst['image_id'] = 'ami-test'
+        inst['image_id'] = 1
         inst['reservation_id'] = 'r-fakeres'
         inst['launch_time'] = '10'
         inst['user_id'] = self.user.id

nova/tests/test_console.py

@@ -57,7 +57,7 @@ class ConsoleTestCase(test.TestCase):
         inst = {}
         #inst['host'] = self.host
         #inst['name'] = 'instance-1234'
-        inst['image_id'] = 'ami-test'
+        inst['image_id'] = 1
         inst['reservation_id'] = 'r-fakeres'
         inst['launch_time'] = '10'
         inst['user_id'] = self.user.id

nova/tests/test_scheduler.py

@@ -155,7 +155,7 @@ class SimpleDriverTestCase(test.TestCase):
     def _create_instance(self, **kwargs):
         """Create a test instance"""
         inst = {}
-        inst['image_id'] = 'ami-test'
+        inst['image_id'] = 1
         inst['reservation_id'] = 'r-fakeres'
         inst['user_id'] = self.user.id
         inst['project_id'] = self.project.id
@@ -169,8 +169,6 @@ class SimpleDriverTestCase(test.TestCase):
     def _create_volume(self):
         """Create a test volume"""
         vol = {}
-        vol['image_id'] = 'ami-test'
-        vol['reservation_id'] = 'r-fakeres'
         vol['size'] = 1
         vol['availability_zone'] = 'test'
         return db.volume_create(self.context, vol)['id']

nova/tests/test_volume.py

@@ -99,7 +99,7 @@ class VolumeTestCase(test.TestCase):
     def test_run_attach_detach_volume(self):
         """Make sure volume can be attached and detached from instance."""
         inst = {}
-        inst['image_id'] = 'ami-test'
+        inst['image_id'] = 1
         inst['reservation_id'] = 'r-fakeres'
         inst['launch_time'] = '10'
         inst['user_id'] = 'fake'