Merge branch 'master' of github.com:angst/cc

bin/nova-import-canonical-imagestore

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+"""
+  Download images from Canonical Image Store
+"""
+
+import json
+import os
+import tempfile
+import shutil
+import subprocess
+import sys
+import urllib2
+
+from nova.objectstore import image
+from nova import flags
+from nova import utils
+
+FLAGS = flags.FLAGS
+
+api_url = 'https://imagestore.canonical.com/api/dashboard'
+
+image_cache = None
+def images():
+    global image_cache
+    if not image_cache:
+        try:
+            images = json.load(urllib2.urlopen(api_url))['images']
+            image_cache = [i for i in images if i['title'].find('amd64') > -1]
+        except Exception:
+            print 'unable to download canonical image list'
+            sys.exit(1)
+    return image_cache
+
+# FIXME(ja): add checksum/signature checks
+def download(img):
+    tempdir = tempfile.mkdtemp(prefix='cis-')
+
+    kernel_id = None
+    ramdisk_id = None
+
+    for f in img['files']:
+        if f['kind'] == 'kernel':
+            dest = os.path.join(tempdir, 'kernel')
+            subprocess.call(['curl', f['url'], '-o', dest])
+            kernel_id = image.Image.add(dest,
+                description='kernel/' + img['title'], kernel=True)
+
+    for f in img['files']:
+        if f['kind'] == 'ramdisk':
+            dest = os.path.join(tempdir, 'ramdisk')
+            subprocess.call(['curl', f['url'], '-o', dest])
+            ramdisk_id = image.Image.add(dest,
+                description='ramdisk/' + img['title'], ramdisk=True)
+
+    for f in img['files']:
+        if f['kind'] == 'image':
+            dest = os.path.join(tempdir, 'image')
+            subprocess.call(['curl', f['url'], '-o', dest])
+            ramdisk_id = image.Image.add(dest,
+                description=img['title'], kernel=kernel_id, ramdisk=ramdisk_id)
+
+    shutil.rmtree(tempdir)
+
+def main():
+    utils.default_flagfile()
+    argv = FLAGS(sys.argv)
+
+    if len(argv) == 2:
+        for img in images():
+            if argv[1] == 'all' or argv[1] == img['title']:
+                download(img)
+    else:
+        print 'usage: %s (title|all)'
+        print 'available images:'
+        for image in images():
+            print image['title']
+
+if __name__ == '__main__':
+    main()
+

bin/nova-rsapi

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright [2010] [Anso Labs, LLC]
+# 
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+# 
+#        http://www.apache.org/licenses/LICENSE-2.0
+# 
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+"""
+  WSGI daemon for the main API endpoint.  
+"""
+
+import logging
+from wsgiref import simple_server
+
+from nova import vendor
+from tornado import ioloop
+
+from nova import flags
+from nova import rpc
+from nova import server
+from nova import utils
+from nova.auth import users
+from nova.endpoint import rackspace
+
+FLAGS = flags.FLAGS
+flags.DEFINE_integer('cc_port', 8773, 'cloud controller port')
+
+def main(_argv):
+    user_manager = users.UserManager()
+    api_instance = rackspace.Api(user_manager)
+    conn = rpc.Connection.instance()
+    rpc_consumer = rpc.AdapterConsumer(connection=conn,
+                                       topic=FLAGS.cloud_topic,
+                                       proxy=api_instance)
+
+#   TODO: fire rpc response listener (without attach to tornado)
+#    io_inst = ioloop.IOLoop.instance()
+#    _injected = consumer.attach_to_tornado(io_inst)
+
+    http_server = simple_server.WSGIServer(('0.0.0.0', FLAGS.cc_port), simple_server.WSGIRequestHandler)
+    http_server.set_app(api_instance.handler)
+    logging.debug('Started HTTP server on port %i' % FLAGS.cc_port)
+    while True:
+        http_server.handle_request()
+#    io_inst.start()
+
+if __name__ == '__main__':
+    utils.default_flagfile()
+    server.serve('nova-rsapi', main)

exercise_rsapi.py

@@ -0,0 +1,33 @@
+import cloudservers
+
+class IdFake:
+    def __init__(self, id):
+        self.id = id
+
+# to get your access key:
+# from nova.auth import users
+# users.UserManger.instance().get_users()[0].access
+rscloud = cloudservers.CloudServers(
+            'admin',
+            '6cca875e-5ab3-4c60-9852-abf5c5c60cc6'
+          )
+rscloud.client.AUTH_URL = 'http://localhost:8773/v1.0'
+
+
+rv = rscloud.servers.list()
+print "SERVERS: %s" % rv
+
+if len(rv) == 0:
+    server = rscloud.servers.create(
+               "test-server",
+               IdFake("ami-tiny"),
+               IdFake("m1.tiny")
+             )
+    print "LAUNCH: %s" % server
+else:
+    server = rv[0]
+    print "Server to kill: %s" % server
+
+raw_input("press enter key to kill the server")
+
+server.delete()

nova/endpoint/cloud.py

@@ -514,6 +514,18 @@ class CloudController(object):
         # vpn image is private so it doesn't show up on lists
         if kwargs['image_id'] != FLAGS.vpn_image_id:
             image = self._get_image(context, kwargs['image_id'])
+
+        # FIXME(ja): if image is cloudpipe, this breaks
+
+        # get defaults from imagestore
+        image_id = image['imageId']
+        kernel_id = image.get('kernelId', None)
+        ramdisk_id = image.get('ramdiskId', None)
+
+        # API parameters overrides of defaults
+        kernel_id = kwargs.get('kernel_id', kernel_id)
+        ramdisk_id = kwargs.get('ramdisk_id', ramdisk_id)
+
         logging.debug("Going to run instances...")
         reservation_id = utils.generate_uid('r')
         launch_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
@@ -534,12 +546,9 @@ class CloudController(object):
             bridge_name = net['bridge_name']
         for num in range(int(kwargs['max_count'])):
             inst = self.instdir.new()
-            # TODO(ja): add ari, aki
+            inst['image_id'] = image_id
-            inst['image_id'] = kwargs['image_id']
+            inst['kernel_id'] = kernel_id
-            if 'kernel_id' in kwargs:
+            inst['ramdisk_id'] = ramdisk_id
-                inst['kernel_id'] = kwargs['kernel_id']
-            if 'ramdisk_id' in kwargs:
-                inst['ramdisk_id'] = kwargs['ramdisk_id']
             inst['user_data'] = kwargs.get('user_data', '')
             inst['instance_type'] = kwargs.get('instance_type', 'm1.small')
             inst['reservation_id'] = reservation_id

nova/endpoint/rackspace.py

@@ -0,0 +1,226 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+# Copyright [2010] [Anso Labs, LLC]
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+"""
+Rackspace API
+"""
+
+import base64
+import json
+import logging
+import multiprocessing
+import os
+import time
+
+from nova import vendor
+import tornado.web
+from twisted.internet import defer
+
+from nova import datastore
+from nova import flags
+from nova import rpc
+from nova import utils
+from nova import exception
+from nova.auth import users
+from nova.compute import model
+from nova.compute import network
+from nova.endpoint import wsgi
+from nova.endpoint import images
+from nova.volume import storage
+
+
+FLAGS = flags.FLAGS
+flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on')
+
+
+# TODO(todd): subclass Exception so we can bubble meaningful errors
+
+
+class Api(object):
+
+    def __init__(self, rpc_mechanism):
+        self.controllers = {
+            "v1.0":   RackspaceAuthenticationApi(),
+            "servers": RackspaceCloudServerApi()
+        }
+        self.rpc_mechanism = rpc_mechanism
+
+    def handler(self, environ, responder):
+        environ['nova.context'] = self.build_context(environ)
+        controller, path = wsgi.Util.route(
+                             environ['PATH_INFO'],
+                             self.controllers
+                           )
+        if not controller:
+            # TODO(todd): Exception (404)
+            raise Exception("Missing Controller")
+        rv = controller.process(path, environ)
+        if type(rv) is tuple:
+            responder(rv[0], rv[1])
+            rv = rv[2]
+        else:
+            responder("200 OK", [])
+        return rv
+
+    def build_context(self, env):
+        rv = {}
+        if env.has_key("HTTP_X_AUTH_TOKEN"):
+            rv['user'] = users.UserManager.instance().get_user_from_access_key(
+                           env['HTTP_X_AUTH_TOKEN']
+                         )
+            if rv['user']:
+                rv['project'] = users.UserManager.instance().get_project(
+                                  rv['user'].name
+                                )
+        return rv
+
+
+class RackspaceApiEndpoint(object):
+    def process(self, path, env):
+        if not self.check_authentication(env):
+            # TODO(todd): Exception (Unauthorized)
+            raise Exception("Unable to authenticate")
+
+        if len(path) == 0:
+            return self.index(env)
+
+        action = path.pop(0)
+        if hasattr(self, action):
+            method = getattr(self, action)
+            return method(path, env)
+        else:
+            # TODO(todd): Exception (404)
+            raise Exception("Missing method %s" % path[0])
+
+    def check_authentication(self, env):
+        if hasattr(self, "process_without_authentication") \
+        and getattr(self, "process_without_authentication"):
+            return True
+        if not env['nova.context']['user']:
+            return False
+        return True
+
+
+class RackspaceAuthenticationApi(RackspaceApiEndpoint):
+
+    def __init__(self):
+        self.process_without_authentication = True
+
+    # TODO(todd): make a actual session with a unique token
+    # just pass the auth key back through for now
+    def index(self, env):
+        response = '204 No Content'
+        headers = [
+            ('X-Server-Management-Url', 'http://%s' % env['HTTP_HOST']),
+            ('X-Storage-Url', 'http://%s' % env['HTTP_HOST']),
+            ('X-CDN-Managment-Url', 'http://%s' % env['HTTP_HOST']),
+            ('X-Auth-Token', env['HTTP_X_AUTH_KEY'])
+        ]
+        body = ""
+        return (response, headers, body)
+
+
+class RackspaceCloudServerApi(RackspaceApiEndpoint):
+
+    def __init__(self):
+        self.instdir = model.InstanceDirectory()
+        self.network = network.PublicNetworkController()
+
+    def index(self, env):
+        if env['REQUEST_METHOD'] == 'GET':
+            return self.detail(env)
+        elif env['REQUEST_METHOD'] == 'POST':
+            return self.launch_server(env)
+
+    def detail(self, args, env):
+        value = {
+            "servers":
+                []
+        }
+        for inst in self.instdir.all:
+            value["servers"].append(self.instance_details(inst))
+
+        return json.dumps(value)
+
+    ##
+    ##
+
+    def launch_server(self, env):
+        data = json.loads(env['wsgi.input'].read(int(env['CONTENT_LENGTH'])))
+        inst = self.build_server_instance(data, env['nova.context'])
+        self.schedule_launch_of_instance(inst)
+        return json.dumps({"server": self.instance_details(inst)})
+
+    def instance_details(self, inst):
+        return {
+            "id": inst.get("instance_id", None),
+            "imageId": inst.get("image_id", None),
+            "flavorId": inst.get("instacne_type", None),
+            "hostId": inst.get("node_name", None),
+            "status": inst.get("state", "pending"),
+            "addresses": {
+                "public": [self.network.get_public_ip_for_instance(
+                            inst.get("instance_id", None)
+                          )],
+                "private": [inst.get("private_dns_name", None)]
+            },
+
+            # implemented only by Rackspace, not AWS
+            "name": inst.get("name", "Not-Specified"),
+
+            # not supported
+            "progress": "Not-Supported",
+            "metadata": {
+                "Server Label": "Not-Supported",
+                "Image Version": "Not-Supported"
+            }
+        }
+
+    def build_server_instance(self, env, context):
+        reservation = utils.generate_uid('r')
+        ltime = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
+        inst = self.instdir.new()
+        inst['name'] = env['server']['name']
+        inst['image_id'] = env['server']['imageId']
+        inst['instance_type'] = env['server']['flavorId']
+        inst['user_id'] = context['user'].id
+        inst['project_id'] = context['project'].id
+        inst['reservation_id'] = reservation
+        inst['launch_time'] = ltime
+        inst['mac_address'] = utils.generate_mac()
+        address = network.allocate_ip(
+                    inst['user_id'],
+                    inst['project_id'],
+                    mac=inst['mac_address']
+                  )
+        inst['private_dns_name'] = str(address)
+        inst['bridge_name'] = network.BridgedNetwork.get_network_for_project(
+                                inst['user_id'],
+                                inst['project_id'],
+                                'default' # security group
+                              )['bridge_name']
+        # key_data, key_name, ami_launch_index
+        # TODO(todd): key data or root password
+        inst.save()
+        return inst
+
+    def schedule_launch_of_instance(self, inst):
+        rpc.cast(
+            FLAGS.compute_topic,
+            {
+                "method": "run_instance",
+                "args": {"instance_id": inst.instance_id}
+            }
+        )

nova/endpoint/wsgi.py

@@ -0,0 +1,23 @@
+
+'''
+Utility methods for working with WSGI servers
+'''
+
+class Util(object):
+
+    @staticmethod
+    def route(reqstr, controllers):
+        if len(reqstr) == 0:
+            return Util.select_root_controller(controllers), []
+        parts = [x for x in reqstr.split("/") if len(x) > 0]
+        if len(parts) == 0:
+            return Util.select_root_controller(controllers), []
+        return controllers[parts[0]], parts[1:]
+
+    @staticmethod
+    def select_root_controller(controllers):
+        if '' in controllers:
+            return controllers['']
+        else:
+            return None
+

nova/tests/objectstore_unittest.py

@@ -155,7 +155,7 @@ class ObjectStoreTestCase(test.BaseTestCase):
             bucket[os.path.basename(path)] = open(path, 'rb').read()
 
         # register an image
-        objectstore.image.Image.create('i-testing', 'image_bucket/1mb.manifest.xml', self.context)
+        objectstore.image.Image.register_aws_image('i-testing', 'image_bucket/1mb.manifest.xml', self.context)
 
         # verify image
         my_img = objectstore.image.Image('i-testing')