Merge trunk and resolve bin/nova-vncproxy conflict

.mailmap

@@ -47,3 +47,7 @@
 <vishvananda@gmail.com> <root@mirror.nasanebula.net>
 <vishvananda@gmail.com> <root@ubuntu>
 <vishvananda@gmail.com> <vishvananda@yahoo.com>
+<ilyaalekseyev@acm.org> <ialekseev@griddynamics.com>
+<ilyaalekseyev@acm.org> <ilya@oscloud.ru>
+<reldan@oscloud.ru> <enugaev@griddynamics.com>
+<kshileev@gmail.com> <kshileev@griddynamics.com>

Authors

@@ -22,14 +22,14 @@ David Pravec <David.Pravec@danix.org>
 Dean Troyer <dtroyer@gmail.com>
 Devin Carlen <devin.carlen@gmail.com>
 Ed Leafe <ed@leafe.com>
-Eldar Nugaev <enugaev@griddynamics.com>
+Eldar Nugaev <reldan@oscloud.ru>
 Eric Day <eday@oddments.org>
 Eric Windisch <eric@cloudscaling.com>
 Ewan Mellor <ewan.mellor@citrix.com>
 Gabe Westmaas <gabe.westmaas@rackspace.com>
 Hisaharu Ishii <ishii.hisaharu@lab.ntt.co.jp>
 Hisaki Ohara <hisaki.ohara@intel.com>
-Ilya Alekseyev <ialekseev@griddynamics.com>
+Ilya Alekseyev <ilyaalekseyev@acm.org>
 Isaku Yamahata <yamahata@valinux.co.jp>
 Jason Cannavale <jason.cannavale@rackspace.com>
 Jason Koelker <jason@koelker.net>
@@ -53,6 +53,7 @@ Kei Masumoto <masumotok@nttdata.co.jp>
 Ken Pepple <ken.pepple@gmail.com>
 Kevin Bringard <kbringard@attinteractive.com>
 Kevin L. Mitchell <kevin.mitchell@rackspace.com>
+Kirill Shileev <kshileev@gmail.com>
 Koji Iida <iida.koji@lab.ntt.co.jp>
 Lorin Hochstein <lorin@isi.edu>
 Lvov Maxim <usrleon@gmail.com>

bin/instance-usage-audit

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2011 Openstack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Cron script to generate usage notifications for instances neither created
+   nor destroyed in a given time period.
+
+   Together with the notifications generated by compute on instance
+   create/delete/resize, over that ime period, this allows an external
+   system consuming usage notification feeds to calculate instance usage
+   for each tenant.
+
+   Time periods are specified like so:
+   <number>[mdy]
+
+   1m = previous month. If the script is run April 1, it will generate usages
+        for March 1 thry March 31.
+   3m = 3 previous months.
+   90d = previous 90 days.
+   1y = previous year. If run on Jan 1, it generates usages for
+        Jan 1 thru Dec 31 of the previous year.
+"""
+
+import datetime
+import gettext
+import os
+import sys
+import time
+
+# If ../nova/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'nova', '__init__.py')):
+    sys.path.insert(0, POSSIBLE_TOPDIR)
+
+gettext.install('nova', unicode=1)
+
+
+from nova import context
+from nova import db
+from nova import exception
+from nova import flags
+from nova import log as logging
+from nova import utils
+
+from nova.notifier import api as notifier_api
+
+FLAGS = flags.FLAGS
+flags.DEFINE_string('instance_usage_audit_period', '1m',
+                    'time period to generate instance usages for.')
+
+
+def time_period(period):
+    today = datetime.date.today()
+    unit = period[-1]
+    if unit not in 'mdy':
+        raise ValueError('Time period must be m, d, or y')
+    n = int(period[:-1])
+    if unit == 'm':
+        year = today.year - (n // 12)
+        n = n % 12
+        if n >= today.month:
+            year -= 1
+            month = 12 + (today.month - n)
+        else:
+            month = today.month - n
+        begin = datetime.datetime(day=1, month=month, year=year)
+        end = datetime.datetime(day=1, month=today.month, year=today.year)
+
+    elif unit == 'y':
+        begin = datetime.datetime(day=1, month=1, year=today.year - n)
+        end = datetime.datetime(day=1, month=1, year=today.year)
+
+    elif unit == 'd':
+        b = today - datetime.timedelta(days=n)
+        begin = datetime.datetime(day=b.day, month=b.month, year=b.year)
+        end = datetime.datetime(day=today.day,
+                               month=today.month,
+                               year=today.year)
+
+    return (begin, end)
+
+if __name__ == '__main__':
+    utils.default_flagfile()
+    flags.FLAGS(sys.argv)
+    logging.setup()
+    begin, end = time_period(FLAGS.instance_usage_audit_period)
+    print "Creating usages for %s until %s" % (str(begin), str(end))
+    instances = db.instance_get_active_by_window(context.get_admin_context(),
+                                                 begin,
+                                                 end)
+    print "%s instances" % len(instances)
+    for instance_ref in instances:
+        usage_info = utils.usage_from_instance(instance_ref,
+                              audit_period_begining=str(begin),
+                              audit_period_ending=str(end))
+        notifier_api.notify('compute.%s' % FLAGS.host,
+                            'compute.instance.exists',
+                            notifier_api.INFO,
+                            usage_info)

bin/nova-ajax-console-proxy

@@ -137,8 +137,9 @@ if __name__ == '__main__':
     utils.default_flagfile()
     FLAGS(sys.argv)
     logging.setup()
-    server = wsgi.Server()
+    acp_port = FLAGS.ajax_console_proxy_port
     acp = AjaxConsoleProxy()
     acp.register_listeners()
-    server.start(acp, FLAGS.ajax_console_proxy_port, host='0.0.0.0')
+    server = wsgi.Server("AJAX Console Proxy", acp, port=acp_port)
+    server.start()
     server.wait()

bin/nova-api

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# pylint: disable=C0103
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2010 United States Government as represented by the
@@ -18,44 +17,40 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
-"""Starter script for Nova API."""
+"""Starter script for Nova API.
+
+Starts both the EC2 and OpenStack APIs in separate processes.
+
+"""
 
-import gettext
 import os
 import sys
 
-# If ../nova/__init__.py exists, add ../ to Python search path, so that
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(
-# it will override what happens to be installed in /usr/(local/)lib/python...
+        sys.argv[0]), os.pardir, os.pardir))
-possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+if os.path.exists(os.path.join(possible_topdir, "nova", "__init__.py")):
-                                   os.pardir,
-                                   os.pardir))
-if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
     sys.path.insert(0, possible_topdir)
 
-gettext.install('nova', unicode=1)
+import nova.service
-
+import nova.utils
-from nova import flags
-from nova import log as logging
-from nova import service
-from nova import utils
-from nova import version
-from nova import wsgi
 
 
-LOG = logging.getLogger('nova.api')
+def main():
+    """Launch EC2 and OSAPI services."""
+    nova.utils.Bootstrapper.bootstrap_binary(sys.argv)
+
+    ec2 = nova.service.WSGIService("ec2")
+    osapi = nova.service.WSGIService("osapi")
+
+    launcher = nova.service.Launcher()
+    launcher.launch_service(ec2)
+    launcher.launch_service(osapi)
+
+    try:
+        launcher.wait()
+    except KeyboardInterrupt:
+        launcher.stop()
 
-FLAGS = flags.FLAGS
 
 if __name__ == '__main__':
-    utils.default_flagfile()
+    sys.exit(main())
-    FLAGS(sys.argv)
-    logging.setup()
-    LOG.audit(_("Starting nova-api node (version %s)"),
-              version.version_string_with_vcs())
-    LOG.debug(_("Full set of FLAGS:"))
-    for flag in FLAGS:
-        flag_get = FLAGS.get(flag, None)
-        LOG.debug("%(flag)s : %(flag_get)s" % locals())
-
-    service = service.serve_wsgi(service.ApiService)
-    service.wait()

bin/nova-direct-api

@@ -93,6 +93,9 @@ if __name__ == '__main__':
     with_req = direct.PostParamsMiddleware(with_json)
     with_auth = direct.DelegatedAuthMiddleware(with_req)
 
-    server = wsgi.Server()
+    server = wsgi.Server("Direct API",
-    server.start(with_auth, FLAGS.direct_port, host=FLAGS.direct_host)
+                         with_auth,
+                         host=FLAGS.direct_host,
+                         port=FLAGS.direct_port)
+    server.start()
     server.wait()

bin/nova-manage

@@ -56,11 +56,11 @@
 import gettext
 import glob
 import json
+import netaddr
 import os
 import sys
 import time
 
-import IPy
 
 # If ../nova/__init__.py exists, add ../ to Python search path, so that
 # it will override what happens to be installed in /usr/(local/)lib/python...
@@ -257,6 +257,11 @@ class RoleCommands(object):
         """adds role to user
         if project is specified, adds project specific role
         arguments: user, role [project]"""
+        if project:
+            projobj = self.manager.get_project(project)
+            if not projobj.has_member(user):
+                print "%s not a member of %s" % (user, project)
+                return
         self.manager.add_role(user, role, project)
 
     def has(self, user, role, project=None):
@@ -513,7 +518,7 @@ class FloatingIpCommands(object):
     def create(self, host, range):
         """Creates floating ips for host by range
         arguments: host ip_range"""
-        for address in IPy.IP(range):
+        for address in netaddr.IPNetwork(range):
             db.floating_ip_create(context.get_admin_context(),
                                   {'address': str(address),
                                    'host': host})
@@ -521,7 +526,7 @@ class FloatingIpCommands(object):
     def delete(self, ip_range):
         """Deletes floating ips by range
         arguments: range"""
-        for address in IPy.IP(ip_range):
+        for address in netaddr.IPNetwork(ip_range):
             db.floating_ip_destroy(context.get_admin_context(),
                                    str(address))
 
@@ -612,7 +617,7 @@ class VmCommands(object):
         :param host: show all instance on specified host.
         :param instance: show specificed instance.
         """
-        print "%-10s %-15s %-10s %-10s %-19s %-12s %-12s %-12s" \
+        print "%-10s %-15s %-10s %-10s %-26s %-9s %-9s %-9s" \
               "  %-10s %-10s %-10s %-5s" % (
             _('instance'),
             _('node'),
@@ -634,14 +639,14 @@ class VmCommands(object):
                            context.get_admin_context(), host)
 
         for instance in instances:
-            print "%-10s %-15s %-10s %-10s %-19s %-12s %-12s %-12s" \
+            print "%-10s %-15s %-10s %-10s %-26s %-9s %-9s %-9s" \
                   "  %-10s %-10s %-10s %-5d" % (
                 instance['hostname'],
                 instance['host'],
-                instance['instance_type'],
+                instance['instance_type'].name,
                 instance['state_description'],
                 instance['launched_at'],
-                instance['image_id'],
+                instance['image_ref'],
                 instance['kernel_id'],
                 instance['ramdisk_id'],
                 instance['project_id'],
@@ -873,7 +878,7 @@ class InstanceTypeCommands(object):
         try:
             instance_types.create(name, memory, vcpus, local_gb,
                                   flavorid, swap, rxtx_quota, rxtx_cap)
-        except exception.InvalidInput:
+        except exception.InvalidInput, e:
             print "Must supply valid parameters to create instance_type"
             print e
             sys.exit(1)

bin/nova-objectstore

@@ -50,6 +50,9 @@ if __name__ == '__main__':
     FLAGS(sys.argv)
     logging.setup()
     router = s3server.S3Application(FLAGS.buckets_path)
-    server = wsgi.Server()
+    server = wsgi.Server("S3 Objectstore",
-    server.start(router, FLAGS.s3_port, host=FLAGS.s3_host)
+                         router,
+                         port=FLAGS.s3_port,
+                         host=FLAGS.s3_host)
+    server.start()
     server.wait()

bin/nova-vncproxy

@@ -106,7 +106,11 @@ if __name__ == "__main__":
 
     service.serve()
 
-    server = wsgi.Server()
+    server = wsgi.Server("VNC Proxy",
-    server.start(with_auth, FLAGS.vncproxy_port, host=FLAGS.vncproxy_host)
+                         with_auth,
+                         host=FLAGS.vncproxy_host,
+                         port=FLAGS.vncproxy_port)
+    server.start()
     server.start_tcp(handle_flash_socket_policy, 843, host=FLAGS.vncproxy_host)
+
     server.wait()

nova/__init__.py

@@ -30,3 +30,8 @@
 .. moduleauthor:: Manish Singh <yosh@gimp.org>
 .. moduleauthor:: Andy Smith <andy@anarkystic.com>
 """
+
+import gettext
+
+
+gettext.install("nova", unicode=1)

nova/auth/fakeldap.py

@@ -100,6 +100,11 @@ class OBJECT_CLASS_VIOLATION(Exception):  # pylint: disable=C0103
     pass
 
 
+class SERVER_DOWN(Exception):  # pylint: disable=C0103
+    """Duplicate exception class from real LDAP module."""
+    pass
+
+
 def initialize(_uri):
     """Opens a fake connection with an LDAP server."""
     return FakeLDAP()
@@ -202,25 +207,38 @@ def _to_json(unencoded):
     return json.dumps(list(unencoded))
 
 
+server_fail = False
+
+
 class FakeLDAP(object):
     """Fake LDAP connection."""
 
     def simple_bind_s(self, dn, password):
         """This method is ignored, but provided for compatibility."""
+        if server_fail:
+            raise SERVER_DOWN
         pass
 
     def unbind_s(self):
         """This method is ignored, but provided for compatibility."""
+        if server_fail:
+            raise SERVER_DOWN
         pass
 
     def add_s(self, dn, attr):
         """Add an object with the specified attributes at dn."""
+        if server_fail:
+            raise SERVER_DOWN
+
         key = "%s%s" % (self.__prefix, dn)
         value_dict = dict([(k, _to_json(v)) for k, v in attr])
         Store.instance().hmset(key, value_dict)
 
     def delete_s(self, dn):
         """Remove the ldap object at specified dn."""
+        if server_fail:
+            raise SERVER_DOWN
+
         Store.instance().delete("%s%s" % (self.__prefix, dn))
 
     def modify_s(self, dn, attrs):
@@ -232,6 +250,9 @@ class FakeLDAP(object):
             ([MOD_ADD | MOD_DELETE | MOD_REPACE], attribute, value)
 
         """
+        if server_fail:
+            raise SERVER_DOWN
+
         store = Store.instance()
         key = "%s%s" % (self.__prefix, dn)
 
@@ -255,6 +276,9 @@ class FakeLDAP(object):
         fields -- fields to return. Returns all fields if not specified
 
         """
+        if server_fail:
+            raise SERVER_DOWN
+
         if scope != SCOPE_BASE and scope != SCOPE_SUBTREE:
             raise NotImplementedError(str(scope))
         store = Store.instance()

nova/auth/ldapdriver.py

@@ -101,6 +101,41 @@ def sanitize(fn):
     return _wrapped
 
 
+class LDAPWrapper(object):
+    def __init__(self, ldap, url, user, password):
+        self.ldap = ldap
+        self.url = url
+        self.user = user
+        self.password = password
+        self.conn = None
+
+    def __wrap_reconnect(f):
+        def inner(self, *args, **kwargs):
+            if self.conn is None:
+                self.connect()
+                return f(self.conn)(*args, **kwargs)
+            else:
+                try:
+                    return f(self.conn)(*args, **kwargs)
+                except self.ldap.SERVER_DOWN:
+                    self.connect()
+                    return f(self.conn)(*args, **kwargs)
+        return inner
+
+    def connect(self):
+        try:
+            self.conn = self.ldap.initialize(self.url)
+            self.conn.simple_bind_s(self.user, self.password)
+        except self.ldap.SERVER_DOWN:
+            self.conn = None
+            raise
+
+    search_s = __wrap_reconnect(lambda conn: conn.search_s)
+    add_s = __wrap_reconnect(lambda conn: conn.add_s)
+    delete_s = __wrap_reconnect(lambda conn: conn.delete_s)
+    modify_s = __wrap_reconnect(lambda conn: conn.modify_s)
+
+
 class LdapDriver(object):
     """Ldap Auth driver
 
@@ -124,8 +159,8 @@ class LdapDriver(object):
             LdapDriver.project_objectclass = 'novaProject'
         self.__cache = None
         if LdapDriver.conn is None:
-            LdapDriver.conn = self.ldap.initialize(FLAGS.ldap_url)
+            LdapDriver.conn = LDAPWrapper(self.ldap, FLAGS.ldap_url,
-            LdapDriver.conn.simple_bind_s(FLAGS.ldap_user_dn,
+                                          FLAGS.ldap_user_dn,
                                           FLAGS.ldap_password)
         if LdapDriver.mc is None:
             LdapDriver.mc = memcache.Client(FLAGS.memcached_servers, debug=0)

nova/exception.py

@@ -361,6 +361,10 @@ class NoFixedIpsFoundForInstance(NotFound):
 
 
 class FloatingIpNotFound(NotFound):
+    message = _("Floating ip %(floating_ip)s not found")
+
+
+class FloatingIpNotFoundForFixedAddress(NotFound):
     message = _("Floating ip not found for fixed address %(fixed_ip)s.")
 
 
@@ -504,6 +508,11 @@ class InstanceMetadataNotFound(NotFound):
                 "key %(metadata_key)s.")
 
 
+class InstanceTypeExtraSpecsNotFound(NotFound):
+    message = _("Instance Type %(instance_type_id)s has no extra specs with "
+                "key %(extra_specs_key)s.")
+
+
 class LDAPObjectNotFound(NotFound):
     message = _("LDAP object could not be found")
 
@@ -549,6 +558,14 @@ class GlobalRoleNotAllowed(NotAllowed):
     message = _("Unable to use global role %(role_id)s")
 
 
+class ImageRotationNotAllowed(NovaException):
+    message = _("Rotation is not allowed for snapshots")
+
+
+class RotationRequiredForBackup(NovaException):
+    message = _("Rotation param is required for backup image_type")
+
+
 #TODO(bcwaldon): EOL this exception!
 class Duplicate(NovaException):
     pass
@@ -589,3 +606,11 @@ class MigrationError(NovaException):
 
 class MalformedRequestBody(NovaException):
     message = _("Malformed message body: %(reason)s")
+
+
+class PasteConfigNotFound(NotFound):
+    message = _("Could not find paste config at %(path)s")
+
+
+class PasteAppNotFound(NotFound):
+    message = _("Could not load paste app '%(name)s' from %(path)s")

nova/log.py

@@ -314,3 +314,14 @@ logging.setLoggerClass(NovaLogger)
 def audit(msg, *args, **kwargs):
     """Shortcut for logging to root log with sevrity 'AUDIT'."""
     logging.root.log(AUDIT, msg, *args, **kwargs)
+
+
+class WritableLogger(object):
+    """A thin wrapper that responds to `write` and logs."""
+
+    def __init__(self, logger, level=logging.INFO):
+        self.logger = logger
+        self.level = level
+
+    def write(self, msg):
+        self.logger.log(self.level, msg)

nova/notifier/test_notifier.py

@@ -0,0 +1,28 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+
+from nova import flags
+from nova import log as logging
+
+FLAGS = flags.FLAGS
+
+NOTIFICATIONS = []
+
+
+def notify(message):
+    """Test notifier, stores notifications in memory for unittests."""
+    NOTIFICATIONS.append(message)

nova/rpc.py

@@ -275,6 +275,11 @@ class FanoutAdapterConsumer(AdapterConsumer):
         unique = uuid.uuid4().hex
         self.queue = '%s_fanout_%s' % (topic, unique)
         self.durable = False
+        # Fanout creates unique queue names, so we should auto-remove
+        # them when done, so they're not left around on restart.
+        # Also, we're the only one that should be consuming.  exclusive
+        # implies auto_delete, so we'll just set that..
+        self.exclusive = True
         LOG.info(_('Created "%(exchange)s" fanout exchange '
                    'with "%(key)s" routing key'),
                  dict(exchange=self.exchange, key=self.routing_key))

nova/scheduler/api.py

@@ -162,32 +162,53 @@ def child_zone_helper(zone_list, func):
                     _wrap_method(_process, func), zone_list)]
 
 
-def _issue_novaclient_command(nova, zone, collection, method_name, item_id):
+def _issue_novaclient_command(nova, zone, collection,
+        method_name, *args, **kwargs):
     """Use novaclient to issue command to a single child zone.
-       One of these will be run in parallel for each child zone."""
+       One of these will be run in parallel for each child zone.
+    """
     manager = getattr(nova, collection)
-    result = None
+
-    try:
+    # NOTE(comstud): This is not ideal, but we have to do this based on
+    # how novaclient is implemented right now.
+    # 'find' is special cased as novaclient requires kwargs for it to
+    # filter on a 'get_all'.
+    # Every other method first needs to do a 'get' on the first argument
+    # passed, which should be a UUID.  If it's 'get' itself that we want,
+    # we just return the result.  Otherwise, we next call the real method
+    # that's wanted... passing other arguments that may or may not exist.
+    if method_name in ['find', 'findall']:
         try:
-            result = manager.get(int(item_id))
+            return getattr(manager, method_name)(**kwargs)
-        except ValueError, e:
+        except novaclient.NotFound:
-            result = manager.find(name=item_id)
+            url = zone.api_url
+            LOG.debug(_("%(collection)s.%(method_name)s didn't find "
+                    "anything matching '%(kwargs)s' on '%(url)s'" %
+                    locals()))
+            return None
+
+    args = list(args)
+    # pop off the UUID to look up
+    item = args.pop(0)
+    try:
+        result = manager.get(item)
     except novaclient.NotFound:
         url = zone.api_url
-        LOG.debug(_("%(collection)s '%(item_id)s' not found on '%(url)s'" %
+        LOG.debug(_("%(collection)s '%(item)s' not found on '%(url)s'" %
                                                 locals()))
         return None
 
-    if method_name.lower() not in ['get', 'find']:
+    if method_name.lower() != 'get':
-        result = getattr(result, method_name)()
+        # if we're doing something other than 'get', call it passing args.
+        result = getattr(result, method_name)(*args, **kwargs)
     return result
 
 
-def wrap_novaclient_function(f, collection, method_name, item_id):
+def wrap_novaclient_function(f, collection, method_name, *args, **kwargs):
-    """Appends collection, method_name and item_id to the incoming
+    """Appends collection, method_name and arguments to the incoming
     (nova, zone) call from child_zone_helper."""
     def inner(nova, zone):
-        return f(nova, zone, collection, method_name, item_id)
+        return f(nova, zone, collection, method_name, *args, **kwargs)
 
     return inner
 
@@ -220,7 +241,7 @@ class reroute_compute(object):
            the wrapped method. (This ensures that zone-local code can
            continue to use integer IDs).
 
-        4. If the item was not found, we delgate the call to a child zone
+        4. If the item was not found, we delegate the call to a child zone
            using the UUID.
     """
     def __init__(self, method_name):

nova/scheduler/host_filter.py

@@ -93,6 +93,26 @@ class InstanceTypeFilter(HostFilter):
         """Use instance_type to filter hosts."""
         return (self._full_name(), instance_type)
 
+    def _satisfies_extra_specs(self, capabilities, instance_type):
+        """Check that the capabilities provided by the compute service
+        satisfy the extra specs associated with the instance type"""
+
+        if 'extra_specs' not in instance_type:
+            return True
+
+        # Note(lorinh): For now, we are just checking exact matching on the
+        # values. Later on, we  want to handle numerical
+        # values so we can represent things like number of GPU cards
+
+        try:
+            for key, value in instance_type['extra_specs'].iteritems():
+                if capabilities[key] != value:
+                    return False
+        except KeyError:
+            return False
+
+        return True
+
     def filter_hosts(self, zone_manager, query):
         """Return a list of hosts that can create instance_type."""
         instance_type = query
@@ -103,7 +123,11 @@ class InstanceTypeFilter(HostFilter):
             disk_bytes = capabilities['disk_available']
             spec_ram = instance_type['memory_mb']
             spec_disk = instance_type['local_gb']
-            if host_ram_mb >= spec_ram and disk_bytes >= spec_disk:
+            extra_specs = instance_type['extra_specs']
+
+            if host_ram_mb >= spec_ram and \
+               disk_bytes >= spec_disk and \
+               self._satisfies_extra_specs(capabilities, instance_type):
                 selected_hosts.append((host, capabilities))
         return selected_hosts
 
@@ -305,8 +329,9 @@ class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler):
                     'instance_type': <InstanceType dict>}
     """
 
-    def filter_hosts(self, num, request_spec):
+    def filter_hosts(self, topic, request_spec, hosts=None):
         """Filter the full host list (from the ZoneManager)"""
+
         filter_name = request_spec.get('filter', None)
         host_filter = choose_host_filter(filter_name)
 
@@ -317,8 +342,9 @@ class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler):
         name, query = host_filter.instance_type_to_filter(instance_type)
         return host_filter.filter_hosts(self.zone_manager, query)
 
-    def weigh_hosts(self, num, request_spec, hosts):
+    def weigh_hosts(self, topic, request_spec, hosts):
         """Derived classes must override this method and return
         a lists of hosts in [{weight, hostname}] format.
         """
-        return [dict(weight=1, hostname=host) for host, caps in hosts]
+        return [dict(weight=1, hostname=hostname, capabilities=caps)
+                for hostname, caps in hosts]

nova/scheduler/least_cost.py

@@ -48,25 +48,43 @@ def noop_cost_fn(host):
     return 1
 
 
-flags.DEFINE_integer('fill_first_cost_fn_weight', 1,
+flags.DEFINE_integer('compute_fill_first_cost_fn_weight', 1,
                      'How much weight to give the fill-first cost function')
 
 
-def fill_first_cost_fn(host):
+def compute_fill_first_cost_fn(host):
     """Prefer hosts that have less ram available, filter_hosts will exclude
     hosts that don't have enough ram"""
     hostname, caps = host
-    free_mem = caps['compute']['host_memory_free']
+    free_mem = caps['host_memory_free']
     return free_mem
 
 
 class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler):
-    def get_cost_fns(self):
+    def __init__(self, *args, **kwargs):
+        self.cost_fns_cache = {}
+        super(LeastCostScheduler, self).__init__(*args, **kwargs)
+
+    def get_cost_fns(self, topic):
         """Returns a list of tuples containing weights and cost functions to
         use for weighing hosts
         """
+
+        if topic in self.cost_fns_cache:
+            return self.cost_fns_cache[topic]
+
         cost_fns = []
         for cost_fn_str in FLAGS.least_cost_scheduler_cost_functions:
+            if '.' in cost_fn_str:
+                short_name = cost_fn_str.split('.')[-1]
+            else:
+                short_name = cost_fn_str
+                cost_fn_str = "%s.%s.%s" % (
+                        __name__, self.__class__.__name__, short_name)
+
+            if not (short_name.startswith('%s_' % topic) or
+                    short_name.startswith('noop')):
+                continue
 
             try:
                 # NOTE(sirp): import_class is somewhat misnamed since it can
@@ -84,23 +102,23 @@ class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler):
 
             cost_fns.append((weight, cost_fn))
 
+        self.cost_fns_cache[topic] = cost_fns
         return cost_fns
 
-    def weigh_hosts(self, num, request_spec, hosts):
+    def weigh_hosts(self, topic, request_spec, hosts):
         """Returns a list of dictionaries of form:
-            [ {weight: weight, hostname: hostname} ]"""
+           [ {weight: weight, hostname: hostname, capabilities: capabs} ]
+        """
 
-        # FIXME(sirp): weigh_hosts should handle more than just instances
+        cost_fns = self.get_cost_fns(topic)
-        hostnames = [hostname for hostname, caps in hosts]
-
-        cost_fns = self.get_cost_fns()
         costs = weighted_sum(domain=hosts, weighted_fns=cost_fns)
 
         weighted = []
         weight_log = []
-        for cost, hostname in zip(costs, hostnames):
+        for cost, (hostname, caps) in zip(costs, hosts):
             weight_log.append("%s: %s" % (hostname, "%.2f" % cost))
-            weight_dict = dict(weight=cost, hostname=hostname)
+            weight_dict = dict(weight=cost, hostname=hostname,
+                    capabilities=caps)
             weighted.append(weight_dict)
 
         LOG.debug(_("Weighted Costs => %s") % weight_log)
@@ -127,7 +145,8 @@ def weighted_sum(domain, weighted_fns, normalize=True):
     weighted_fns - list of weights and functions like:
         [(weight, objective-functions)]
 
-    Returns an unsorted of scores. To pair with hosts do: zip(scores, hosts)
+    Returns an unsorted list of scores. To pair with hosts do:
+        zip(scores, hosts)
     """
     # Table of form:
     #   { domain1: [score1, score2, ..., scoreM]
@@ -150,7 +169,6 @@ def weighted_sum(domain, weighted_fns, normalize=True):
     domain_scores = []
     for idx in sorted(score_table):
         elem_score = sum(score_table[idx])
-        elem = domain[idx]
         domain_scores.append(elem_score)
 
     return domain_scores

nova/scheduler/zone_aware_scheduler.py

@@ -180,18 +180,22 @@ class ZoneAwareScheduler(driver.Scheduler):
                                     request_spec, kwargs)
             return None
 
+        num_instances = request_spec.get('num_instances', 1)
+        LOG.debug(_("Attempting to build %(num_instances)d instance(s)") %
+                locals())
+
         # Create build plan and provision ...
         build_plan = self.select(context, request_spec)
         if not build_plan:
             raise driver.NoValidHost(_('No hosts were available'))
 
-        for num in xrange(request_spec['num_instances']):
+        for num in xrange(num_instances):
             if not build_plan:
                 break
 
-            item = build_plan.pop(0)
+            build_plan_item = build_plan.pop(0)
-            self._provision_resource(context, item, instance_id, request_spec,
+            self._provision_resource(context, build_plan_item, instance_id,
-                                    kwargs)
+                                     request_spec, kwargs)
 
         # Returning None short-circuits the routing to Compute (since
         # we've already done it here)
@@ -224,18 +228,36 @@ class ZoneAwareScheduler(driver.Scheduler):
             raise NotImplemented(_("Zone Aware Scheduler only understands "
                                    "Compute nodes (for now)"))
 
-        #TODO(sandy): how to infer this from OS API params?
+        num_instances = request_spec.get('num_instances', 1)
-        num_instances = 1
+        instance_type = request_spec['instance_type']
 
-        # Filter local hosts based on requirements ...
+        weighted = []
-        host_list = self.filter_hosts(num_instances, request_spec)
+        host_list = None
 
-        # TODO(sirp): weigh_hosts should also be a function of 'topic' or
+        for i in xrange(num_instances):
-        # resources, so that we can apply different objective functions to it
+            # Filter local hosts based on requirements ...
+            #
+            # The first pass through here will pass 'None' as the
+            # host_list.. which tells the filter to build the full
+            # list of hosts.
+            # On a 2nd pass, the filter can modify the host_list with
+            # any updates it needs to make based on resources that
+            # may have been consumed from a previous build..
+            host_list = self.filter_hosts(topic, request_spec, host_list)
+            if not host_list:
+                LOG.warn(_("Filter returned no hosts after processing "
+                        "%(i)d of %(num_instances)d instances") % locals())
+                break
 
-        # then weigh the selected hosts.
+            # then weigh the selected hosts.
-        # weighted = [{weight=weight, name=hostname}, ...]
+            # weighted = [{weight=weight, hostname=hostname,
-        weighted = self.weigh_hosts(num_instances, request_spec, host_list)
+            #              capabilities=capabs}, ...]
+            weights = self.weigh_hosts(topic, request_spec, host_list)
+            weights.sort(key=operator.itemgetter('weight'))
+            best_weight = weights[0]
+            weighted.append(best_weight)
+            self.consume_resources(topic, best_weight['capabilities'],
+                    instance_type)
 
         # Next, tack on the best weights from the child zones ...
         json_spec = json.dumps(request_spec)
@@ -254,18 +276,65 @@ class ZoneAwareScheduler(driver.Scheduler):
         weighted.sort(key=operator.itemgetter('weight'))
         return weighted
 
-    def filter_hosts(self, num, request_spec):
+    def compute_filter(self, hostname, capabilities, request_spec):
-        """Derived classes must override this method and return
+        """Return whether or not we can schedule to this compute node.
-           a list of hosts in [(hostname, capability_dict)] format.
+        Derived classes should override this and return True if the host
+        is acceptable for scheduling.
         """
-        # NOTE(sirp): The default logic is the equivalent to AllHostsFilter
+        instance_type = request_spec['instance_type']
-        service_states = self.zone_manager.service_states
+        requested_mem = instance_type['memory_mb'] * 1024 * 1024
-        return [(host, services)
+        return capabilities['host_memory_free'] >= requested_mem
-                for host, services in service_states.iteritems()]
 
-    def weigh_hosts(self, num, request_spec, hosts):
+    def filter_hosts(self, topic, request_spec, host_list=None):
+        """Return a list of hosts which are acceptable for scheduling.
+        Return value should be a list of (hostname, capability_dict)s.
+        Derived classes may override this, but may find the
+        '<topic>_filter' function more appropriate.
+        """
+
+        def _default_filter(self, hostname, capabilities, request_spec):
+            """Default filter function if there's no <topic>_filter"""
+            # NOTE(sirp): The default logic is the equivalent to
+            # AllHostsFilter
+            return True
+
+        filter_func = getattr(self, '%s_filter' % topic, _default_filter)
+
+        if host_list is None:
+            first_run = True
+            host_list = self.zone_manager.service_states.iteritems()
+        else:
+            first_run = False
+
+        filtered_hosts = []
+        for host, services in host_list:
+            if first_run:
+                if topic not in services:
+                    continue
+                services = services[topic]
+            if filter_func(host, services, request_spec):
+                filtered_hosts.append((host, services))
+        return filtered_hosts
+
+    def weigh_hosts(self, topic, request_spec, hosts):
         """Derived classes may override this to provide more sophisticated
         scheduling objectives
         """
         # NOTE(sirp): The default logic is the same as the NoopCostFunction
-        return [dict(weight=1, hostname=host) for host, caps in hosts]
+        return [dict(weight=1, hostname=hostname, capabilities=capabilities)
+                for hostname, capabilities in hosts]
+
+    def compute_consume(self, capabilities, instance_type):
+        """Consume compute resources for selected host"""
+
+        requested_mem = max(instance_type['memory_mb'], 0) * 1024 * 1024
+        capabilities['host_memory_free'] -= requested_mem
+
+    def consume_resources(self, topic, capabilities, instance_type):
+        """Consume resources for a specific host.  'host' is a tuple
+        of the hostname and the services"""
+
+        consume_func = getattr(self, '%s_consume' % topic, None)
+        if not consume_func:
+            return
+        consume_func(capabilities, instance_type)

nova/service.py

@@ -19,10 +19,12 @@
 
 """Generic Node baseclass for all workers that run on hosts."""
 
-import greenlet
 import inspect
+import multiprocessing
 import os
 
+import greenlet
+
 from eventlet import greenthread
 
 from nova import context
@@ -36,6 +38,8 @@ from nova import version
 from nova import wsgi
 
 
+LOG = logging.getLogger('nova.service')
+
 FLAGS = flags.FLAGS
 flags.DEFINE_integer('report_interval', 10,
                      'seconds between nodes reporting state to datastore',
@@ -53,6 +57,63 @@ flags.DEFINE_string('api_paste_config', "api-paste.ini",
                     'File name for the paste.deploy config for nova-api')
 
 
+class Launcher(object):
+    """Launch one or more services and wait for them to complete."""
+
+    def __init__(self):
+        """Initialize the service launcher.
+
+        :returns: None
+
+        """
+        self._services = []
+
+    @staticmethod
+    def run_service(service):
+        """Start and wait for a service to finish.
+
+        :param service: Service to run and wait for.
+        :returns: None
+
+        """
+        service.start()
+        try:
+            service.wait()
+        except KeyboardInterrupt:
+            service.stop()
+
+    def launch_service(self, service):
+        """Load and start the given service.
+
+        :param service: The service you would like to start.
+        :returns: None
+
+        """
+        process = multiprocessing.Process(target=self.run_service,
+                                          args=(service,))
+        process.start()
+        self._services.append(process)
+
+    def stop(self):
+        """Stop all services which are currently running.
+
+        :returns: None
+
+        """
+        for service in self._services:
+            if service.is_alive():
+                service.terminate()
+
+    def wait(self):
+        """Waits until all services have been stopped, and then returns.
+
+        :returns: None
+
+        """
+        for service in self._services:
+            service.join()
+
+
 class Service(object):
     """Base class for workers that run on hosts."""
 
@@ -232,45 +293,54 @@ class Service(object):
                 logging.exception(_('model server went away'))
 
 
-class WsgiService(object):
+class WSGIService(object):
-    """Base class for WSGI based services.
+    """Provides ability to launch API from a 'paste' configuration."""
 
-    For each api you define, you must also define these flags:
+    def __init__(self, name, loader=None):
-    :<api>_listen: The address on which to listen
+        """Initialize, but do not start the WSGI service.
-    :<api>_listen_port: The port on which to listen
 
-    """
+        :param name: The name of the WSGI service given to the loader.
+        :param loader: Loads the WSGI application using the given name.
+        :returns: None
 
-    def __init__(self, conf, apis):
+        """
-        self.conf = conf
+        self.name = name
-        self.apis = apis
+        self.loader = loader or wsgi.Loader()
-        self.wsgi_app = None
+        self.app = self.loader.load_app(name)
+        self.host = getattr(FLAGS, '%s_listen' % name, "0.0.0.0")
+        self.port = getattr(FLAGS, '%s_listen_port' % name, 0)
+        self.server = wsgi.Server(name,
+                                  self.app,
+                                  host=self.host,
+                                  port=self.port)
 
     def start(self):
-        self.wsgi_app = _run_wsgi(self.conf, self.apis)
+        """Start serving this service using loaded configuration.
+
+        Also, retrieve updated port number in case '0' was passed in, which
+        indicates a random port should be used.
+
+        :returns: None
+
+        """
+        self.server.start()
+        self.port = self.server.port
+
+    def stop(self):
+        """Stop serving this API.
+
+        :returns: None
+
+        """
+        self.server.stop()
 
     def wait(self):
-        self.wsgi_app.wait()
+        """Wait for the service to stop serving this API.
 
-    def get_socket_info(self, api_name):
+        :returns: None
-        """Returns the (host, port) that an API was started on."""
-        return self.wsgi_app.socket_info[api_name]
 
-
+        """
-class ApiService(WsgiService):
+        self.server.wait()
-    """Class for our nova-api service."""
-
-    @classmethod
-    def create(cls, conf=None):
-        if not conf:
-            conf = wsgi.paste_config_file(FLAGS.api_paste_config)
-            if not conf:
-                message = (_('No paste configuration found for: %s'),
-                           FLAGS.api_paste_config)
-                raise exception.Error(message)
-        api_endpoints = ['ec2', 'osapi']
-        service = cls(conf, api_endpoints)
-        return service
 
 
 def serve(*services):
@@ -302,48 +372,3 @@ def serve(*services):
 def wait():
     while True:
         greenthread.sleep(5)
-
-
-def serve_wsgi(cls, conf=None):
-    try:
-        service = cls.create(conf)
-    except Exception:
-        logging.exception('in WsgiService.create()')
-        raise
-    finally:
-        # After we've loaded up all our dynamic bits, check
-        # whether we should print help
-        flags.DEFINE_flag(flags.HelpFlag())
-        flags.DEFINE_flag(flags.HelpshortFlag())
-        flags.DEFINE_flag(flags.HelpXMLFlag())
-        FLAGS.ParseNewFlags()
-
-    service.start()
-
-    return service
-
-
-def _run_wsgi(paste_config_file, apis):
-    logging.debug(_('Using paste.deploy config at: %s'), paste_config_file)
-    apps = []
-    for api in apis:
-        config = wsgi.load_paste_configuration(paste_config_file, api)
-        if config is None:
-            logging.debug(_('No paste configuration for app: %s'), api)
-            continue
-        logging.debug(_('App Config: %(api)s\n%(config)r') % locals())
-        logging.info(_('Running %s API'), api)
-        app = wsgi.load_paste_app(paste_config_file, api)
-        apps.append((app,
-                     getattr(FLAGS, '%s_listen_port' % api),
-                     getattr(FLAGS, '%s_listen' % api),
-                     api))
-    if len(apps) == 0:
-        logging.error(_('No known API applications configured in %s.'),
-                      paste_config_file)
-        return
-
-    server = wsgi.Server()
-    for app in apps:
-        server.start(*app)
-    return server

nova/test.py

@@ -38,7 +38,6 @@ from nova import flags
 from nova import rpc
 from nova import utils
 from nova import service
-from nova import wsgi
 from nova.virt import fake
 
 
@@ -81,7 +80,6 @@ class TestCase(unittest.TestCase):
         self.injected = []
         self._services = []
         self._monkey_patch_attach()
-        self._monkey_patch_wsgi()
         self._original_flags = FLAGS.FlagValuesDict()
         rpc.ConnectionPool = rpc.Pool(max_size=FLAGS.rpc_conn_pool_size)
 
@@ -107,7 +105,6 @@ class TestCase(unittest.TestCase):
 
             # Reset our monkey-patches
             rpc.Consumer.attach_to_eventlet = self.original_attach
-            wsgi.Server.start = self.original_start
 
             # Stop any timers
             for x in self.injected:
@@ -163,26 +160,6 @@ class TestCase(unittest.TestCase):
         _wrapped.func_name = self.original_attach.func_name
         rpc.Consumer.attach_to_eventlet = _wrapped
 
-    def _monkey_patch_wsgi(self):
-        """Allow us to kill servers spawned by wsgi.Server."""
-        self.original_start = wsgi.Server.start
-
-        @functools.wraps(self.original_start)
-        def _wrapped_start(inner_self, *args, **kwargs):
-            original_spawn_n = inner_self.pool.spawn_n
-
-            @functools.wraps(original_spawn_n)
-            def _wrapped_spawn_n(*args, **kwargs):
-                rv = greenthread.spawn(*args, **kwargs)
-                self._services.append(rv)
-
-            inner_self.pool.spawn_n = _wrapped_spawn_n
-            self.original_start(inner_self, *args, **kwargs)
-            inner_self.pool.spawn_n = original_spawn_n
-
-        _wrapped_start.func_name = self.original_start.func_name
-        wsgi.Server.start = _wrapped_start
-
     # Useful assertions
     def assertDictMatch(self, d1, d2, approx_equal=False, tolerance=0.001):
         """Assert two dicts are equivalent.

nova/tests/__init__.py

@@ -50,7 +50,7 @@ def setup():
 
     testdb = os.path.join(FLAGS.state_path, FLAGS.sqlite_db)
     if os.path.exists(testdb):
-        os.unlink(testdb)
+        return
     migration.db_sync()
     ctxt = context.get_admin_context()
     network_manager.VlanManager().create_networks(ctxt,

nova/tests/scheduler/test_host_filter.py

@@ -67,7 +67,18 @@ class HostFilterTestCase(test.TestCase):
                 flavorid=1,
                 swap=500,
                 rxtx_quota=30000,
-                rxtx_cap=200)
+                rxtx_cap=200,
+                extra_specs={})
+        self.gpu_instance_type = dict(name='tiny.gpu',
+                memory_mb=50,
+                vcpus=10,
+                local_gb=500,
+                flavorid=2,
+                swap=500,
+                rxtx_quota=30000,
+                rxtx_cap=200,
+                extra_specs={'xpu_arch': 'fermi',
+                             'xpu_info': 'Tesla 2050'})
 
         self.zone_manager = FakeZoneManager()
         states = {}
@@ -75,6 +86,18 @@ class HostFilterTestCase(test.TestCase):
             states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)}
         self.zone_manager.service_states = states
 
+        # Add some extra capabilities to some hosts
+        host07 = self.zone_manager.service_states['host07']['compute']
+        host07['xpu_arch'] = 'fermi'
+        host07['xpu_info'] = 'Tesla 2050'
+
+        host08 = self.zone_manager.service_states['host08']['compute']
+        host08['xpu_arch'] = 'radeon'
+
+        host09 = self.zone_manager.service_states['host09']['compute']
+        host09['xpu_arch'] = 'fermi'
+        host09['xpu_info'] = 'Tesla 2150'
+
     def tearDown(self):
         FLAGS.default_host_filter = self.old_flag
 
@@ -116,6 +139,17 @@ class HostFilterTestCase(test.TestCase):
         self.assertEquals('host05', just_hosts[0])
         self.assertEquals('host10', just_hosts[5])
 
+    def test_instance_type_filter_extra_specs(self):
+        hf = host_filter.InstanceTypeFilter()
+        # filter all hosts that can support 50 ram and 500 disk
+        name, cooked = hf.instance_type_to_filter(self.gpu_instance_type)
+        self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter',
+                          name)
+        hosts = hf.filter_hosts(self.zone_manager, cooked)
+        self.assertEquals(1, len(hosts))
+        just_hosts = [host for host, caps in hosts]
+        self.assertEquals('host07', just_hosts[0])
+
     def test_json_filter(self):
         hf = host_filter.JsonFilter()
         # filter all hosts that can support 50 ram and 500 disk

nova/tests/scheduler/test_least_cost_scheduler.py

@@ -122,15 +122,16 @@ class LeastCostSchedulerTestCase(test.TestCase):
                     for hostname, caps in hosts]
         self.assertWeights(expected, num, request_spec, hosts)
 
-    def test_fill_first_cost_fn(self):
+    def test_compute_fill_first_cost_fn(self):
         FLAGS.least_cost_scheduler_cost_functions = [
-            'nova.scheduler.least_cost.fill_first_cost_fn',
+            'nova.scheduler.least_cost.compute_fill_first_cost_fn',
         ]
-        FLAGS.fill_first_cost_fn_weight = 1
+        FLAGS.compute_fill_first_cost_fn_weight = 1
 
         num = 1
-        request_spec = {}
+        instance_type = {'memory_mb': 1024}
-        hosts = self.sched.filter_hosts(num, request_spec)
+        request_spec = {'instance_type': instance_type}
+        hosts = self.sched.filter_hosts('compute', request_spec, None)
 
         expected = []
         for idx, (hostname, caps) in enumerate(hosts):

nova/tests/scheduler/test_scheduler.py

@@ -1074,7 +1074,7 @@ class DynamicNovaClientTest(test.TestCase):
 
         self.assertEquals(api._issue_novaclient_command(
                     FakeNovaClient(FakeServerCollection()),
-                    zone, "servers", "find", "name").b, 22)
+                    zone, "servers", "find", name="test").b, 22)
 
         self.assertEquals(api._issue_novaclient_command(
                     FakeNovaClient(FakeServerCollection()),
@@ -1088,7 +1088,7 @@ class DynamicNovaClientTest(test.TestCase):
 
         self.assertEquals(api._issue_novaclient_command(
                     FakeNovaClient(FakeEmptyServerCollection()),
-                    zone, "servers", "find", "name"), None)
+                    zone, "servers", "find", name="test"), None)
 
         self.assertEquals(api._issue_novaclient_command(
                     FakeNovaClient(FakeEmptyServerCollection()),

nova/tests/scheduler/test_zone_aware_scheduler.py

@@ -55,29 +55,21 @@ def fake_zone_manager_service_states(num_hosts):
 
 
 class FakeZoneAwareScheduler(zone_aware_scheduler.ZoneAwareScheduler):
-    def filter_hosts(self, num, specs):
+    # No need to stub anything at the moment
-        # NOTE(sirp): this is returning [(hostname, services)]
+    pass
-        return self.zone_manager.service_states.items()
-
-    def weigh_hosts(self, num, specs, hosts):
-        fake_weight = 99
-        weighted = []
-        for hostname, caps in hosts:
-            weighted.append(dict(weight=fake_weight, name=hostname))
-        return weighted
 
 
 class FakeZoneManager(zone_manager.ZoneManager):
     def __init__(self):
         self.service_states = {
             'host1': {
-                'compute': {'ram': 1000},
+                'compute': {'host_memory_free': 1073741824},
             },
             'host2': {
-                'compute': {'ram': 2000},
+                'compute': {'host_memory_free': 2147483648},
             },
             'host3': {
-                'compute': {'ram': 3000},
+                'compute': {'host_memory_free': 3221225472},
             },
         }
 
@@ -154,8 +146,8 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
 
     def test_zone_aware_scheduler(self):
         """
-        Create a nested set of FakeZones, ensure that a select call returns the
+        Create a nested set of FakeZones, try to build multiple instances
-        appropriate build plan.
+        and ensure that a select call returns the appropriate build plan.
         """
         sched = FakeZoneAwareScheduler()
         self.stubs.Set(sched, '_call_zone_method', fake_call_zone_method)
@@ -164,13 +156,17 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
         sched.set_zone_manager(zm)
 
         fake_context = {}
-        build_plan = sched.select(fake_context, {})
+        build_plan = sched.select(fake_context,
+                {'instance_type': {'memory_mb': 512},
+                    'num_instances': 4})
 
-        self.assertEqual(15, len(build_plan))
+        # 4 from local zones, 12 from remotes
+        self.assertEqual(16, len(build_plan))
 
-        hostnames = [plan_item['name']
+        hostnames = [plan_item['hostname']
-                     for plan_item in build_plan if 'name' in plan_item]
+                     for plan_item in build_plan if 'hostname' in plan_item]
-        self.assertEqual(3, len(hostnames))
+        # 4 local hosts
+        self.assertEqual(4, len(hostnames))
 
     def test_empty_zone_aware_scheduler(self):
         """
@@ -185,8 +181,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
         fake_context = {}
         self.assertRaises(driver.NoValidHost, sched.schedule_run_instance,
                           fake_context, 1,
-                          dict(host_filter=None,
+                          dict(host_filter=None, instance_type={}))
-                               request_spec={'instance_type': {}}))
 
     def test_schedule_do_not_schedule_with_hint(self):
         """

nova/tests/test_adminapi.py

@@ -0,0 +1,111 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from eventlet import greenthread
+
+from nova import context
+from nova import db
+from nova import flags
+from nova import log as logging
+from nova import rpc
+from nova import test
+from nova import utils
+from nova.auth import manager
+from nova.api.ec2 import admin
+from nova.image import fake
+
+
+FLAGS = flags.FLAGS
+LOG = logging.getLogger('nova.tests.adminapi')
+
+
+class AdminApiTestCase(test.TestCase):
+    def setUp(self):
+        super(AdminApiTestCase, self).setUp()
+        self.flags(connection_type='fake')
+
+        self.conn = rpc.Connection.instance()
+
+        # set up our cloud
+        self.api = admin.AdminController()
+
+        # set up services
+        self.compute = self.start_service('compute')
+        self.scheduter = self.start_service('scheduler')
+        self.network = self.start_service('network')
+        self.volume = self.start_service('volume')
+        self.image_service = utils.import_object(FLAGS.image_service)
+
+        self.manager = manager.AuthManager()
+        self.user = self.manager.create_user('admin', 'admin', 'admin', True)
+        self.project = self.manager.create_project('proj', 'admin', 'proj')
+        self.context = context.RequestContext(user=self.user,
+                                              project=self.project)
+        host = self.network.get_network_host(self.context.elevated())
+
+        def fake_show(meh, context, id):
+            return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1,
+                    'type': 'machine', 'image_state': 'available'}}
+
+        self.stubs.Set(fake._FakeImageService, 'show', fake_show)
+        self.stubs.Set(fake._FakeImageService, 'show_by_name', fake_show)
+
+        # NOTE(vish): set up a manual wait so rpc.cast has a chance to finish
+        rpc_cast = rpc.cast
+
+        def finish_cast(*args, **kwargs):
+            rpc_cast(*args, **kwargs)
+            greenthread.sleep(0.2)
+
+        self.stubs.Set(rpc, 'cast', finish_cast)
+
+    def tearDown(self):
+        network_ref = db.project_get_network(self.context,
+                                             self.project.id)
+        db.network_disassociate(self.context, network_ref['id'])
+        self.manager.delete_project(self.project)
+        self.manager.delete_user(self.user)
+        super(AdminApiTestCase, self).tearDown()
+
+    def test_block_external_ips(self):
+        """Make sure provider firewall rules are created."""
+        result = self.api.block_external_addresses(self.context, '1.1.1.1/32')
+        self.api.remove_external_address_block(self.context, '1.1.1.1/32')
+        self.assertEqual('OK', result['status'])
+        self.assertEqual('Added 3 rules', result['message'])
+
+    def test_list_blocked_ips(self):
+        """Make sure we can see the external blocks that exist."""
+        self.api.block_external_addresses(self.context, '1.1.1.2/32')
+        result = self.api.describe_external_address_blocks(self.context)
+        num = len(db.provider_fw_rule_get_all(self.context))
+        self.api.remove_external_address_block(self.context, '1.1.1.2/32')
+        # we only list IP, not tcp/udp/icmp rules
+        self.assertEqual(num / 3, len(result['externalIpBlockInfo']))
+
+    def test_remove_ip_block(self):
+        """Remove ip blocks."""
+        result = self.api.block_external_addresses(self.context, '1.1.1.3/32')
+        self.assertEqual('OK', result['status'])
+        num0 = len(db.provider_fw_rule_get_all(self.context))
+        result = self.api.remove_external_address_block(self.context,
+                                                        '1.1.1.3/32')
+        self.assertEqual('OK', result['status'])
+        self.assertEqual('Deleted 3 rules', result['message'])
+        num1 = len(db.provider_fw_rule_get_all(self.context))
+        self.assert_(num1 < num0)

nova/tests/test_auth.py

@@ -25,6 +25,7 @@ from nova import log as logging
 from nova import test
 from nova.auth import manager
 from nova.api.ec2 import cloud
+from nova.auth import fakeldap
 
 FLAGS = flags.FLAGS
 LOG = logging.getLogger('nova.tests.auth_unittest')
@@ -369,6 +370,15 @@ class _AuthManagerBaseTestCase(test.TestCase):
 class AuthManagerLdapTestCase(_AuthManagerBaseTestCase):
     auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver'
 
+    def test_reconnect_on_server_failure(self):
+        self.manager.get_users()
+        fakeldap.server_fail = True
+        try:
+            self.assertRaises(fakeldap.SERVER_DOWN, self.manager.get_users)
+        finally:
+            fakeldap.server_fail = False
+        self.manager.get_users()
+
 
 class AuthManagerDbTestCase(_AuthManagerBaseTestCase):
     auth_driver = 'nova.auth.dbdriver.DbDriver'

nova/tests/test_compute.py

@@ -37,6 +37,7 @@ from nova import log as logging
 from nova import rpc
 from nova import test
 from nova import utils
+from nova.notifier import test_notifier
 
 LOG = logging.getLogger('nova.tests.compute')
 FLAGS = flags.FLAGS
@@ -62,6 +63,7 @@ class ComputeTestCase(test.TestCase):
         super(ComputeTestCase, self).setUp()
         self.flags(connection_type='fake',
                    stub_network=True,
+                   notification_driver='nova.notifier.test_notifier',
                    network_manager='nova.network.manager.FlatManager')
         self.compute = utils.import_object(FLAGS.compute_manager)
         self.compute_api = compute.API()
@@ -69,6 +71,7 @@ class ComputeTestCase(test.TestCase):
         self.user = self.manager.create_user('fake', 'fake', 'fake')
         self.project = self.manager.create_project('fake', 'fake', 'fake')
         self.context = context.RequestContext('fake', 'fake', False)
+        test_notifier.NOTIFICATIONS = []
 
         def fake_show(meh, context, id):
             return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}}
@@ -327,6 +330,50 @@ class ComputeTestCase(test.TestCase):
         self.assert_(console)
         self.compute.terminate_instance(self.context, instance_id)
 
+    def test_run_instance_usage_notification(self):
+        """Ensure run instance generates apropriate usage notification"""
+        instance_id = self._create_instance()
+        self.compute.run_instance(self.context, instance_id)
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        msg = test_notifier.NOTIFICATIONS[0]
+        self.assertEquals(msg['priority'], 'INFO')
+        self.assertEquals(msg['event_type'], 'compute.instance.create')
+        payload = msg['payload']
+        self.assertEquals(payload['tenant_id'], self.project.id)
+        self.assertEquals(payload['user_id'], self.user.id)
+        self.assertEquals(payload['instance_id'], instance_id)
+        self.assertEquals(payload['instance_type'], 'm1.tiny')
+        type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
+        self.assertEquals(str(payload['instance_type_id']), str(type_id))
+        self.assertTrue('display_name' in payload)
+        self.assertTrue('created_at' in payload)
+        self.assertTrue('launched_at' in payload)
+        self.assertEquals(payload['image_ref'], '1')
+        self.compute.terminate_instance(self.context, instance_id)
+
+    def test_terminate_usage_notification(self):
+        """Ensure terminate_instance generates apropriate usage notification"""
+        instance_id = self._create_instance()
+        self.compute.run_instance(self.context, instance_id)
+        test_notifier.NOTIFICATIONS = []
+        self.compute.terminate_instance(self.context, instance_id)
+
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        msg = test_notifier.NOTIFICATIONS[0]
+        self.assertEquals(msg['priority'], 'INFO')
+        self.assertEquals(msg['event_type'], 'compute.instance.delete')
+        payload = msg['payload']
+        self.assertEquals(payload['tenant_id'], self.project.id)
+        self.assertEquals(payload['user_id'], self.user.id)
+        self.assertEquals(payload['instance_id'], instance_id)
+        self.assertEquals(payload['instance_type'], 'm1.tiny')
+        type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
+        self.assertEquals(str(payload['instance_type_id']), str(type_id))
+        self.assertTrue('display_name' in payload)
+        self.assertTrue('created_at' in payload)
+        self.assertTrue('launched_at' in payload)
+        self.assertEquals(payload['image_ref'], '1')
+
     def test_run_instance_existing(self):
         """Ensure failure when running an instance that already exists"""
         instance_id = self._create_instance()
@@ -378,6 +425,36 @@ class ComputeTestCase(test.TestCase):
 
         self.compute.terminate_instance(self.context, instance_id)
 
+    def test_resize_instance_notification(self):
+        """Ensure notifications on instance migrate/resize"""
+        instance_id = self._create_instance()
+        context = self.context.elevated()
+
+        self.compute.run_instance(self.context, instance_id)
+        test_notifier.NOTIFICATIONS = []
+
+        db.instance_update(self.context, instance_id, {'host': 'foo'})
+        self.compute.prep_resize(context, instance_id, 1)
+        migration_ref = db.migration_get_by_instance_and_status(context,
+                instance_id, 'pre-migrating')
+
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        msg = test_notifier.NOTIFICATIONS[0]
+        self.assertEquals(msg['priority'], 'INFO')
+        self.assertEquals(msg['event_type'], 'compute.instance.resize.prep')
+        payload = msg['payload']
+        self.assertEquals(payload['tenant_id'], self.project.id)
+        self.assertEquals(payload['user_id'], self.user.id)
+        self.assertEquals(payload['instance_id'], instance_id)
+        self.assertEquals(payload['instance_type'], 'm1.tiny')
+        type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
+        self.assertEquals(str(payload['instance_type_id']), str(type_id))
+        self.assertTrue('display_name' in payload)
+        self.assertTrue('created_at' in payload)
+        self.assertTrue('launched_at' in payload)
+        self.assertEquals(payload['image_ref'], '1')
+        self.compute.terminate_instance(context, instance_id)
+
     def test_resize_instance(self):
         """Ensure instance can be migrated/resized"""
         instance_id = self._create_instance()

nova/tests/test_flat_network.py

@@ -18,7 +18,7 @@
 """
 Unit Tests for flat network code
 """
-import IPy
+import netaddr
 import os
 import unittest
 
@@ -45,8 +45,8 @@ class FlatNetworkTestCase(base.NetworkTestCase):
 
         self.context._project = self.projects[0]
         self.context.project_id = self.projects[0].id
-        pubnet = IPy.IP(flags.FLAGS.floating_range)
+        pubnet = netaddr.IPRange(flags.FLAGS.floating_range)
-        address = str(pubnet[0])
+        address = str(list(pubnet)[0])
         try:
             db.floating_ip_get_by_address(context.get_admin_context(), address)
         except exception.NotFound:

nova/tests/test_host_filter.py

@@ -67,7 +67,8 @@ class HostFilterTestCase(test.TestCase):
                 flavorid=1,
                 swap=500,
                 rxtx_quota=30000,
-                rxtx_cap=200)
+                rxtx_cap=200,
+                extra_specs={})
 
         self.zone_manager = FakeZoneManager()
         states = {}

nova/tests/test_libvirt.py

@@ -799,7 +799,9 @@ class IptablesFirewallTestCase(test.TestCase):
         self.network = utils.import_object(FLAGS.network_manager)
 
         class FakeLibvirtConnection(object):
-            pass
+            def nwfilterDefineXML(*args, **kwargs):
+                """setup_basic_rules in nwfilter calls this."""
+                pass
         self.fake_libvirt_connection = FakeLibvirtConnection()
         self.fw = firewall.IptablesFirewallDriver(
                       get_connection=lambda: self.fake_libvirt_connection)
@@ -1035,7 +1037,6 @@ class IptablesFirewallTestCase(test.TestCase):
                                fakefilter.filterDefineXMLMock
         self.fw.nwfilter._conn.nwfilterLookupByName =\
                                fakefilter.nwfilterLookupByName
-
         instance_ref = self._create_instance_ref()
         inst_id = instance_ref['id']
         instance = db.instance_get(self.context, inst_id)
@@ -1057,6 +1058,70 @@ class IptablesFirewallTestCase(test.TestCase):
 
         db.instance_destroy(admin_ctxt, instance_ref['id'])
 
+    def test_provider_firewall_rules(self):
+        # setup basic instance data
+        instance_ref = self._create_instance_ref()
+        nw_info = _create_network_info(1)
+        ip = '10.11.12.13'
+        network_ref = db.project_get_network(self.context, 'fake')
+        admin_ctxt = context.get_admin_context()
+        fixed_ip = {'address': ip, 'network_id': network_ref['id']}
+        db.fixed_ip_create(admin_ctxt, fixed_ip)
+        db.fixed_ip_update(admin_ctxt, ip, {'allocated': True,
+                                            'instance_id': instance_ref['id']})
+        # FRAGILE: peeks at how the firewall names chains
+        chain_name = 'inst-%s' % instance_ref['id']
+
+        # create a firewall via setup_basic_filtering like libvirt_conn.spawn
+        # should have a chain with 0 rules
+        self.fw.setup_basic_filtering(instance_ref, network_info=nw_info)
+        self.assertTrue('provider' in self.fw.iptables.ipv4['filter'].chains)
+        rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules
+                      if rule.chain == 'provider']
+        self.assertEqual(0, len(rules))
+
+        # add a rule and send the update message, check for 1 rule
+        provider_fw0 = db.provider_fw_rule_create(admin_ctxt,
+                                                  {'protocol': 'tcp',
+                                                   'cidr': '10.99.99.99/32',
+                                                   'from_port': 1,
+                                                   'to_port': 65535})
+        self.fw.refresh_provider_fw_rules()
+        rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules
+                      if rule.chain == 'provider']
+        self.assertEqual(1, len(rules))
+
+        # Add another, refresh, and make sure number of rules goes to two
+        provider_fw1 = db.provider_fw_rule_create(admin_ctxt,
+                                                  {'protocol': 'udp',
+                                                   'cidr': '10.99.99.99/32',
+                                                   'from_port': 1,
+                                                   'to_port': 65535})
+        self.fw.refresh_provider_fw_rules()
+        rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules
+                      if rule.chain == 'provider']
+        self.assertEqual(2, len(rules))
+
+        # create the instance filter and make sure it has a jump rule
+        self.fw.prepare_instance_filter(instance_ref, network_info=nw_info)
+        self.fw.apply_instance_filter(instance_ref)
+        inst_rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules
+                           if rule.chain == chain_name]
+        jump_rules = [rule for rule in inst_rules if '-j' in rule.rule]
+        provjump_rules = []
+        # IptablesTable doesn't make rules unique internally
+        for rule in jump_rules:
+            if 'provider' in rule.rule and rule not in provjump_rules:
+                provjump_rules.append(rule)
+        self.assertEqual(1, len(provjump_rules))
+
+        # remove a rule from the db, cast to compute to refresh rule
+        db.provider_fw_rule_destroy(admin_ctxt, provider_fw1['id'])
+        self.fw.refresh_provider_fw_rules()
+        rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules
+                      if rule.chain == 'provider']
+        self.assertEqual(1, len(rules))
+
 
 class NWFilterTestCase(test.TestCase):
     def setUp(self):

nova/tests/test_network.py

@@ -18,7 +18,7 @@
 """
 Unit Tests for network code
 """
-import IPy
+import netaddr
 import os
 
 from nova import test
@@ -164,3 +164,33 @@ class IptablesManagerTestCase(test.TestCase):
             self.assertTrue('-A %s -j run_tests.py-%s' \
                             % (chain, chain) in new_lines,
                             "Built-in chain %s not wrapped" % (chain,))
+
+    def test_will_empty_chain(self):
+        self.manager.ipv4['filter'].add_chain('test-chain')
+        self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP')
+        old_count = len(self.manager.ipv4['filter'].rules)
+        self.manager.ipv4['filter'].empty_chain('test-chain')
+        self.assertEqual(old_count - 1, len(self.manager.ipv4['filter'].rules))
+
+    def test_will_empty_unwrapped_chain(self):
+        self.manager.ipv4['filter'].add_chain('test-chain', wrap=False)
+        self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP',
+                                             wrap=False)
+        old_count = len(self.manager.ipv4['filter'].rules)
+        self.manager.ipv4['filter'].empty_chain('test-chain', wrap=False)
+        self.assertEqual(old_count - 1, len(self.manager.ipv4['filter'].rules))
+
+    def test_will_not_empty_wrapped_when_unwrapped(self):
+        self.manager.ipv4['filter'].add_chain('test-chain')
+        self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP')
+        old_count = len(self.manager.ipv4['filter'].rules)
+        self.manager.ipv4['filter'].empty_chain('test-chain', wrap=False)
+        self.assertEqual(old_count, len(self.manager.ipv4['filter'].rules))
+
+    def test_will_not_empty_unwrapped_when_wrapped(self):
+        self.manager.ipv4['filter'].add_chain('test-chain', wrap=False)
+        self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP',
+                                             wrap=False)
+        old_count = len(self.manager.ipv4['filter'].rules)
+        self.manager.ipv4['filter'].empty_chain('test-chain')
+        self.assertEqual(old_count, len(self.manager.ipv4['filter'].rules))

nova/tests/test_vlan_network.py

@@ -18,7 +18,7 @@
 """
 Unit Tests for vlan network code
 """
-import IPy
+import netaddr
 import os
 
 from nova import context
@@ -44,8 +44,8 @@ class VlanNetworkTestCase(base.NetworkTestCase):
         # TODO(vish): better way of adding floating ips
         self.context._project = self.projects[0]
         self.context.project_id = self.projects[0].id
-        pubnet = IPy.IP(flags.FLAGS.floating_range)
+        pubnet = netaddr.IPNetwork(flags.FLAGS.floating_range)
-        address = str(pubnet[0])
+        address = str(list(pubnet)[0])
         try:
             db.floating_ip_get_by_address(context.get_admin_context(), address)
         except exception.NotFound:

nova/utils.py

@@ -46,6 +46,7 @@ from eventlet.green import subprocess
 from nova import exception
 from nova import flags
 from nova import log as logging
+from nova import version
 
 
 LOG = logging.getLogger("nova.utils")
@@ -226,8 +227,10 @@ def novadir():
     return os.path.abspath(nova.__file__).split('nova/__init__.pyc')[0]
 
 
-def default_flagfile(filename='nova.conf'):
+def default_flagfile(filename='nova.conf', args=None):
-    for arg in sys.argv:
+    if args is None:
+        args = sys.argv
+    for arg in args:
         if arg.find('flagfile') != -1:
             break
     else:
@@ -239,8 +242,8 @@ def default_flagfile(filename='nova.conf'):
             filename = "./nova.conf"
             if not os.path.exists(filename):
                 filename = '/etc/nova/nova.conf'
-        flagfile = ['--flagfile=%s' % filename]
+        flagfile = '--flagfile=%s' % filename
-        sys.argv = sys.argv[:1] + flagfile + sys.argv[1:]
+        args.insert(1, flagfile)
 
 
 def debug(arg):
@@ -279,6 +282,22 @@ EASIER_PASSWORD_SYMBOLS = ('23456789'  # Removed: 0, 1
                            'ABCDEFGHJKLMNPQRSTUVWXYZ')  # Removed: I, O
 
 
+def usage_from_instance(instance_ref, **kw):
+    usage_info = dict(
+          tenant_id=instance_ref['project_id'],
+          user_id=instance_ref['user_id'],
+          instance_id=instance_ref['id'],
+          instance_type=instance_ref['instance_type']['name'],
+          instance_type_id=instance_ref['instance_type_id'],
+          display_name=instance_ref['display_name'],
+          created_at=str(instance_ref['created_at']),
+          launched_at=str(instance_ref['launched_at']) \
+                      if instance_ref['launched_at'] else '',
+          image_ref=instance_ref['image_ref'])
+    usage_info.update(kw)
+    return usage_info
+
+
 def generate_password(length=20, symbols=DEFAULT_PASSWORD_SYMBOLS):
     """Generate a random password from the supplied symbols.
 
@@ -526,6 +545,16 @@ def loads(s):
     return json.loads(s)
 
 
+try:
+    import anyjson
+except ImportError:
+    pass
+else:
+    anyjson._modules.append(("nova.utils", "dumps", TypeError,
+                                           "loads", ValueError))
+    anyjson.force_implementation("nova.utils")
+
+
 _semaphores = {}
 
 
@@ -741,3 +770,50 @@ def is_uuid_like(val):
     if not isinstance(val, basestring):
         return False
     return (len(val) == 36) and (val.count('-') == 4)
+
+
+def bool_from_str(val):
+    """Convert a string representation of a bool into a bool value"""
+
+    if not val:
+        return False
+    try:
+        return True if int(val) else False
+    except ValueError:
+        return val.lower() == 'true'
+
+
+class Bootstrapper(object):
+    """Provides environment bootstrapping capabilities for entry points."""
+
+    @staticmethod
+    def bootstrap_binary(argv):
+        """Initialize the Nova environment using command line arguments."""
+        Bootstrapper.setup_flags(argv)
+        Bootstrapper.setup_logging()
+        Bootstrapper.log_flags()
+
+    @staticmethod
+    def setup_logging():
+        """Initialize logging and log a message indicating the Nova version."""
+        logging.setup()
+        logging.audit(_("Nova Version (%s)") %
+                        version.version_string_with_vcs())
+
+    @staticmethod
+    def setup_flags(input_flags):
+        """Initialize flags, load flag file, and print help if needed."""
+        default_flagfile(args=input_flags)
+        FLAGS(input_flags or [])
+        flags.DEFINE_flag(flags.HelpFlag())
+        flags.DEFINE_flag(flags.HelpshortFlag())
+        flags.DEFINE_flag(flags.HelpXMLFlag())
+        FLAGS.ParseNewFlags()
+
+    @staticmethod
+    def log_flags():
+        """Log the list of all active flags being used."""
+        logging.audit(_("Currently active flags:"))
+        for key in FLAGS:
+            value = FLAGS.get(key, None)
+            logging.audit(_("%(key)s : %(value)s" % locals()))

nova/wsgi.py

@@ -21,16 +21,16 @@
 
 import os
 import sys
+
 from xml.dom import minidom
 
 import eventlet
 import eventlet.wsgi
-eventlet.patcher.monkey_patch(all=False, socket=True, time=True)
+import greenlet
-import routes
 import routes.middleware
-import webob
 import webob.dec
 import webob.exc
+
 from paste import deploy
 
 from nova import exception
@@ -39,36 +39,73 @@ from nova import log as logging
 from nova import utils
 
 
+eventlet.patcher.monkey_patch(socket=True, time=True)
+
+
 FLAGS = flags.FLAGS
 LOG = logging.getLogger('nova.wsgi')
 
 
-class WritableLogger(object):
-    """A thin wrapper that responds to `write` and logs."""
-
-    def __init__(self, logger, level=logging.DEBUG):
-        self.logger = logger
-        self.level = level
-
-    def write(self, msg):
-        self.logger.log(self.level, msg)
-
-
 class Server(object):
-    """Server class to manage multiple WSGI sockets and applications."""
+    """Server class to manage a WSGI server, serving a WSGI application."""
 
-    def __init__(self, threads=1000):
+    default_pool_size = 1000
-        self.pool = eventlet.GreenPool(threads)
-        self.socket_info = {}
 
-    def start(self, application, port, host='0.0.0.0', key=None, backlog=128):
+    def __init__(self, name, app, host=None, port=None, pool_size=None):
-        """Run a WSGI server with the given application."""
+        """Initialize, but do not start, a WSGI server.
-        arg0 = sys.argv[0]
+
-        logging.audit(_('Starting %(arg0)s on %(host)s:%(port)s') % locals())
+        :param name: Pretty name for logging.
-        socket = eventlet.listen((host, port), backlog=backlog)
+        :param app: The WSGI application to serve.
-        self.pool.spawn_n(self._run, application, socket)
+        :param host: IP address to serve the application.
-        if key:
+        :param port: Port number to server the application.
-            self.socket_info[key] = socket.getsockname()
+        :param pool_size: Maximum number of eventlets to spawn concurrently.
+        :returns: None
+
+        """
+        self.name = name
+        self.app = app
+        self.host = host or "0.0.0.0"
+        self.port = port or 0
+        self._server = None
+        self._socket = None
+        self._pool = eventlet.GreenPool(pool_size or self.default_pool_size)
+        self._logger = logging.getLogger("eventlet.wsgi.server")
+        self._wsgi_logger = logging.WritableLogger(self._logger)
+
+    def _start(self):
+        """Run the blocking eventlet WSGI server.
+
+        :returns: None
+
+        """
+        eventlet.wsgi.server(self._socket,
+                             self.app,
+                             custom_pool=self._pool,
+                             log=self._wsgi_logger)
+
+    def start(self, backlog=128):
+        """Start serving a WSGI application.
+
+        :param backlog: Maximum number of queued connections.
+        :returns: None
+
+        """
+        self._socket = eventlet.listen((self.host, self.port), backlog=backlog)
+        self._server = eventlet.spawn(self._start)
+        (self.host, self.port) = self._socket.getsockname()
+        LOG.info(_("Started %(name)s on %(host)s:%(port)s") % self.__dict__)
+
+    def stop(self):
+        """Stop this server.
+
+        This is not a very nice action, as currently the method by which a
+        server is stopped is by killing it's eventlet.
+
+        :returns: None
+
+        """
+        LOG.info(_("Stopping WSGI server."))
+        self._server.kill()
 
     def start_tcp(self, listener, port, host='0.0.0.0', key=None, backlog=128):
         """Run a raw TCP server with the given application."""
@@ -80,17 +117,17 @@ class Server(object):
             self.socket_info[key] = socket.getsockname()
 
     def wait(self):
-        """Wait until all servers have completed running."""
+        """Block, until the server has stopped.
-        try:
-            self.pool.waitall()
-        except KeyboardInterrupt:
-            pass
 
-    def _run(self, application, socket):
+        Waits on the server's eventlet to finish, then returns.
-        """Start a WSGI server in a new green thread."""
+
-        logger = logging.getLogger('eventlet.wsgi.server')
+        :returns: None
-        eventlet.wsgi.server(socket, application, custom_pool=self.pool,
+
-                             log=WritableLogger(logger))
+        """
+        try:
+            self._server.wait()
+        except greenlet.GreenletExit:
+            LOG.info(_("WSGI server has stopped."))
 
     def _run_tcp(self, listener, socket):
         """Start a raw TCP server in a new green thread."""
@@ -327,55 +364,51 @@ class Router(object):
         return app
 
 
-def paste_config_file(basename):
+class Loader(object):
-    """Find the best location in the system for a paste config file.
+    """Used to load WSGI applications from paste configurations."""
 
-    Search Order
+    def __init__(self, config_path=None):
-    ------------
+        """Initialize the loader, and attempt to find the config.
 
-    The search for a paste config file honors `FLAGS.state_path`, which in a
+        :param config_path: Full or relative path to the paste config.
-    version checked out from bzr will be the `nova` directory in the top level
+        :returns: None
-    of the checkout, and in an installation for a package for your distribution
-    will likely point to someplace like /etc/nova.
 
-    This method tries to load places likely to be used in development or
+        """
-    experimentation before falling back to the system-wide configuration
+        config_path = config_path or FLAGS.api_paste_config
-    in `/etc/nova/`.
+        self.config_path = self._find_config(config_path)
 
-    * Current working directory
+    def _find_config(self, config_path):
-    * the `etc` directory under state_path, because when working on a checkout
+        """Find the paste configuration file using the given hint.
-      from bzr this will point to the default
-    * top level of FLAGS.state_path, for distributions
-    * /etc/nova, which may not be diffrerent from state_path on your distro
 
-    """
+        :param config_path: Full or relative path to the paste config.
-    configfiles = [basename,
+        :returns: Full path of the paste config, if it exists.
-                   os.path.join(FLAGS.state_path, 'etc', 'nova', basename),
+        :raises: `nova.exception.PasteConfigNotFound`
-                   os.path.join(FLAGS.state_path, 'etc', basename),
-                   os.path.join(FLAGS.state_path, basename),
-                   '/etc/nova/%s' % basename]
-    for configfile in configfiles:
-        if os.path.exists(configfile):
-            return configfile
 
+        """
+        possible_locations = [
+            config_path,
+            os.path.join(FLAGS.state_path, "etc", "nova", config_path),
+            os.path.join(FLAGS.state_path, "etc", config_path),
+            os.path.join(FLAGS.state_path, config_path),
+            "/etc/nova/%s" % config_path,
+        ]
 
-def load_paste_configuration(filename, appname):
+        for path in possible_locations:
-    """Returns a paste configuration dict, or None."""
+            if os.path.exists(path):
-    filename = os.path.abspath(filename)
+                return os.path.abspath(path)
-    config = None
-    try:
-        config = deploy.appconfig('config:%s' % filename, name=appname)
-    except LookupError:
-        pass
-    return config
 
+        raise exception.PasteConfigNotFound(path=os.path.abspath(config_path))
 
-def load_paste_app(filename, appname):
+    def load_app(self, name):
-    """Builds a wsgi app from a paste config, None if app not configured."""
+        """Return the paste URLMap wrapped WSGI application.
-    filename = os.path.abspath(filename)
+
-    app = None
+        :param name: Name of the application to load.
-    try:
+        :returns: Paste URLMap object wrapping the requested application.
-        app = deploy.loadapp('config:%s' % filename, name=appname)
+        :raises: `nova.exception.PasteAppNotFound`
-    except LookupError:
+
-        pass
+        """
-    return app
+        try:
+            return deploy.loadapp("config:%s" % self.config_path, name=name)
+        except LookupError as err:
+            LOG.error(err)
+            raise exception.PasteAppNotFound(name=name, path=self.config_path)

run_tests.py

@@ -69,7 +69,6 @@ from nose import core
 from nose import result
 
 from nova import log as logging
-from nova.tests import fake_flags
 
 
 class _AnsiColorizer(object):
@@ -211,11 +210,11 @@ class NovaTestResult(result.TextTestResult):
                 break
         sys.stdout = stdout
 
-        # NOTE(lorinh): Initialize start_time in case a sqlalchemy-migrate 
+        # NOTE(lorinh): Initialize start_time in case a sqlalchemy-migrate
-        # error results in it failing to be initialized later. Otherwise, 
+        # error results in it failing to be initialized later. Otherwise,
         # _handleElapsedTime will fail, causing the wrong error message to
         # be outputted.
-        self.start_time = time.time() 
+        self.start_time = time.time()
 
     def getDescription(self, test):
         return str(test)

run_tests.sh

@@ -6,6 +6,8 @@ function usage {
   echo ""
   echo "  -V, --virtual-env        Always use virtualenv.  Install automatically if not present"
   echo "  -N, --no-virtual-env     Don't use virtualenv.  Run tests in local environment"
+  echo "  -r, --recreate-db        Recreate the test database (deprecated, as this is now the default)."
+  echo "  -n, --no-recreate-db     Don't recreate the test database."
   echo "  -x, --stop               Stop running tests after the first error or failure."
   echo "  -f, --force              Force a clean re-build of the virtual environment. Useful when dependencies have been added."
   echo "  -p, --pep8               Just run pep8"
@@ -23,6 +25,8 @@ function process_option {
     -h|--help) usage;;
     -V|--virtual-env) let always_venv=1; let never_venv=0;;
     -N|--no-virtual-env) let always_venv=0; let never_venv=1;;
+    -r|--recreate-db) let recreate_db=1;;
+    -n|--no-recreate-db) let recreate_db=0;;
     -f|--force) let force=1;;
     -p|--pep8) let just_pep8=1;;
     -*) noseopts="$noseopts $1";;
@@ -39,6 +43,7 @@ noseargs=
 noseopts=
 wrapper=""
 just_pep8=0
+recreate_db=1
 
 for arg in "$@"; do
   process_option $arg
@@ -108,6 +113,10 @@ if [ $just_pep8 -eq 1 ]; then
     exit
 fi
 
+if [ $recreate_db -eq 1 ]; then
+    rm tests.sqlite
+fi
+
 run_tests || exit
 
 # NOTE(sirp): we only want to run pep8 when we're running the full-test suite,