From 0020f14f43aa6f024d9aab7dc67c79caaaeb8257 Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Tue, 15 Feb 2011 11:15:59 -0800
Subject: [PATCH 02/19] zone/info works

---
 bin/nova-combined              | 4 ++--
 nova/api/openstack/__init__.py | 6 +++---
 nova/api/openstack/zones.py    | 7 ++++++-
 nova/flags.py                  | 5 +++++
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/bin/nova-combined b/bin/nova-combined
index 913c866bf28e..a0f552d643fa 100755
--- a/bin/nova-combined
+++ b/bin/nova-combined
@@ -53,11 +53,11 @@ if __name__ == '__main__':
 
     compute = service.Service.create(binary='nova-compute')
     network = service.Service.create(binary='nova-network')
-    volume = service.Service.create(binary='nova-volume')
+    #volume = service.Service.create(binary='nova-volume')
     scheduler = service.Service.create(binary='nova-scheduler')
     #objectstore = service.Service.create(binary='nova-objectstore')
 
-    service.serve(compute, network, volume, scheduler)
+    service.serve(compute, network, scheduler)
 
     apps = []
     paste_config_file = wsgi.paste_config_file('nova-api.conf')
diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py
index 33d040ab3d93..95fce7f84ada 100644
--- a/nova/api/openstack/__init__.py
+++ b/nova/api/openstack/__init__.py
@@ -76,13 +76,13 @@ class APIRouter(wsgi.Router):
             LOG.debug(_("Including admin operations in API."))
             server_members['pause'] = 'POST'
             server_members['unpause'] = 'POST'
-            server_members["diagnostics"] = "GET"
-            server_members["actions"] = "GET"
+            server_members['diagnostics'] = 'GET'
+            server_members['actions'] = 'GET'
             server_members['suspend'] = 'POST'
             server_members['resume'] = 'POST'
 
             mapper.resource("zone", "zones", controller=zones.Controller(),
-                        collection={'detail': 'GET'})
+                        collection={'detail': 'GET', 'info': 'GET'}),
 
         mapper.resource("server", "servers", controller=servers.Controller(),
                         collection={'detail': 'GET'},
diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py
index 830464ffd63a..16e5e366bbf7 100644
--- a/nova/api/openstack/zones.py
+++ b/nova/api/openstack/zones.py
@@ -42,7 +42,7 @@ class Controller(wsgi.Controller):
     _serialization_metadata = {
         'application/xml': {
             "attributes": {
-                "zone": ["id", "api_url"]}}}
+                "zone": ["id", "api_url", "name", "capabilities"]}}}
 
     def index(self, req):
         """Return all zones in brief"""
@@ -55,6 +55,11 @@ class Controller(wsgi.Controller):
         """Return all zones in detail"""
         return self.index(req)
 
+    def info(self, req):
+        """Return name and capabilities for this zone."""
+        return dict(zone=dict(name=FLAGS.zone_name, 
+                    capabilities=FLAGS.zone_capabilities))
+
     def show(self, req, id):
         """Return data about the given zone id"""
         zone_id = int(id)
diff --git a/nova/flags.py b/nova/flags.py
index 3ba3fe6faa1e..0a45499f334c 100644
--- a/nova/flags.py
+++ b/nova/flags.py
@@ -312,3 +312,8 @@ DEFINE_string('host', socket.gethostname(),
 
 DEFINE_string('node_availability_zone', 'nova',
               'availability zone of this node')
+
+DEFINE_string('zone_name', 'nova', 'name of this zone')
+DEFINE_string('zone_capabilities', 'xen, linux', 
+              'comma-delimited list of tags which represent boolean'
+              ' capabilities of this zone')

From 5afc1f05d303cd58fb0bf94e5d35e5bc28b9d75c Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Tue, 15 Feb 2011 13:40:54 -0800
Subject: [PATCH 03/19] polling working

---
 nova/scheduler/manager.py | 55 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py
index e9b47512e0f6..00cab60cf4bb 100644
--- a/nova/scheduler/manager.py
+++ b/nova/scheduler/manager.py
@@ -22,6 +22,9 @@ Scheduler Service
 """
 
 import functools
+import novatools
+
+from datetime import datetime
 
 from nova import db
 from nova import flags
@@ -35,20 +38,72 @@ FLAGS = flags.FLAGS
 flags.DEFINE_string('scheduler_driver',
                     'nova.scheduler.chance.ChanceScheduler',
                     'Driver to use for the scheduler')
+flags.DEFINE_integer('zone_db_check_interval',
+                    60,
+                    'Seconds between getting fresh zone info from db.')
 
 
+class ZoneState(object):
+    """Holds the state of all connected child zones."""
+    def __init__(self):
+       self.is_active = True
+       self.name = None
+       self.capabilities = None
+       self.retry = 0
+       self.last_seen = datetime.min
+
+    def update(self, zone):
+       self.zone_id = zone.id
+       self.api_url = zone.api_url
+       self.username = zone.username
+       self.password = zone.password
+
+
+class ZoneManager(object):
+    """Keeps the zone states updated."""
+    def __init__(self):
+        self.last_zone_db_check = datetime.min
+        self.zone_states = {}
+
+    def _refresh_from_db(self, context):
+        zones = db.zone_get_all(context)
+        existing = self.zone_states.keys()
+        for zone in zones:
+            if zone.id not in existing:
+                self.zone_state[zone.id] = ZoneState()
+            self.zone_state[zones.id].update(zone)
+ 
+    def _poll_zones(self, context):
+        pass
+
+    def ping(self, context=None):
+        """Ping should be called periodically to update zone status."""
+        logging.debug("ZoneManager PING")
+        diff = datetime.now() - self.last_zone_db_check
+        if diff.seconds >=  FLAGS.zone_db_check_interval:
+            logging.debug("ZoneManager RECHECKING DB ")
+            self.last_zone_db_check = datetime.now()
+            self._refresh_from_db(context)
+        self._poll_zones(context)
+            
+
 class SchedulerManager(manager.Manager):
     """Chooses a host to run instances on."""
     def __init__(self, scheduler_driver=None, *args, **kwargs):
         if not scheduler_driver:
             scheduler_driver = FLAGS.scheduler_driver
         self.driver = utils.import_object(scheduler_driver)
+        self.zone_manager = ZoneManager()
         super(SchedulerManager, self).__init__(*args, **kwargs)
 
     def __getattr__(self, key):
         """Converts all method calls to use the schedule method"""
         return functools.partial(self._schedule, key)
 
+    def periodic_tasks(self, context=None):
+        """Poll child zones periodically to get status."""
+        self.zone_manager.ping(context)
+
     def _schedule(self, method, context, topic, *args, **kwargs):
         """Tries to call schedule_* method on the driver to retrieve host.
 

From 89a2ee5ee5ea7dc3d9fed4a2d5aa2fe2faed9f2b Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Wed, 16 Feb 2011 11:04:48 -0800
Subject: [PATCH 04/19] novatools call to child zones done

---
 nova/scheduler/manager.py | 51 +++++++++++++++++++++++++++++----------
 1 file changed, 38 insertions(+), 13 deletions(-)

diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py
index 00cab60cf4bb..693f8cb4b0cb 100644
--- a/nova/scheduler/manager.py
+++ b/nova/scheduler/manager.py
@@ -23,8 +23,10 @@ Scheduler Service
 
 import functools
 import novatools
+import thread
 
 from datetime import datetime
+from eventlet.greenpool import GreenPool
 
 from nova import db
 from nova import flags
@@ -46,17 +48,28 @@ flags.DEFINE_integer('zone_db_check_interval',
 class ZoneState(object):
     """Holds the state of all connected child zones."""
     def __init__(self):
-       self.is_active = True
-       self.name = None
-       self.capabilities = None
-       self.retry = 0
-       self.last_seen = datetime.min
-
+        self.is_active = True
+        self.name = None
+        self.capabilities = None
+        self.retry = 0
+        self.last_seen = datetime.min
+ 
     def update(self, zone):
-       self.zone_id = zone.id
-       self.api_url = zone.api_url
-       self.username = zone.username
-       self.password = zone.password
+        """Update zone credentials from db"""
+        self.zone_id = zone.id
+        self.api_url = zone.api_url
+        self.username = zone.username
+        self.password = zone.password
+ 
+
+def _poll_zone(zone):
+    """Eventlet worker to poll a zone."""
+    logging.debug("_POLL_ZONE: STARTING")
+    os = novatools.OpenStack(zone.username, zone.password, zone.api_url)
+    zone_metadata = os.zones.info()
+    logging.debug("_POLL_ZONE: GOT %s" % zone_metadata._info)
+
+    # Stuff this in our cache.
 
 
 class ZoneManager(object):
@@ -66,15 +79,27 @@ class ZoneManager(object):
         self.zone_states = {}
 
     def _refresh_from_db(self, context):
+        """Make our zone state map match the db."""
+        # Add/update existing zones ...
         zones = db.zone_get_all(context)
         existing = self.zone_states.keys()
+        db_keys = []
         for zone in zones:
+            db_keys.append(zone.id)
             if zone.id not in existing:
-                self.zone_state[zone.id] = ZoneState()
-            self.zone_state[zones.id].update(zone)
+                self.zone_states[zone.id] = ZoneState()
+            self.zone_states[zone.id].update(zone)
+
+        # Cleanup zones removed from db ...
+        for zone_id in self.zone_states.keys():
+            if zone_id not in db_keys:
+                del self.zone_states[zone_id]
  
     def _poll_zones(self, context):
-        pass
+        """Try to connect to each child zone and get update."""
+
+        green_pool = GreenPool()
+        green_pool.imap(_poll_zone, self.zone_states.values())
 
     def ping(self, context=None):
         """Ping should be called periodically to update zone status."""

From ec39332356d204d14c8910bf72056efd9e943dd0 Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Wed, 16 Feb 2011 11:05:03 -0800
Subject: [PATCH 05/19] novatools call to child zones done

---
 nova/scheduler/zone_manager.py | 126 +++++++++++++++++++++++++++++++++
 1 file changed, 126 insertions(+)
 create mode 100644 nova/scheduler/zone_manager.py

diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
new file mode 100644
index 000000000000..dd910eb0970c
--- /dev/null
+++ b/nova/scheduler/zone_manager.py
@@ -0,0 +1,126 @@
+# Copyright (c) 2010 Openstack, LLC.
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+ZoneManager oversees all communications with child Zones.
+"""
+
+import novatools
+import thread
+
+from datetime import datetime
+from eventlet.greenpool import GreenPool
+
+from nova import db
+from nova import flags
+from nova import log as logging
+
+FLAGS = flags.FLAGS
+flags.DEFINE_integer('zone_db_check_interval', 60,
+                    'Seconds between getting fresh zone info from db.')
+flags.DEFINE_integer('zone_failures_to_offline', 3,
+             'Number of consecutive errors before marking zone offline')
+
+
+class ZoneState(object):
+    """Holds the state of all connected child zones."""
+    def __init__(self):
+        self.is_active = True
+        self.name = None
+        self.capabilities = None
+        self.attempt = 0
+        self.last_seen = datetime.min
+        self.last_exception = None
+        self.last_exception_time = None
+ 
+    def update_credentials(self, zone):
+        """Update zone credentials from db"""
+        self.zone_id = zone.id
+        self.api_url = zone.api_url
+        self.username = zone.username
+        self.password = zone.password
+ 
+    def update_metadata(self, zone_metadata):
+        """Update zone metadata after successful communications with
+           child zone."""
+        self.last_seen = datetime.now()
+        self.attempt = 0
+        self.name = zone_metadata["name"]
+        self.capabilities = zone_metadata["capabilities"]
+        self.is_active = True
+
+    def log_error(self, exception):
+        """Something went wrong. Check to see if zone should be
+           marked as offline."""
+        self.last_exception = exception
+        self.last_exception_time = datetime.now()
+        logging.warning(_("%s error talking to zone %s") % (exception,
+            zone.api_url, FLAGS.zone_failures_to_offline))
+
+       self.attempt += 1
+        if self.attempt >= FLAGS.zone_failures_to_offline:
+           self.is_active = False
+           logging.error(_("No answer from zone %s after %d "
+               "attempts. Marking inactive.") % (zone.api_url,
+               FLAGS.zone_failures_to_offline))
+
+def _poll_zone(zone):
+    """Eventlet worker to poll a zone."""
+    logging.debug("_POLL_ZONE: STARTING")
+    os = novatools.OpenStack(zone.username, zone.password, zone.api_url)
+    try:
+        zone.update_metadata(os.zones.info()._info)
+    except Exception, e:
+        zone.log_error(e)
+
+class ZoneManager(object):
+    """Keeps the zone states updated."""
+    def __init__(self):
+        self.last_zone_db_check = datetime.min
+        self.zone_states = {}
+
+    def _refresh_from_db(self, context):
+        """Make our zone state map match the db."""
+        # Add/update existing zones ...
+        zones = db.zone_get_all(context)
+        existing = self.zone_states.keys()
+        db_keys = []
+        for zone in zones:
+            db_keys.append(zone.id)
+            if zone.id not in existing:
+                self.zone_states[zone.id] = ZoneState()
+            self.zone_states[zone.id].update_credentials(zone)
+
+        # Cleanup zones removed from db ...
+        for zone_id in self.zone_states.keys():
+            if zone_id not in db_keys:
+                del self.zone_states[zone_id]
+ 
+    def _poll_zones(self, context):
+        """Try to connect to each child zone and get update."""
+        green_pool = GreenPool()
+        green_pool.imap(_poll_zone, self.zone_states.values())
+
+    def ping(self, context=None):
+        """Ping should be called periodically to update zone status."""
+        logging.debug("ZoneManager PING")
+        diff = datetime.now() - self.last_zone_db_check
+        if diff.seconds >=  FLAGS.zone_db_check_interval:
+            logging.debug("ZoneManager RECHECKING DB ")
+            self.last_zone_db_check = datetime.now()
+            self._refresh_from_db(context)
+        self._poll_zones(context)

From 49a7e430ca30768a68a111223068652c781206fe Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Wed, 16 Feb 2011 13:17:42 -0800
Subject: [PATCH 06/19] zone manager tests

---
 nova/scheduler/zone_manager.py |   2 +-
 nova/tests/test_zones.py       | 132 +++++++++++++++++++++++++++++++++
 2 files changed, 133 insertions(+), 1 deletion(-)
 create mode 100644 nova/tests/test_zones.py

diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index dd910eb0970c..0974f271bc85 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -71,7 +71,7 @@ class ZoneState(object):
         logging.warning(_("%s error talking to zone %s") % (exception,
             zone.api_url, FLAGS.zone_failures_to_offline))
 
-       self.attempt += 1
+        self.attempt += 1
         if self.attempt >= FLAGS.zone_failures_to_offline:
            self.is_active = False
            logging.error(_("No answer from zone %s after %d "
diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py
new file mode 100644
index 000000000000..b4c8815d5d41
--- /dev/null
+++ b/nova/tests/test_zones.py
@@ -0,0 +1,132 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Tests For ZoneManager
+"""
+
+import datetime
+import mox
+
+from nova import context
+from nova import db
+from nova import flags
+from nova import service
+from nova import test
+from nova import rpc
+from nova import utils
+from nova.auth import manager as auth_manager
+from nova.scheduler import zone_manager
+
+
+class FakeZone:
+    """Represents a fake zone from the db"""
+    def __init__(self, *args, **kwargs):
+        for k, v in kwargs.iteritems():
+            setattr(self, k, v)
+
+
+class ZoneManagerTestCase(test.TestCase):
+    """Test case for zone manager"""
+    def test_ping(self):
+        zm = zone_manager.ZoneManager()
+        self.mox.StubOutWithMock(zm, '_refresh_from_db')
+        self.mox.StubOutWithMock(zm, '_poll_zones')
+        zm._refresh_from_db(mox.IgnoreArg())
+        zm._poll_zones(mox.IgnoreArg())
+
+        self.mox.ReplayAll()
+        zm.ping(None)
+        self.mox.VerifyAll()
+
+    def test_refresh_from_db_new(self):
+        zm = zone_manager.ZoneManager()
+
+        self.mox.StubOutWithMock(db, 'zone_get_all')
+        db.zone_get_all(mox.IgnoreArg()).AndReturn([
+               FakeZone(id=1, api_url='http://foo.com', username='user1',
+                    password='pass1'),
+            ])
+        
+        self.assertEquals(len(zm.zone_states), 0)
+
+        self.mox.ReplayAll()
+        zm._refresh_from_db(None)
+        self.mox.VerifyAll()
+
+        self.assertEquals(len(zm.zone_states), 1)
+        self.assertEquals(zm.zone_states[1].username, 'user1')
+
+    def test_refresh_from_db_replace_existing(self):
+        zm = zone_manager.ZoneManager()
+        zone_state = zone_manager.ZoneState()
+        zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com',
+                        username='user1', password='pass1'))
+        zm.zone_states[1] = zone_state
+
+        self.mox.StubOutWithMock(db, 'zone_get_all')
+        db.zone_get_all(mox.IgnoreArg()).AndReturn([
+               FakeZone(id=1, api_url='http://foo.com', username='user2',
+                    password='pass2'),
+            ])
+        
+        self.assertEquals(len(zm.zone_states), 1)
+
+        self.mox.ReplayAll()
+        zm._refresh_from_db(None)
+        self.mox.VerifyAll()
+
+        self.assertEquals(len(zm.zone_states), 1)
+        self.assertEquals(zm.zone_states[1].username, 'user2')
+
+    def test_refresh_from_db_missing(self):
+        zm = zone_manager.ZoneManager()
+        zone_state = zone_manager.ZoneState()
+        zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com',
+                        username='user1', password='pass1'))
+        zm.zone_states[1] = zone_state
+
+        self.mox.StubOutWithMock(db, 'zone_get_all')
+        db.zone_get_all(mox.IgnoreArg()).AndReturn([ ])
+        
+        self.assertEquals(len(zm.zone_states), 1)
+
+        self.mox.ReplayAll()
+        zm._refresh_from_db(None)
+        self.mox.VerifyAll()
+
+        self.assertEquals(len(zm.zone_states), 0)
+
+    def test_refresh_from_db_add_and_delete(self):
+        zm = zone_manager.ZoneManager()
+        zone_state = zone_manager.ZoneState()
+        zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com',
+                        username='user1', password='pass1'))
+        zm.zone_states[1] = zone_state
+
+        self.mox.StubOutWithMock(db, 'zone_get_all')
+        
+        db.zone_get_all(mox.IgnoreArg()).AndReturn([
+               FakeZone(id=2, api_url='http://foo.com', username='user2',
+                    password='pass2'),
+            ])
+        self.assertEquals(len(zm.zone_states), 1)
+
+        self.mox.ReplayAll()
+        zm._refresh_from_db(None)
+        self.mox.VerifyAll()
+
+        self.assertEquals(len(zm.zone_states), 1)
+        self.assertEquals(zm.zone_states[2].username, 'user2')

From 719dbda7f8b856af334744de4807036e6ee704c1 Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Wed, 16 Feb 2011 18:30:56 -0800
Subject: [PATCH 07/19] polling tests

---
 nova/scheduler/zone_manager.py | 15 ++++++++-----
 nova/tests/test_zones.py       | 41 ++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index 0974f271bc85..a6bbc2ebdbd4 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -68,22 +68,27 @@ class ZoneState(object):
            marked as offline."""
         self.last_exception = exception
         self.last_exception_time = datetime.now()
-        logging.warning(_("%s error talking to zone %s") % (exception,
-            zone.api_url, FLAGS.zone_failures_to_offline))
+        logging.warning(_("'%s' error talking to zone %s") % (exception,
+            self.api_url))
 
         self.attempt += 1
         if self.attempt >= FLAGS.zone_failures_to_offline:
            self.is_active = False
            logging.error(_("No answer from zone %s after %d "
-               "attempts. Marking inactive.") % (zone.api_url,
+               "attempts. Marking inactive.") % (self.api_url,
                FLAGS.zone_failures_to_offline))
 
+
+def _call_novatools(zone):
+    """Call novatools. Broken out for testing purposes."""
+    os = novatools.OpenStack(zone.username, zone.password, zone.api_url)
+    return os.zones.info()._info
+
 def _poll_zone(zone):
     """Eventlet worker to poll a zone."""
     logging.debug("_POLL_ZONE: STARTING")
-    os = novatools.OpenStack(zone.username, zone.password, zone.api_url)
     try:
-        zone.update_metadata(os.zones.info()._info)
+        zone.update_metadata(_call_novatools(zone))
     except Exception, e:
         zone.log_error(e)
 
diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py
index b4c8815d5d41..2cb070acacbb 100644
--- a/nova/tests/test_zones.py
+++ b/nova/tests/test_zones.py
@@ -19,6 +19,7 @@ Tests For ZoneManager
 
 import datetime
 import mox
+import novatools
 
 from nova import context
 from nova import db
@@ -30,6 +31,8 @@ from nova import utils
 from nova.auth import manager as auth_manager
 from nova.scheduler import zone_manager
 
+FLAGS = flags.FLAGS
+
 
 class FakeZone:
     """Represents a fake zone from the db"""
@@ -38,6 +41,11 @@ class FakeZone:
             setattr(self, k, v)
 
 
+def exploding_novatools(zone):
+    """Used when we want to simulate a novatools call failing."""
+    raise Exception("kaboom")
+
+
 class ZoneManagerTestCase(test.TestCase):
     """Test case for zone manager"""
     def test_ping(self):
@@ -130,3 +138,36 @@ class ZoneManagerTestCase(test.TestCase):
 
         self.assertEquals(len(zm.zone_states), 1)
         self.assertEquals(zm.zone_states[2].username, 'user2')
+
+    def test_poll_zone(self):
+        self.mox.StubOutWithMock(zone_manager, '_call_novatools')
+        zone_manager._call_novatools(mox.IgnoreArg()).AndReturn(
+                        dict(name='zohan', capabilities='hairdresser'))
+
+        zone_state = zone_manager.ZoneState()
+        zone_state.update_credentials(FakeZone(id=2,
+                       api_url='http://foo.com', username='user2',
+                       password='pass2'))
+        zone_state.attempt = 1
+
+        self.mox.ReplayAll()
+        zone_manager._poll_zone(zone_state)
+        self.mox.VerifyAll()
+        self.assertEquals(zone_state.attempt, 0)
+        self.assertEquals(zone_state.name, 'zohan')
+
+    def test_poll_zone_fails(self):
+        self.stubs.Set(zone_manager, "_call_novatools", exploding_novatools)
+
+        zone_state = zone_manager.ZoneState()
+        zone_state.update_credentials(FakeZone(id=2,
+                       api_url='http://foo.com', username='user2',
+                       password='pass2'))
+        zone_state.attempt = FLAGS.zone_failures_to_offline - 1
+
+        self.mox.ReplayAll()
+        zone_manager._poll_zone(zone_state)
+        self.mox.VerifyAll()
+        self.assertEquals(zone_state.attempt, 3)
+        self.assertFalse(zone_state.is_active)
+        self.assertEquals(zone_state.name, None)

From 984db08a205bdd9196c3e1cc3415873a853c33ba Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Wed, 16 Feb 2011 18:35:43 -0800
Subject: [PATCH 08/19] style cleanup

---
 nova/scheduler/zone_manager.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index a6bbc2ebdbd4..a35acb000050 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -84,14 +84,16 @@ def _call_novatools(zone):
     os = novatools.OpenStack(zone.username, zone.password, zone.api_url)
     return os.zones.info()._info
 
+
 def _poll_zone(zone):
     """Eventlet worker to poll a zone."""
-    logging.debug("_POLL_ZONE: STARTING")
+    logging.debug(_("Polling zone: %s") % zone.api_url)
     try:
         zone.update_metadata(_call_novatools(zone))
     except Exception, e:
         zone.log_error(e)
 
+
 class ZoneManager(object):
     """Keeps the zone states updated."""
     def __init__(self):
@@ -122,10 +124,9 @@ class ZoneManager(object):
 
     def ping(self, context=None):
         """Ping should be called periodically to update zone status."""
-        logging.debug("ZoneManager PING")
         diff = datetime.now() - self.last_zone_db_check
         if diff.seconds >=  FLAGS.zone_db_check_interval:
-            logging.debug("ZoneManager RECHECKING DB ")
+            logging.debug("Updating zone cache from db.")
             self.last_zone_db_check = datetime.now()
             self._refresh_from_db(context)
         self._poll_zones(context)

From aa71a25c9f9bf5df3aea781138fa8d69654f06d9 Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Thu, 17 Feb 2011 12:12:19 -0800
Subject: [PATCH 09/19] zone list now comes from scheduler zonemanager

---
 nova/api/openstack/zones.py    | 32 ++++++++++++-
 nova/scheduler/manager.py      | 82 +++-------------------------------
 nova/scheduler/zone_manager.py | 11 ++++-
 3 files changed, 46 insertions(+), 79 deletions(-)

diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py
index 16e5e366bbf7..bd2c488d96f4 100644
--- a/nova/api/openstack/zones.py
+++ b/nova/api/openstack/zones.py
@@ -19,6 +19,7 @@ import logging
 from nova import flags
 from nova import wsgi
 from nova import db
+from nova import rpc
 
 
 FLAGS = flags.FLAGS
@@ -33,6 +34,10 @@ def _filter_keys(item, keys):
     return dict((k, v) for k, v in item.iteritems() if k in keys)
 
 
+def _exclude_keys(item, keys):
+    return dict((k, v) for k, v in item.iteritems() if k not in keys)
+
+
 def _scrub_zone(zone):
     return _filter_keys(zone, ('id', 'api_url'))
 
@@ -44,11 +49,34 @@ class Controller(wsgi.Controller):
             "attributes": {
                 "zone": ["id", "api_url", "name", "capabilities"]}}}
 
+    def _call_scheduler(self, method, context, params=None):
+        """Generic handler for RPC calls to the scheduler.
+
+        :param params: Optional dictionary of arguments to be passed to the
+                       scheduler worker
+
+        :retval: Result returned by scheduler worker
+        """
+        if not params:
+            params = {}
+        queue = FLAGS.scheduler_topic
+        kwargs = {'method': method, 'args': params}
+        return rpc.call(context, queue, kwargs)
+
     def index(self, req):
         """Return all zones in brief"""
-        items = db.zone_get_all(req.environ['nova.context'])
+        # Ask the ZoneManager in the Scheduler for most recent data.
+        items = self._call_scheduler('get_zone_list', 
+                          req.environ['nova.context'])
+        for item in items:
+            item['api_url'] = item['api_url'].replace('\\/', '/')
+
+        # Or fall-back to the database ...
+        if len(items) == 0:
+            items = db.zone_get_all(req.environ['nova.context'])
         items = common.limited(items, req)
-        items = [_scrub_zone(item) for item in items]
+        items = [_exclude_keys(item, ['username', 'password']) 
+                      for item in items]
         return dict(zones=items)
 
     def detail(self, req):
diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py
index 693f8cb4b0cb..00a0f4100bc7 100644
--- a/nova/scheduler/manager.py
+++ b/nova/scheduler/manager.py
@@ -22,11 +22,6 @@ Scheduler Service
 """
 
 import functools
-import novatools
-import thread
-
-from datetime import datetime
-from eventlet.greenpool import GreenPool
 
 from nova import db
 from nova import flags
@@ -34,83 +29,14 @@ from nova import log as logging
 from nova import manager
 from nova import rpc
 from nova import utils
+from nova.scheduler.zone_manager import ZoneManager
 
 LOG = logging.getLogger('nova.scheduler.manager')
 FLAGS = flags.FLAGS
 flags.DEFINE_string('scheduler_driver',
                     'nova.scheduler.chance.ChanceScheduler',
                     'Driver to use for the scheduler')
-flags.DEFINE_integer('zone_db_check_interval',
-                    60,
-                    'Seconds between getting fresh zone info from db.')
-
-
-class ZoneState(object):
-    """Holds the state of all connected child zones."""
-    def __init__(self):
-        self.is_active = True
-        self.name = None
-        self.capabilities = None
-        self.retry = 0
-        self.last_seen = datetime.min
- 
-    def update(self, zone):
-        """Update zone credentials from db"""
-        self.zone_id = zone.id
-        self.api_url = zone.api_url
-        self.username = zone.username
-        self.password = zone.password
- 
-
-def _poll_zone(zone):
-    """Eventlet worker to poll a zone."""
-    logging.debug("_POLL_ZONE: STARTING")
-    os = novatools.OpenStack(zone.username, zone.password, zone.api_url)
-    zone_metadata = os.zones.info()
-    logging.debug("_POLL_ZONE: GOT %s" % zone_metadata._info)
-
-    # Stuff this in our cache.
-
-
-class ZoneManager(object):
-    """Keeps the zone states updated."""
-    def __init__(self):
-        self.last_zone_db_check = datetime.min
-        self.zone_states = {}
-
-    def _refresh_from_db(self, context):
-        """Make our zone state map match the db."""
-        # Add/update existing zones ...
-        zones = db.zone_get_all(context)
-        existing = self.zone_states.keys()
-        db_keys = []
-        for zone in zones:
-            db_keys.append(zone.id)
-            if zone.id not in existing:
-                self.zone_states[zone.id] = ZoneState()
-            self.zone_states[zone.id].update(zone)
-
-        # Cleanup zones removed from db ...
-        for zone_id in self.zone_states.keys():
-            if zone_id not in db_keys:
-                del self.zone_states[zone_id]
- 
-    def _poll_zones(self, context):
-        """Try to connect to each child zone and get update."""
-
-        green_pool = GreenPool()
-        green_pool.imap(_poll_zone, self.zone_states.values())
-
-    def ping(self, context=None):
-        """Ping should be called periodically to update zone status."""
-        logging.debug("ZoneManager PING")
-        diff = datetime.now() - self.last_zone_db_check
-        if diff.seconds >=  FLAGS.zone_db_check_interval:
-            logging.debug("ZoneManager RECHECKING DB ")
-            self.last_zone_db_check = datetime.now()
-            self._refresh_from_db(context)
-        self._poll_zones(context)
-            
+           
 
 class SchedulerManager(manager.Manager):
     """Chooses a host to run instances on."""
@@ -129,6 +55,10 @@ class SchedulerManager(manager.Manager):
         """Poll child zones periodically to get status."""
         self.zone_manager.ping(context)
 
+    def get_zone_list(self, context=None):
+        """Get a list of zones from the ZoneManager."""
+        return self.zone_manager.get_zone_list()
+
     def _schedule(self, method, context, topic, *args, **kwargs):
         """Tries to call schedule_* method on the driver to retrieve host.
 
diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index a35acb000050..4fa5289739ad 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -21,6 +21,7 @@ ZoneManager oversees all communications with child Zones.
 
 import novatools
 import thread
+import traceback
 
 from datetime import datetime
 from eventlet.greenpool import GreenPool
@@ -63,6 +64,11 @@ class ZoneState(object):
         self.capabilities = zone_metadata["capabilities"]
         self.is_active = True
 
+    def to_dict(self):
+        return dict(name=self.name, capabilities=self.capabilities,
+                    is_active=self.is_active, api_url=self.api_url,
+                    id=self.zone_id)
+
     def log_error(self, exception):
         """Something went wrong. Check to see if zone should be
            marked as offline."""
@@ -91,7 +97,7 @@ def _poll_zone(zone):
     try:
         zone.update_metadata(_call_novatools(zone))
     except Exception, e:
-        zone.log_error(e)
+        zone.log_error(traceback.format_exc())
 
 
 class ZoneManager(object):
@@ -100,6 +106,9 @@ class ZoneManager(object):
         self.last_zone_db_check = datetime.min
         self.zone_states = {}
 
+    def get_zone_list(self):
+        return [ zone.to_dict() for zone in self.zone_states.values() ]
+
     def _refresh_from_db(self, context):
         """Make our zone state map match the db."""
         # Add/update existing zones ...

From e77f8751dd59e5d650d047a6711c3d137947dda7 Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Thu, 17 Feb 2011 16:18:03 -0400
Subject: [PATCH 10/19] fixup

---
 bin/nova-combined              |  4 ++--
 nova/api/openstack/zones.py    |  6 +++---
 nova/flags.py                  |  2 +-
 nova/scheduler/manager.py      |  2 +-
 nova/scheduler/zone_manager.py | 18 +++++++++---------
 nova/tests/test_zones.py       | 10 +++++-----
 6 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/bin/nova-combined b/bin/nova-combined
index a0f552d643fa..913c866bf28e 100755
--- a/bin/nova-combined
+++ b/bin/nova-combined
@@ -53,11 +53,11 @@ if __name__ == '__main__':
 
     compute = service.Service.create(binary='nova-compute')
     network = service.Service.create(binary='nova-network')
-    #volume = service.Service.create(binary='nova-volume')
+    volume = service.Service.create(binary='nova-volume')
     scheduler = service.Service.create(binary='nova-scheduler')
     #objectstore = service.Service.create(binary='nova-objectstore')
 
-    service.serve(compute, network, scheduler)
+    service.serve(compute, network, volume, scheduler)
 
     apps = []
     paste_config_file = wsgi.paste_config_file('nova-api.conf')
diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py
index bd2c488d96f4..f7517682472c 100644
--- a/nova/api/openstack/zones.py
+++ b/nova/api/openstack/zones.py
@@ -66,7 +66,7 @@ class Controller(wsgi.Controller):
     def index(self, req):
         """Return all zones in brief"""
         # Ask the ZoneManager in the Scheduler for most recent data.
-        items = self._call_scheduler('get_zone_list', 
+        items = self._call_scheduler('get_zone_list',
                           req.environ['nova.context'])
         for item in items:
             item['api_url'] = item['api_url'].replace('\\/', '/')
@@ -75,7 +75,7 @@ class Controller(wsgi.Controller):
         if len(items) == 0:
             items = db.zone_get_all(req.environ['nova.context'])
         items = common.limited(items, req)
-        items = [_exclude_keys(item, ['username', 'password']) 
+        items = [_exclude_keys(item, ['username', 'password'])
                       for item in items]
         return dict(zones=items)
 
@@ -85,7 +85,7 @@ class Controller(wsgi.Controller):
 
     def info(self, req):
         """Return name and capabilities for this zone."""
-        return dict(zone=dict(name=FLAGS.zone_name, 
+        return dict(zone=dict(name=FLAGS.zone_name,
                     capabilities=FLAGS.zone_capabilities))
 
     def show(self, req, id):
diff --git a/nova/flags.py b/nova/flags.py
index 0a45499f334c..60d7cdd0694b 100644
--- a/nova/flags.py
+++ b/nova/flags.py
@@ -314,6 +314,6 @@ DEFINE_string('node_availability_zone', 'nova',
               'availability zone of this node')
 
 DEFINE_string('zone_name', 'nova', 'name of this zone')
-DEFINE_string('zone_capabilities', 'xen, linux', 
+DEFINE_string('zone_capabilities', 'xen, linux',
               'comma-delimited list of tags which represent boolean'
               ' capabilities of this zone')
diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py
index 00a0f4100bc7..7ced33b9c175 100644
--- a/nova/scheduler/manager.py
+++ b/nova/scheduler/manager.py
@@ -36,7 +36,7 @@ FLAGS = flags.FLAGS
 flags.DEFINE_string('scheduler_driver',
                     'nova.scheduler.chance.ChanceScheduler',
                     'Driver to use for the scheduler')
-           
+
 
 class SchedulerManager(manager.Manager):
     """Chooses a host to run instances on."""
diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index 4fa5289739ad..e7c37a9a6802 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -47,14 +47,14 @@ class ZoneState(object):
         self.last_seen = datetime.min
         self.last_exception = None
         self.last_exception_time = None
- 
+
     def update_credentials(self, zone):
         """Update zone credentials from db"""
         self.zone_id = zone.id
         self.api_url = zone.api_url
         self.username = zone.username
         self.password = zone.password
- 
+
     def update_metadata(self, zone_metadata):
         """Update zone metadata after successful communications with
            child zone."""
@@ -79,10 +79,10 @@ class ZoneState(object):
 
         self.attempt += 1
         if self.attempt >= FLAGS.zone_failures_to_offline:
-           self.is_active = False
-           logging.error(_("No answer from zone %s after %d "
-               "attempts. Marking inactive.") % (self.api_url,
-               FLAGS.zone_failures_to_offline))
+            self.is_active = False
+            logging.error(_("No answer from zone %s after %d "
+                "attempts. Marking inactive.") % (self.api_url,
+                FLAGS.zone_failures_to_offline))
 
 
 def _call_novatools(zone):
@@ -107,7 +107,7 @@ class ZoneManager(object):
         self.zone_states = {}
 
     def get_zone_list(self):
-        return [ zone.to_dict() for zone in self.zone_states.values() ]
+        return [zone.to_dict() for zone in self.zone_states.values()]
 
     def _refresh_from_db(self, context):
         """Make our zone state map match the db."""
@@ -125,7 +125,7 @@ class ZoneManager(object):
         for zone_id in self.zone_states.keys():
             if zone_id not in db_keys:
                 del self.zone_states[zone_id]
- 
+
     def _poll_zones(self, context):
         """Try to connect to each child zone and get update."""
         green_pool = GreenPool()
@@ -134,7 +134,7 @@ class ZoneManager(object):
     def ping(self, context=None):
         """Ping should be called periodically to update zone status."""
         diff = datetime.now() - self.last_zone_db_check
-        if diff.seconds >=  FLAGS.zone_db_check_interval:
+        if diff.seconds >= FLAGS.zone_db_check_interval:
             logging.debug("Updating zone cache from db.")
             self.last_zone_db_check = datetime.now()
             self._refresh_from_db(context)
diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py
index 2cb070acacbb..7036ebe58269 100644
--- a/nova/tests/test_zones.py
+++ b/nova/tests/test_zones.py
@@ -67,7 +67,7 @@ class ZoneManagerTestCase(test.TestCase):
                FakeZone(id=1, api_url='http://foo.com', username='user1',
                     password='pass1'),
             ])
-        
+
         self.assertEquals(len(zm.zone_states), 0)
 
         self.mox.ReplayAll()
@@ -89,7 +89,7 @@ class ZoneManagerTestCase(test.TestCase):
                FakeZone(id=1, api_url='http://foo.com', username='user2',
                     password='pass2'),
             ])
-        
+
         self.assertEquals(len(zm.zone_states), 1)
 
         self.mox.ReplayAll()
@@ -107,8 +107,8 @@ class ZoneManagerTestCase(test.TestCase):
         zm.zone_states[1] = zone_state
 
         self.mox.StubOutWithMock(db, 'zone_get_all')
-        db.zone_get_all(mox.IgnoreArg()).AndReturn([ ])
-        
+        db.zone_get_all(mox.IgnoreArg()).AndReturn([])
+
         self.assertEquals(len(zm.zone_states), 1)
 
         self.mox.ReplayAll()
@@ -125,7 +125,7 @@ class ZoneManagerTestCase(test.TestCase):
         zm.zone_states[1] = zone_state
 
         self.mox.StubOutWithMock(db, 'zone_get_all')
-        
+
         db.zone_get_all(mox.IgnoreArg()).AndReturn([
                FakeZone(id=2, api_url='http://foo.com', username='user2',
                     password='pass2'),

From b48201be9a5fa08ce21ef241052071800e5777ca Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Thu, 17 Feb 2011 12:43:22 -0800
Subject: [PATCH 11/19] fixed zone list tests

---
 nova/tests/api/openstack/test_zones.py | 32 +++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py
index 5542a1cf3e2d..6c06fa8b8b2a 100644
--- a/nova/tests/api/openstack/test_zones.py
+++ b/nova/tests/api/openstack/test_zones.py
@@ -52,7 +52,20 @@ def zone_delete(context, zone_id):
     pass
 
 
-def zone_get_all(context):
+def zone_get_all_scheduler(x, y, z):
+    return [
+        dict(id=1, api_url='http://foo.com', username='bob',
+                 password='xxx'),
+        dict(id=2, api_url='http://blah.com', username='alice',
+                 password='qwerty')
+    ]
+
+
+def zone_get_all_scheduler_empty(x, y, z):
+    return [] 
+
+
+def zone_get_all_db(context):
     return [
         dict(id=1, api_url='http://foo.com', username='bob',
                  password='xxx'),
@@ -74,7 +87,6 @@ class ZonesTest(unittest.TestCase):
         FLAGS.allow_admin_api = True
 
         self.stubs.Set(nova.db, 'zone_get', zone_get)
-        self.stubs.Set(nova.db, 'zone_get_all', zone_get_all)
         self.stubs.Set(nova.db, 'zone_update', zone_update)
         self.stubs.Set(nova.db, 'zone_create', zone_create)
         self.stubs.Set(nova.db, 'zone_delete', zone_delete)
@@ -83,7 +95,9 @@ class ZonesTest(unittest.TestCase):
         self.stubs.UnsetAll()
         FLAGS.allow_admin_api = self.allow_admin
 
-    def test_get_zone_list(self):
+    def test_get_zone_list_scheduler(self):
+        self.stubs.Set(zones.Controller, '_call_scheduler',
+                            zone_get_all_scheduler)
         req = webob.Request.blank('/v1.0/zones')
         res = req.get_response(fakes.wsgi_app())
         res_dict = json.loads(res.body)
@@ -91,6 +105,18 @@ class ZonesTest(unittest.TestCase):
         self.assertEqual(res.status_int, 200)
         self.assertEqual(len(res_dict['zones']), 2)
 
+    def test_get_zone_list_db(self):
+        self.stubs.Set(zones.Controller, '_call_scheduler', 
+            zone_get_all_scheduler_empty)
+        self.stubs.Set(nova.db, 'zone_get_all', zone_get_all_db)
+        req = webob.Request.blank('/v1.0/zones')
+        res = req.get_response(fakes.wsgi_app())
+        res_dict = json.loads(res.body)
+
+        self.assertEqual(res.status_int, 200)
+        self.assertEqual(len(res_dict['zones']), 2)
+
+
     def test_get_zone_by_id(self):
         req = webob.Request.blank('/v1.0/zones/1')
         res = req.get_response(fakes.wsgi_app())

From b9a03524d03c0ce7fa98fab5531db720941bbfdb Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Thu, 17 Feb 2011 13:23:56 -0800
Subject: [PATCH 12/19] multi positional string fix

---
 nova/scheduler/zone_manager.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index e7c37a9a6802..af0b90f9ff8a 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -74,15 +74,17 @@ class ZoneState(object):
            marked as offline."""
         self.last_exception = exception
         self.last_exception_time = datetime.now()
-        logging.warning(_("'%s' error talking to zone %s") % (exception,
-            self.api_url))
+        api_url = self.api_url
+        logging.warning(_("'%(exception)s' error talking to "
+                          "zone %(api_url)s") % locals())
 
+        max_errors = FLAGS.zone_failures_to_offline:
         self.attempt += 1
-        if self.attempt >= FLAGS.zone_failures_to_offline:
+        if self.attempt >= max_errors:
             self.is_active = False
-            logging.error(_("No answer from zone %s after %d "
-                "attempts. Marking inactive.") % (self.api_url,
-                FLAGS.zone_failures_to_offline))
+            logging.error(_("No answer from zone %(api_url)s "
+                            "after %(max_errors)d "
+                            "attempts. Marking inactive.") % locals())
 
 
 def _call_novatools(zone):

From a33f5495fe261641713131901fee1e83ccc4890f Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Thu, 17 Feb 2011 13:29:19 -0800
Subject: [PATCH 13/19] fixed strings

---
 nova/scheduler/zone_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index af0b90f9ff8a..4bf6e36c665c 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -78,7 +78,7 @@ class ZoneState(object):
         logging.warning(_("'%(exception)s' error talking to "
                           "zone %(api_url)s") % locals())
 
-        max_errors = FLAGS.zone_failures_to_offline:
+        max_errors = FLAGS.zone_failures_to_offline
         self.attempt += 1
         if self.attempt >= max_errors:
             self.is_active = False

From 46269872192b843c80d72206a05c8b759c9f66a8 Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Thu, 17 Feb 2011 17:32:25 -0400
Subject: [PATCH 14/19] merge from dev

---
 nova/tests/api/openstack/test_zones.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py
index 6c06fa8b8b2a..65cc1c023a1d 100644
--- a/nova/tests/api/openstack/test_zones.py
+++ b/nova/tests/api/openstack/test_zones.py
@@ -62,7 +62,7 @@ def zone_get_all_scheduler(x, y, z):
 
 
 def zone_get_all_scheduler_empty(x, y, z):
-    return [] 
+    return []
 
 
 def zone_get_all_db(context):
@@ -106,7 +106,7 @@ class ZonesTest(unittest.TestCase):
         self.assertEqual(len(res_dict['zones']), 2)
 
     def test_get_zone_list_db(self):
-        self.stubs.Set(zones.Controller, '_call_scheduler', 
+        self.stubs.Set(zones.Controller, '_call_scheduler',
             zone_get_all_scheduler_empty)
         self.stubs.Set(nova.db, 'zone_get_all', zone_get_all_db)
         req = webob.Request.blank('/v1.0/zones')
@@ -116,7 +116,6 @@ class ZonesTest(unittest.TestCase):
         self.assertEqual(res.status_int, 200)
         self.assertEqual(len(res_dict['zones']), 2)
 
-
     def test_get_zone_by_id(self):
         req = webob.Request.blank('/v1.0/zones/1')
         res = req.get_response(fakes.wsgi_app())

From e5443fa3e436a95de9d1c353e8772436c7cba8b6 Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Thu, 17 Feb 2011 17:58:38 -0400
Subject: [PATCH 15/19] pip requires novatools

---
 tools/pip-requires | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/pip-requires b/tools/pip-requires
index 3587df644e6a..a5d4675bea89 100644
--- a/tools/pip-requires
+++ b/tools/pip-requires
@@ -10,6 +10,7 @@ boto==1.9b
 carrot==0.10.5
 eventlet==0.9.12
 lockfile==0.8
+python-novatools==2.0
 python-daemon==1.5.5
 python-gflags==1.3
 redis==2.0.0

From e5d979596ff8c588c7bbe82b7f1cb90de8af041a Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Thu, 17 Feb 2011 18:49:30 -0400
Subject: [PATCH 16/19] missing docstring and fixed copyrights

---
 nova/scheduler/zone_manager.py | 3 +--
 nova/tests/test_zones.py       | 1 -
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index 4bf6e36c665c..3e7c1eba8ca4 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -1,6 +1,4 @@
 # Copyright (c) 2010 Openstack, LLC.
-# Copyright 2010 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -109,6 +107,7 @@ class ZoneManager(object):
         self.zone_states = {}
 
     def get_zone_list(self):
+        """Return the list of zones we know about."""
         return [zone.to_dict() for zone in self.zone_states.values()]
 
     def _refresh_from_db(self, context):
diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py
index 7036ebe58269..c273230a93f3 100644
--- a/nova/tests/test_zones.py
+++ b/nova/tests/test_zones.py
@@ -1,5 +1,4 @@
 # Copyright 2010 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may

From c884064e7a9af04b2ebdbbb9ee32318a00716412 Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Fri, 18 Feb 2011 12:08:35 -0400
Subject: [PATCH 17/19] fixups backed on merge comments

---
 nova/api/openstack/zones.py            | 30 ++++------------
 nova/flags.py                          |  5 ++-
 nova/scheduler/api.py                  | 49 ++++++++++++++++++++++++++
 nova/scheduler/zone_manager.py         | 15 ++++----
 nova/tests/api/openstack/test_zones.py | 33 +++++++++--------
 5 files changed, 82 insertions(+), 50 deletions(-)
 create mode 100644 nova/scheduler/api.py

diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py
index f7517682472c..24a4444f7d16 100644
--- a/nova/api/openstack/zones.py
+++ b/nova/api/openstack/zones.py
@@ -1,4 +1,4 @@
-# Copyright 2010 OpenStack LLC.
+# Copyright 2011 OpenStack LLC.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -20,6 +20,7 @@ from nova import flags
 from nova import wsgi
 from nova import db
 from nova import rpc
+from nova.scheduler.api import API
 
 
 FLAGS = flags.FLAGS
@@ -49,31 +50,14 @@ class Controller(wsgi.Controller):
             "attributes": {
                 "zone": ["id", "api_url", "name", "capabilities"]}}}
 
-    def _call_scheduler(self, method, context, params=None):
-        """Generic handler for RPC calls to the scheduler.
-
-        :param params: Optional dictionary of arguments to be passed to the
-                       scheduler worker
-
-        :retval: Result returned by scheduler worker
-        """
-        if not params:
-            params = {}
-        queue = FLAGS.scheduler_topic
-        kwargs = {'method': method, 'args': params}
-        return rpc.call(context, queue, kwargs)
-
     def index(self, req):
         """Return all zones in brief"""
-        # Ask the ZoneManager in the Scheduler for most recent data.
-        items = self._call_scheduler('get_zone_list',
-                          req.environ['nova.context'])
-        for item in items:
-            item['api_url'] = item['api_url'].replace('\\/', '/')
-
-        # Or fall-back to the database ...
-        if len(items) == 0:
+        # Ask the ZoneManager in the Scheduler for most recent data,
+        # or fall-back to the database ...
+        items = API().get_zone_list(req.environ['nova.context'])
+        if not items:
             items = db.zone_get_all(req.environ['nova.context'])
+
         items = common.limited(items, req)
         items = [_exclude_keys(item, ['username', 'password'])
                       for item in items]
diff --git a/nova/flags.py b/nova/flags.py
index 7e4919d6e9ce..41f01fcd7280 100644
--- a/nova/flags.py
+++ b/nova/flags.py
@@ -316,6 +316,5 @@ DEFINE_string('node_availability_zone', 'nova',
               'availability zone of this node')
 
 DEFINE_string('zone_name', 'nova', 'name of this zone')
-DEFINE_string('zone_capabilities', 'xen, linux',
-              'comma-delimited list of tags which represent boolean'
-              ' capabilities of this zone')
+DEFINE_string('zone_capabilities', 'kypervisor:xenserver;os:linux',
+              'Key/Value tags which represent capabilities of this zone')
diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py
new file mode 100644
index 000000000000..8491bf3a9fc4
--- /dev/null
+++ b/nova/scheduler/api.py
@@ -0,0 +1,49 @@
+# Copyright (c) 2011 Openstack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Handles all requests relating to schedulers.
+"""
+
+from nova import flags
+from nova import log as logging
+from nova import rpc
+
+FLAGS = flags.FLAGS
+LOG = logging.getLogger('nova.scheduler.api')
+
+
+class API:
+    """API for interacting with the scheduler."""
+
+    def _call_scheduler(self, method, context, params=None):
+        """Generic handler for RPC calls to the scheduler.
+
+        :param params: Optional dictionary of arguments to be passed to the
+                       scheduler worker
+
+        :retval: Result returned by scheduler worker
+        """
+        if not params:
+            params = {}
+        queue = FLAGS.scheduler_topic
+        kwargs = {'method': method, 'args': params}
+        return rpc.call(context, queue, kwargs)
+
+    def get_zone_list(self, context):
+        items = self._call_scheduler('get_zone_list', context)
+        for item in items:
+            item['api_url'] = item['api_url'].replace('\\/', '/')
+        return items
diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index 3e7c1eba8ca4..783783d06bde 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2010 Openstack, LLC.
+# Copyright (c) 2011 Openstack, LLC.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -87,8 +87,8 @@ class ZoneState(object):
 
 def _call_novatools(zone):
     """Call novatools. Broken out for testing purposes."""
-    os = novatools.OpenStack(zone.username, zone.password, zone.api_url)
-    return os.zones.info()._info
+    client = novatools.OpenStack(zone.username, zone.password, zone.api_url)
+    return client.zones.info()._info
 
 
 def _poll_zone(zone):
@@ -105,6 +105,7 @@ class ZoneManager(object):
     def __init__(self):
         self.last_zone_db_check = datetime.min
         self.zone_states = {}
+        self.green_pool = GreenPool()
 
     def get_zone_list(self):
         """Return the list of zones we know about."""
@@ -123,20 +124,20 @@ class ZoneManager(object):
             self.zone_states[zone.id].update_credentials(zone)
 
         # Cleanup zones removed from db ...
-        for zone_id in self.zone_states.keys():
+        keys = self.zone_states.keys()  # since we're deleting
+        for zone_id in keys:
             if zone_id not in db_keys:
                 del self.zone_states[zone_id]
 
     def _poll_zones(self, context):
         """Try to connect to each child zone and get update."""
-        green_pool = GreenPool()
-        green_pool.imap(_poll_zone, self.zone_states.values())
+        self.green_pool.imap(_poll_zone, self.zone_states.values())
 
     def ping(self, context=None):
         """Ping should be called periodically to update zone status."""
         diff = datetime.now() - self.last_zone_db_check
         if diff.seconds >= FLAGS.zone_db_check_interval:
-            logging.debug("Updating zone cache from db.")
+            logging.debug(_("Updating zone cache from db."))
             self.last_zone_db_check = datetime.now()
             self._refresh_from_db(context)
         self._poll_zones(context)
diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py
index 65cc1c023a1d..4df7c7febfe5 100644
--- a/nova/tests/api/openstack/test_zones.py
+++ b/nova/tests/api/openstack/test_zones.py
@@ -1,4 +1,4 @@
-# Copyright 2010 OpenStack LLC.
+# Copyright 2011 OpenStack LLC.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -24,6 +24,7 @@ from nova import context
 from nova import flags
 from nova.api.openstack import zones
 from nova.tests.api.openstack import fakes
+from nova.scheduler.api import API
 
 
 FLAGS = flags.FLAGS
@@ -31,7 +32,7 @@ FLAGS.verbose = True
 
 
 def zone_get(context, zone_id):
-    return dict(id=1, api_url='http://foo.com', username='bob',
+    return dict(id=1, api_url='http://example.com', username='bob',
                 password='xxx')
 
 
@@ -42,7 +43,7 @@ def zone_create(context, values):
 
 
 def zone_update(context, zone_id, values):
-    zone = dict(id=zone_id, api_url='http://foo.com', username='bob',
+    zone = dict(id=zone_id, api_url='http://example.com', username='bob',
                 password='xxx')
     zone.update(values)
     return zone
@@ -52,24 +53,24 @@ def zone_delete(context, zone_id):
     pass
 
 
-def zone_get_all_scheduler(x, y, z):
+def zone_get_all_scheduler(*args):
     return [
-        dict(id=1, api_url='http://foo.com', username='bob',
+        dict(id=1, api_url='http://example.com', username='bob',
                  password='xxx'),
-        dict(id=2, api_url='http://blah.com', username='alice',
+        dict(id=2, api_url='http://example.org', username='alice',
                  password='qwerty')
     ]
 
 
-def zone_get_all_scheduler_empty(x, y, z):
+def zone_get_all_scheduler_empty(*args):
     return []
 
 
 def zone_get_all_db(context):
     return [
-        dict(id=1, api_url='http://foo.com', username='bob',
+        dict(id=1, api_url='http://example.com', username='bob',
                  password='xxx'),
-        dict(id=2, api_url='http://blah.com', username='alice',
+        dict(id=2, api_url='http://example.org', username='alice',
                  password='qwerty')
     ]
 
@@ -96,8 +97,7 @@ class ZonesTest(unittest.TestCase):
         FLAGS.allow_admin_api = self.allow_admin
 
     def test_get_zone_list_scheduler(self):
-        self.stubs.Set(zones.Controller, '_call_scheduler',
-                            zone_get_all_scheduler)
+        self.stubs.Set(API, '_call_scheduler', zone_get_all_scheduler)
         req = webob.Request.blank('/v1.0/zones')
         res = req.get_response(fakes.wsgi_app())
         res_dict = json.loads(res.body)
@@ -106,8 +106,7 @@ class ZonesTest(unittest.TestCase):
         self.assertEqual(len(res_dict['zones']), 2)
 
     def test_get_zone_list_db(self):
-        self.stubs.Set(zones.Controller, '_call_scheduler',
-            zone_get_all_scheduler_empty)
+        self.stubs.Set(API, '_call_scheduler', zone_get_all_scheduler_empty)
         self.stubs.Set(nova.db, 'zone_get_all', zone_get_all_db)
         req = webob.Request.blank('/v1.0/zones')
         res = req.get_response(fakes.wsgi_app())
@@ -122,7 +121,7 @@ class ZonesTest(unittest.TestCase):
         res_dict = json.loads(res.body)
 
         self.assertEqual(res_dict['zone']['id'], 1)
-        self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com')
+        self.assertEqual(res_dict['zone']['api_url'], 'http://example.com')
         self.assertFalse('password' in res_dict['zone'])
         self.assertEqual(res.status_int, 200)
 
@@ -133,7 +132,7 @@ class ZonesTest(unittest.TestCase):
         self.assertEqual(res.status_int, 200)
 
     def test_zone_create(self):
-        body = dict(zone=dict(api_url='http://blah.zoo', username='fred',
+        body = dict(zone=dict(api_url='http://example.com', username='fred',
                         password='fubar'))
         req = webob.Request.blank('/v1.0/zones')
         req.method = 'POST'
@@ -144,7 +143,7 @@ class ZonesTest(unittest.TestCase):
 
         self.assertEqual(res.status_int, 200)
         self.assertEqual(res_dict['zone']['id'], 1)
-        self.assertEqual(res_dict['zone']['api_url'], 'http://blah.zoo')
+        self.assertEqual(res_dict['zone']['api_url'], 'http://example.com')
         self.assertFalse('username' in res_dict['zone'])
 
     def test_zone_update(self):
@@ -158,7 +157,7 @@ class ZonesTest(unittest.TestCase):
 
         self.assertEqual(res.status_int, 200)
         self.assertEqual(res_dict['zone']['id'], 1)
-        self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com')
+        self.assertEqual(res_dict['zone']['api_url'], 'http://example.com')
         self.assertFalse('username' in res_dict['zone'])
 
 

From 18e573a14414838f11e772edca3eb5510f852c94 Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Fri, 18 Feb 2011 17:45:57 -0400
Subject: [PATCH 18/19] sandy y u no read hacking guide and import classes?

---
 nova/api/openstack/zones.py            | 5 ++---
 nova/scheduler/manager.py              | 4 ++--
 nova/scheduler/zone_manager.py         | 4 ++--
 nova/tests/api/openstack/test_zones.py | 7 ++++---
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py
index 24a4444f7d16..99be0ba0254b 100644
--- a/nova/api/openstack/zones.py
+++ b/nova/api/openstack/zones.py
@@ -19,8 +19,7 @@ import logging
 from nova import flags
 from nova import wsgi
 from nova import db
-from nova import rpc
-from nova.scheduler.api import API
+from nova.scheduler import api
 
 
 FLAGS = flags.FLAGS
@@ -54,7 +53,7 @@ class Controller(wsgi.Controller):
         """Return all zones in brief"""
         # Ask the ZoneManager in the Scheduler for most recent data,
         # or fall-back to the database ...
-        items = API().get_zone_list(req.environ['nova.context'])
+        items = api.API().get_zone_list(req.environ['nova.context'])
         if not items:
             items = db.zone_get_all(req.environ['nova.context'])
 
diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py
index 7ced33b9c175..c94397210ddb 100644
--- a/nova/scheduler/manager.py
+++ b/nova/scheduler/manager.py
@@ -29,7 +29,7 @@ from nova import log as logging
 from nova import manager
 from nova import rpc
 from nova import utils
-from nova.scheduler.zone_manager import ZoneManager
+from nova.scheduler import zone_manager
 
 LOG = logging.getLogger('nova.scheduler.manager')
 FLAGS = flags.FLAGS
@@ -44,7 +44,7 @@ class SchedulerManager(manager.Manager):
         if not scheduler_driver:
             scheduler_driver = FLAGS.scheduler_driver
         self.driver = utils.import_object(scheduler_driver)
-        self.zone_manager = ZoneManager()
+        self.zone_manager = zone_manager.ZoneManager()
         super(SchedulerManager, self).__init__(*args, **kwargs)
 
     def __getattr__(self, key):
diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index 783783d06bde..758c5e3db31e 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -22,7 +22,7 @@ import thread
 import traceback
 
 from datetime import datetime
-from eventlet.greenpool import GreenPool
+from eventlet import greenpool
 
 from nova import db
 from nova import flags
@@ -105,7 +105,7 @@ class ZoneManager(object):
     def __init__(self):
         self.last_zone_db_check = datetime.min
         self.zone_states = {}
-        self.green_pool = GreenPool()
+        self.green_pool = greenpool.GreenPool()
 
     def get_zone_list(self):
         """Return the list of zones we know about."""
diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py
index 4df7c7febfe5..4374cd457bba 100644
--- a/nova/tests/api/openstack/test_zones.py
+++ b/nova/tests/api/openstack/test_zones.py
@@ -24,7 +24,7 @@ from nova import context
 from nova import flags
 from nova.api.openstack import zones
 from nova.tests.api.openstack import fakes
-from nova.scheduler.api import API
+from nova.scheduler import api
 
 
 FLAGS = flags.FLAGS
@@ -97,7 +97,7 @@ class ZonesTest(unittest.TestCase):
         FLAGS.allow_admin_api = self.allow_admin
 
     def test_get_zone_list_scheduler(self):
-        self.stubs.Set(API, '_call_scheduler', zone_get_all_scheduler)
+        self.stubs.Set(api.API, '_call_scheduler', zone_get_all_scheduler)
         req = webob.Request.blank('/v1.0/zones')
         res = req.get_response(fakes.wsgi_app())
         res_dict = json.loads(res.body)
@@ -106,7 +106,8 @@ class ZonesTest(unittest.TestCase):
         self.assertEqual(len(res_dict['zones']), 2)
 
     def test_get_zone_list_db(self):
-        self.stubs.Set(API, '_call_scheduler', zone_get_all_scheduler_empty)
+        self.stubs.Set(api.API, '_call_scheduler',
+                                zone_get_all_scheduler_empty)
         self.stubs.Set(nova.db, 'zone_get_all', zone_get_all_db)
         req = webob.Request.blank('/v1.0/zones')
         res = req.get_response(fakes.wsgi_app())

From 7c6638cc5effc7d727087ed4354c180d75476d1a Mon Sep 17 00:00:00 2001
From: Sandy Walsh <sandy.walsh@rackspace.com>
Date: Fri, 25 Feb 2011 13:40:15 -0800
Subject: [PATCH 19/19] API changed to new style class

---
 nova/scheduler/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py
index 8491bf3a9fc4..2405f1343f64 100644
--- a/nova/scheduler/api.py
+++ b/nova/scheduler/api.py
@@ -25,7 +25,7 @@ FLAGS = flags.FLAGS
 LOG = logging.getLogger('nova.scheduler.api')
 
 
-class API:
+class API(object):
     """API for interacting with the scheduler."""
 
     def _call_scheduler(self, method, context, params=None):