Merge trunk, resolve conflicts, and rename 010_ migrate file to 011_ since another migrate file got into trunk ahead of this...

bin/nova-dhcpbridge

@@ -94,7 +94,7 @@ def init_leases(interface):
     """Get the list of hosts for an interface."""
     ctxt = context.get_admin_context()
     network_ref = db.network_get_by_bridge(ctxt, interface)
-    return linux_net.get_dhcp_hosts(ctxt, network_ref['id'])
+    return linux_net.get_dhcp_leases(ctxt, network_ref['id'])
 
 
 def main():

bin/nova-manage

@@ -276,7 +276,7 @@ def _db_error(caught_exception):
     print caught_exception
     print _("The above error may show that the database has not "
             "been created.\nPlease create a database using "
-            "nova-manage sync db before running this command.")
+            "'nova-manage db sync' before running this command.")
     exit(1)
 
 
@@ -437,6 +437,8 @@ class ProjectCommands(object):
                     "been created.\nPlease create a database by running a "
                     "nova-api server on this host.")
 
+AccountCommands = ProjectCommands
+
 
 class FixedIpCommands(object):
     """Class for managing fixed ip."""
@@ -985,6 +987,7 @@ class ImageCommands(object):
 
 CATEGORIES = [
     ('user', UserCommands),
+    ('account', AccountCommands),
     ('project', ProjectCommands),
     ('role', RoleCommands),
     ('shell', ShellCommands),

nova/flags.py

@@ -356,3 +356,7 @@ DEFINE_string('host', socket.gethostname(),
 
 DEFINE_string('node_availability_zone', 'nova',
               'availability zone of this node')
+
+DEFINE_string('zone_name', 'nova', 'name of this zone')
+DEFINE_string('zone_capabilities', 'kypervisor:xenserver;os:linux',
+              'Key/Value tags which represent capabilities of this zone')

nova/scheduler/api.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2011 Openstack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Handles all requests relating to schedulers.
+"""
+
+from nova import flags
+from nova import log as logging
+from nova import rpc
+
+FLAGS = flags.FLAGS
+LOG = logging.getLogger('nova.scheduler.api')
+
+
+class API(object):
+    """API for interacting with the scheduler."""
+
+    def _call_scheduler(self, method, context, params=None):
+        """Generic handler for RPC calls to the scheduler.
+
+        :param params: Optional dictionary of arguments to be passed to the
+                       scheduler worker
+
+        :retval: Result returned by scheduler worker
+        """
+        if not params:
+            params = {}
+        queue = FLAGS.scheduler_topic
+        kwargs = {'method': method, 'args': params}
+        return rpc.call(context, queue, kwargs)
+
+    def get_zone_list(self, context):
+        items = self._call_scheduler('get_zone_list', context)
+        for item in items:
+            item['api_url'] = item['api_url'].replace('\\/', '/')
+        return items

nova/scheduler/manager.py

@@ -29,6 +29,7 @@ from nova import log as logging
 from nova import manager
 from nova import rpc
 from nova import utils
+from nova.scheduler import zone_manager
 
 LOG = logging.getLogger('nova.scheduler.manager')
 FLAGS = flags.FLAGS
@@ -43,12 +44,21 @@ class SchedulerManager(manager.Manager):
         if not scheduler_driver:
             scheduler_driver = FLAGS.scheduler_driver
         self.driver = utils.import_object(scheduler_driver)
+        self.zone_manager = zone_manager.ZoneManager()
         super(SchedulerManager, self).__init__(*args, **kwargs)
 
     def __getattr__(self, key):
         """Converts all method calls to use the schedule method"""
         return functools.partial(self._schedule, key)
 
+    def periodic_tasks(self, context=None):
+        """Poll child zones periodically to get status."""
+        self.zone_manager.ping(context)
+
+    def get_zone_list(self, context=None):
+        """Get a list of zones from the ZoneManager."""
+        return self.zone_manager.get_zone_list()
+
     def _schedule(self, method, context, topic, *args, **kwargs):
         """Tries to call schedule_* method on the driver to retrieve host.
 

nova/scheduler/zone_manager.py

@@ -0,0 +1,143 @@
+# Copyright (c) 2011 Openstack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+ZoneManager oversees all communications with child Zones.
+"""
+
+import novaclient
+import thread
+import traceback
+
+from datetime import datetime
+from eventlet import greenpool
+
+from nova import db
+from nova import flags
+from nova import log as logging
+
+FLAGS = flags.FLAGS
+flags.DEFINE_integer('zone_db_check_interval', 60,
+                    'Seconds between getting fresh zone info from db.')
+flags.DEFINE_integer('zone_failures_to_offline', 3,
+             'Number of consecutive errors before marking zone offline')
+
+
+class ZoneState(object):
+    """Holds the state of all connected child zones."""
+    def __init__(self):
+        self.is_active = True
+        self.name = None
+        self.capabilities = None
+        self.attempt = 0
+        self.last_seen = datetime.min
+        self.last_exception = None
+        self.last_exception_time = None
+
+    def update_credentials(self, zone):
+        """Update zone credentials from db"""
+        self.zone_id = zone.id
+        self.api_url = zone.api_url
+        self.username = zone.username
+        self.password = zone.password
+
+    def update_metadata(self, zone_metadata):
+        """Update zone metadata after successful communications with
+           child zone."""
+        self.last_seen = datetime.now()
+        self.attempt = 0
+        self.name = zone_metadata["name"]
+        self.capabilities = zone_metadata["capabilities"]
+        self.is_active = True
+
+    def to_dict(self):
+        return dict(name=self.name, capabilities=self.capabilities,
+                    is_active=self.is_active, api_url=self.api_url,
+                    id=self.zone_id)
+
+    def log_error(self, exception):
+        """Something went wrong. Check to see if zone should be
+           marked as offline."""
+        self.last_exception = exception
+        self.last_exception_time = datetime.now()
+        api_url = self.api_url
+        logging.warning(_("'%(exception)s' error talking to "
+                          "zone %(api_url)s") % locals())
+
+        max_errors = FLAGS.zone_failures_to_offline
+        self.attempt += 1
+        if self.attempt >= max_errors:
+            self.is_active = False
+            logging.error(_("No answer from zone %(api_url)s "
+                            "after %(max_errors)d "
+                            "attempts. Marking inactive.") % locals())
+
+
+def _call_novaclient(zone):
+    """Call novaclient. Broken out for testing purposes."""
+    client = novaclient.OpenStack(zone.username, zone.password, zone.api_url)
+    return client.zones.info()._info
+
+
+def _poll_zone(zone):
+    """Eventlet worker to poll a zone."""
+    logging.debug(_("Polling zone: %s") % zone.api_url)
+    try:
+        zone.update_metadata(_call_novaclient(zone))
+    except Exception, e:
+        zone.log_error(traceback.format_exc())
+
+
+class ZoneManager(object):
+    """Keeps the zone states updated."""
+    def __init__(self):
+        self.last_zone_db_check = datetime.min
+        self.zone_states = {}
+        self.green_pool = greenpool.GreenPool()
+
+    def get_zone_list(self):
+        """Return the list of zones we know about."""
+        return [zone.to_dict() for zone in self.zone_states.values()]
+
+    def _refresh_from_db(self, context):
+        """Make our zone state map match the db."""
+        # Add/update existing zones ...
+        zones = db.zone_get_all(context)
+        existing = self.zone_states.keys()
+        db_keys = []
+        for zone in zones:
+            db_keys.append(zone.id)
+            if zone.id not in existing:
+                self.zone_states[zone.id] = ZoneState()
+            self.zone_states[zone.id].update_credentials(zone)
+
+        # Cleanup zones removed from db ...
+        keys = self.zone_states.keys()  # since we're deleting
+        for zone_id in keys:
+            if zone_id not in db_keys:
+                del self.zone_states[zone_id]
+
+    def _poll_zones(self, context):
+        """Try to connect to each child zone and get update."""
+        self.green_pool.imap(_poll_zone, self.zone_states.values())
+
+    def ping(self, context=None):
+        """Ping should be called periodically to update zone status."""
+        diff = datetime.now() - self.last_zone_db_check
+        if diff.seconds >= FLAGS.zone_db_check_interval:
+            logging.debug(_("Updating zone cache from db."))
+            self.last_zone_db_check = datetime.now()
+            self._refresh_from_db(context)
+        self._poll_zones(context)

nova/tests/test_network.py

@@ -20,6 +20,7 @@ Unit Tests for network code
 """
 import IPy
 import os
+import time
 
 from nova import context
 from nova import db
@@ -29,11 +30,153 @@ from nova import log as logging
 from nova import test
 from nova import utils
 from nova.auth import manager
+from nova.network import linux_net
 
 FLAGS = flags.FLAGS
 LOG = logging.getLogger('nova.tests.network')
 
 
+class IptablesManagerTestCase(test.TestCase):
+    sample_filter = ['#Generated by iptables-save on Fri Feb 18 15:17:05 2011',
+                     '*filter',
+                     ':INPUT ACCEPT [2223527:305688874]',
+                     ':FORWARD ACCEPT [0:0]',
+                     ':OUTPUT ACCEPT [2172501:140856656]',
+                     ':nova-compute-FORWARD - [0:0]',
+                     ':nova-compute-INPUT - [0:0]',
+                     ':nova-compute-local - [0:0]',
+                     ':nova-compute-OUTPUT - [0:0]',
+                     ':nova-filter-top - [0:0]',
+                     '-A FORWARD -j nova-filter-top ',
+                     '-A OUTPUT -j nova-filter-top ',
+                     '-A nova-filter-top -j nova-compute-local ',
+                     '-A INPUT -j nova-compute-INPUT ',
+                     '-A OUTPUT -j nova-compute-OUTPUT ',
+                     '-A FORWARD -j nova-compute-FORWARD ',
+                     '-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT ',
+                     '-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT ',
+                     '-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT ',
+                     '-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT ',
+                     '-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT ',
+                     '-A FORWARD -i virbr0 -o virbr0 -j ACCEPT ',
+                     '-A FORWARD -o virbr0 -j REJECT --reject-with '
+                     'icmp-port-unreachable ',
+                     '-A FORWARD -i virbr0 -j REJECT --reject-with '
+                     'icmp-port-unreachable ',
+                     'COMMIT',
+                     '# Completed on Fri Feb 18 15:17:05 2011']
+
+    sample_nat = ['# Generated by iptables-save on Fri Feb 18 15:17:05 2011',
+                  '*nat',
+                  ':PREROUTING ACCEPT [3936:762355]',
+                  ':INPUT ACCEPT [2447:225266]',
+                  ':OUTPUT ACCEPT [63491:4191863]',
+                  ':POSTROUTING ACCEPT [63112:4108641]',
+                  ':nova-compute-OUTPUT - [0:0]',
+                  ':nova-compute-floating-ip-snat - [0:0]',
+                  ':nova-compute-SNATTING - [0:0]',
+                  ':nova-compute-PREROUTING - [0:0]',
+                  ':nova-compute-POSTROUTING - [0:0]',
+                  ':nova-postrouting-bottom - [0:0]',
+                  '-A PREROUTING -j nova-compute-PREROUTING ',
+                  '-A OUTPUT -j nova-compute-OUTPUT ',
+                  '-A POSTROUTING -j nova-compute-POSTROUTING ',
+                  '-A POSTROUTING -j nova-postrouting-bottom ',
+                  '-A nova-postrouting-bottom -j nova-compute-SNATTING ',
+                  '-A nova-compute-SNATTING -j nova-compute-floating-ip-snat ',
+                  'COMMIT',
+                  '# Completed on Fri Feb 18 15:17:05 2011']
+
+    def setUp(self):
+        super(IptablesManagerTestCase, self).setUp()
+        self.manager = linux_net.IptablesManager()
+
+    def test_filter_rules_are_wrapped(self):
+        current_lines = self.sample_filter
+
+        table = self.manager.ipv4['filter']
+        table.add_rule('FORWARD', '-s 1.2.3.4/5 -j DROP')
+        new_lines = self.manager._modify_rules(current_lines, table)
+        self.assertTrue('-A run_tests.py-FORWARD '
+                        '-s 1.2.3.4/5 -j DROP' in new_lines)
+
+        table.remove_rule('FORWARD', '-s 1.2.3.4/5 -j DROP')
+        new_lines = self.manager._modify_rules(current_lines, table)
+        self.assertTrue('-A run_tests.py-FORWARD '
+                        '-s 1.2.3.4/5 -j DROP' not in new_lines)
+
+    def test_nat_rules(self):
+        current_lines = self.sample_nat
+        new_lines = self.manager._modify_rules(current_lines,
+                                               self.manager.ipv4['nat'])
+
+        for line in [':nova-compute-OUTPUT - [0:0]',
+                     ':nova-compute-floating-ip-snat - [0:0]',
+                     ':nova-compute-SNATTING - [0:0]',
+                     ':nova-compute-PREROUTING - [0:0]',
+                     ':nova-compute-POSTROUTING - [0:0]']:
+            self.assertTrue(line in new_lines, "One of nova-compute's chains "
+                                               "went missing.")
+
+        seen_lines = set()
+        for line in new_lines:
+            line = line.strip()
+            self.assertTrue(line not in seen_lines,
+                            "Duplicate line: %s" % line)
+            seen_lines.add(line)
+
+        last_postrouting_line = ''
+
+        for line in new_lines:
+            if line.startswith('-A POSTROUTING'):
+                last_postrouting_line = line
+
+        self.assertTrue('-j nova-postrouting-bottom' in last_postrouting_line,
+                        "Last POSTROUTING rule does not jump to "
+                        "nova-postouting-bottom: %s" % last_postrouting_line)
+
+        for chain in ['POSTROUTING', 'PREROUTING', 'OUTPUT']:
+            self.assertTrue('-A %s -j run_tests.py-%s' \
+                            % (chain, chain) in new_lines,
+                            "Built-in chain %s not wrapped" % (chain,))
+
+    def test_filter_rules(self):
+        current_lines = self.sample_filter
+        new_lines = self.manager._modify_rules(current_lines,
+                                               self.manager.ipv4['filter'])
+
+        for line in [':nova-compute-FORWARD - [0:0]',
+                     ':nova-compute-INPUT - [0:0]',
+                     ':nova-compute-local - [0:0]',
+                     ':nova-compute-OUTPUT - [0:0]']:
+            self.assertTrue(line in new_lines, "One of nova-compute's chains"
+                                               " went missing.")
+
+        seen_lines = set()
+        for line in new_lines:
+            line = line.strip()
+            self.assertTrue(line not in seen_lines,
+                            "Duplicate line: %s" % line)
+            seen_lines.add(line)
+
+        for chain in ['FORWARD', 'OUTPUT']:
+            for line in new_lines:
+                if line.startswith('-A %s' % chain):
+                    self.assertTrue('-j nova-filter-top' in line,
+                                    "First %s rule does not "
+                                    "jump to nova-filter-top" % chain)
+                    break
+
+        self.assertTrue('-A nova-filter-top '
+                        '-j run_tests.py-local' in new_lines,
+                        "nova-filter-top does not jump to wrapped local chain")
+
+        for chain in ['INPUT', 'OUTPUT', 'FORWARD']:
+            self.assertTrue('-A %s -j run_tests.py-%s' \
+                            % (chain, chain) in new_lines,
+                            "Built-in chain %s not wrapped" % (chain,))
+
+
 class NetworkTestCase(test.TestCase):
     """Test cases for network code"""
     def setUp(self):
@@ -321,6 +464,31 @@ class NetworkTestCase(test.TestCase):
                                                   network['id'])
         self.assertEqual(ip_count, num_available_ips)
 
+    def test_dhcp_lease_output(self):
+        admin_ctxt = context.get_admin_context()
+        address = self._create_address(0, self.instance_id)
+        lease_ip(address)
+        network_ref = db.network_get_by_instance(admin_ctxt, self.instance_id)
+        leases = linux_net.get_dhcp_leases(context.get_admin_context(),
+                                           network_ref['id'])
+        for line in leases.split('\n'):
+            seconds, mac, ip, hostname, client_id = line.split(' ')
+            self.assertTrue(int(seconds) > time.time(), 'Lease expires in '
+                                                        'the past')
+            octets = mac.split(':')
+            self.assertEqual(len(octets), 6, "Wrong number of octets "
+                                             "in %s" % (max,))
+            for octet in octets:
+                self.assertEqual(len(octet), 2, "Oddly sized octet: %s"
+                                                                    % (octet,))
+                # This will throw an exception if the octet is invalid
+                int(octet, 16)
+
+            # And this will raise an exception in case of an invalid IP
+            IPy.IP(ip)
+
+        release_ip(address)
+
 
 def is_allocated_in_project(address, project_id):
     """Returns true if address is in specified project"""

nova/tests/test_virt.py

@@ -17,6 +17,7 @@
 import eventlet
 import mox
 import os
+import re
 import sys
 
 from xml.etree.ElementTree import fromstring as xml_to_tree
@@ -521,16 +522,22 @@ class IptablesFirewallTestCase(test.TestCase):
         self.manager.delete_user(self.user)
         super(IptablesFirewallTestCase, self).tearDown()
 
-    in_rules = [
+    in_nat_rules = [
+      '# Generated by iptables-save v1.4.10 on Sat Feb 19 00:03:19 2011',
+      '*nat',
+      ':PREROUTING ACCEPT [1170:189210]',
+      ':INPUT ACCEPT [844:71028]',
+      ':OUTPUT ACCEPT [5149:405186]',
+      ':POSTROUTING ACCEPT [5063:386098]',
+    ]
+
+    in_filter_rules = [
       '# Generated by iptables-save v1.4.4 on Mon Dec  6 11:54:13 2010',
       '*filter',
       ':INPUT ACCEPT [969615:281627771]',
       ':FORWARD ACCEPT [0:0]',
       ':OUTPUT ACCEPT [915599:63811649]',
       ':nova-block-ipv4 - [0:0]',
-      '-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT ',
-      '-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT ',
-      '-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT ',
       '-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT ',
       '-A FORWARD -d 192.168.122.0/24 -o virbr0 -m state --state RELATED'
       ',ESTABLISHED -j ACCEPT ',
@@ -542,7 +549,7 @@ class IptablesFirewallTestCase(test.TestCase):
       '# Completed on Mon Dec  6 11:54:13 2010',
     ]
 
-    in6_rules = [
+    in6_filter_rules = [
       '# Generated by ip6tables-save v1.4.4 on Tue Jan 18 23:47:56 2011',
       '*filter',
       ':INPUT ACCEPT [349155:75810423]',
@@ -605,21 +612,31 @@ class IptablesFirewallTestCase(test.TestCase):
         def fake_iptables_execute(*cmd, **kwargs):
             process_input = kwargs.get('process_input', None)
             if cmd == ('sudo', 'ip6tables-save', '-t', 'filter'):
-                return '\n'.join(self.in6_rules), None
+                return '\n'.join(self.in6_filter_rules), None
             if cmd == ('sudo', 'iptables-save', '-t', 'filter'):
-                return '\n'.join(self.in_rules), None
+                return '\n'.join(self.in_filter_rules), None
+            if cmd == ('sudo', 'iptables-save', '-t', 'nat'):
+                return '\n'.join(self.in_nat_rules), None
             if cmd == ('sudo', 'iptables-restore'):
-                self.out_rules = process_input.split('\n')
+                lines = process_input.split('\n')
+                if '*filter' in lines:
+                    self.out_rules = lines
                 return '', ''
             if cmd == ('sudo', 'ip6tables-restore'):
-                self.out6_rules = process_input.split('\n')
+                lines = process_input.split('\n')
+                if '*filter' in lines:
+                    self.out6_rules = lines
                 return '', ''
-        self.fw.execute = fake_iptables_execute
+            print cmd, kwargs
+
+        from nova.network import linux_net
+        linux_net.iptables_manager.execute = fake_iptables_execute
 
         self.fw.prepare_instance_filter(instance_ref)
         self.fw.apply_instance_filter(instance_ref)
 
-        in_rules = filter(lambda l: not l.startswith('#'), self.in_rules)
+        in_rules = filter(lambda l: not l.startswith('#'),
+                          self.in_filter_rules)
         for rule in in_rules:
             if not 'nova' in rule:
                 self.assertTrue(rule in self.out_rules,
@@ -642,17 +659,18 @@ class IptablesFirewallTestCase(test.TestCase):
         self.assertTrue(security_group_chain,
                         "The security group chain wasn't added")
 
-        self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -j ACCEPT' % \
+        regex = re.compile('-A .* -p icmp -s 192.168.11.0/24 -j ACCEPT')
-                               security_group_chain in self.out_rules,
+        self.assertTrue(len(filter(regex.match, self.out_rules)) > 0,
                         "ICMP acceptance rule wasn't added")
 
-        self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -m icmp --icmp-type '
+        regex = re.compile('-A .* -p icmp -s 192.168.11.0/24 -m icmp '
-                        '8 -j ACCEPT' % security_group_chain in self.out_rules,
+                           '--icmp-type 8 -j ACCEPT')
+        self.assertTrue(len(filter(regex.match, self.out_rules)) > 0,
                         "ICMP Echo Request acceptance rule wasn't added")
 
-        self.assertTrue('-A %s -p tcp -s 192.168.10.0/24 -m multiport '
+        regex = re.compile('-A .* -p tcp -s 192.168.10.0/24 -m multiport '
-                        '--dports 80:81 -j ACCEPT' % security_group_chain \
+                           '--dports 80:81 -j ACCEPT')
-                            in self.out_rules,
+        self.assertTrue(len(filter(regex.match, self.out_rules)) > 0,
                         "TCP port 80/81 acceptance rule wasn't added")
         db.instance_destroy(admin_ctxt, instance_ref['id'])
 

nova/tests/test_xenapi.py

@@ -18,6 +18,7 @@
 Test suite for XenAPI
 """
 
+import functools
 import stubout
 
 from nova import db
@@ -41,6 +42,21 @@ from nova.tests.glance import stubs as glance_stubs
 FLAGS = flags.FLAGS
 
 
+def stub_vm_utils_with_vdi_attached_here(function, should_return=True):
+    """
+    vm_utils.with_vdi_attached_here needs to be stubbed out because it
+    calls down to the filesystem to attach a vdi. This provides a
+    decorator to handle that.
+    """
+    @functools.wraps(function)
+    def decorated_function(self, *args, **kwargs):
+        orig_with_vdi_attached_here = vm_utils.with_vdi_attached_here
+        vm_utils.with_vdi_attached_here = lambda *x: should_return
+        function(self, *args, **kwargs)
+        vm_utils.with_vdi_attached_here = orig_with_vdi_attached_here
+    return decorated_function
+
+
 class XenAPIVolumeTestCase(test.TestCase):
     """
     Unit tests for Volume operations
@@ -62,6 +78,7 @@ class XenAPIVolumeTestCase(test.TestCase):
                   'ramdisk_id': 3,
                   'instance_type': 'm1.large',
                   'mac_address': 'aa:bb:cc:dd:ee:ff',
+                  'os_type': 'linux'
                   }
 
     def _create_volume(self, size='0'):
@@ -219,7 +236,7 @@ class XenAPIVMTestCase(test.TestCase):
 
         check()
 
-    def check_vm_record(self, conn):
+    def create_vm_record(self, conn, os_type):
         instances = conn.list_instances()
         self.assertEquals(instances, [1])
 
@@ -231,28 +248,63 @@ class XenAPIVMTestCase(test.TestCase):
                in xenapi_fake.get_all_records('VM').iteritems()
                if not rec['is_control_domain']]
         vm = vms[0]
+        self.vm_info = vm_info
+        self.vm = vm
 
+    def check_vm_record(self, conn):
         # Check that m1.large above turned into the right thing.
         instance_type = db.instance_type_get_by_name(conn, 'm1.large')
         mem_kib = long(instance_type['memory_mb']) << 10
         mem_bytes = str(mem_kib << 10)
         vcpus = instance_type['vcpus']
-        self.assertEquals(vm_info['max_mem'], mem_kib)
+        self.assertEquals(self.vm_info['max_mem'], mem_kib)
-        self.assertEquals(vm_info['mem'], mem_kib)
+        self.assertEquals(self.vm_info['mem'], mem_kib)
-        self.assertEquals(vm['memory_static_max'], mem_bytes)
+        self.assertEquals(self.vm['memory_static_max'], mem_bytes)
-        self.assertEquals(vm['memory_dynamic_max'], mem_bytes)
+        self.assertEquals(self.vm['memory_dynamic_max'], mem_bytes)
-        self.assertEquals(vm['memory_dynamic_min'], mem_bytes)
+        self.assertEquals(self.vm['memory_dynamic_min'], mem_bytes)
-        self.assertEquals(vm['VCPUs_max'], str(vcpus))
+        self.assertEquals(self.vm['VCPUs_max'], str(vcpus))
-        self.assertEquals(vm['VCPUs_at_startup'], str(vcpus))
+        self.assertEquals(self.vm['VCPUs_at_startup'], str(vcpus))
 
         # Check that the VM is running according to Nova
-        self.assertEquals(vm_info['state'], power_state.RUNNING)
+        self.assertEquals(self.vm_info['state'], power_state.RUNNING)
 
         # Check that the VM is running according to XenAPI.
-        self.assertEquals(vm['power_state'], 'Running')
+        self.assertEquals(self.vm['power_state'], 'Running')
+
+    def check_vm_params_for_windows(self):
+        self.assertEquals(self.vm['platform']['nx'], 'true')
+        self.assertEquals(self.vm['HVM_boot_params'], {'order': 'dc'})
+        self.assertEquals(self.vm['HVM_boot_policy'], 'BIOS order')
+
+        # check that these are not set
+        self.assertEquals(self.vm['PV_args'], '')
+        self.assertEquals(self.vm['PV_bootloader'], '')
+        self.assertEquals(self.vm['PV_kernel'], '')
+        self.assertEquals(self.vm['PV_ramdisk'], '')
+
+    def check_vm_params_for_linux(self):
+        self.assertEquals(self.vm['platform']['nx'], 'false')
+        self.assertEquals(self.vm['PV_args'], 'clocksource=jiffies')
+        self.assertEquals(self.vm['PV_bootloader'], 'pygrub')
+
+        # check that these are not set
+        self.assertEquals(self.vm['PV_kernel'], '')
+        self.assertEquals(self.vm['PV_ramdisk'], '')
+        self.assertEquals(self.vm['HVM_boot_params'], {})
+        self.assertEquals(self.vm['HVM_boot_policy'], '')
+
+    def check_vm_params_for_linux_with_external_kernel(self):
+        self.assertEquals(self.vm['platform']['nx'], 'false')
+        self.assertEquals(self.vm['PV_args'], 'root=/dev/xvda1')
+        self.assertNotEquals(self.vm['PV_kernel'], '')
+        self.assertNotEquals(self.vm['PV_ramdisk'], '')
+
+        # check that these are not set
+        self.assertEquals(self.vm['HVM_boot_params'], {})
+        self.assertEquals(self.vm['HVM_boot_policy'], '')
 
     def _test_spawn(self, image_id, kernel_id, ramdisk_id,
-                    instance_type="m1.large"):
+                    instance_type="m1.large", os_type="linux"):
         stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests)
         values = {'name': 1,
                   'id': 1,
@@ -263,10 +315,12 @@ class XenAPIVMTestCase(test.TestCase):
                   'ramdisk_id': ramdisk_id,
                   'instance_type': instance_type,
                   'mac_address': 'aa:bb:cc:dd:ee:ff',
+                  'os_type': os_type
                   }
         conn = xenapi_conn.get_connection(False)
         instance = db.instance_create(values)
         conn.spawn(instance)
+        self.create_vm_record(conn, os_type)
         self.check_vm_record(conn)
 
     def test_spawn_not_enough_memory(self):
@@ -283,24 +337,37 @@ class XenAPIVMTestCase(test.TestCase):
         FLAGS.xenapi_image_service = 'objectstore'
         self._test_spawn(1, 2, 3)
 
+    @stub_vm_utils_with_vdi_attached_here
     def test_spawn_raw_glance(self):
         FLAGS.xenapi_image_service = 'glance'
         self._test_spawn(glance_stubs.FakeGlance.IMAGE_RAW, None, None)
+        self.check_vm_params_for_linux()
 
-    def test_spawn_vhd_glance(self):
+    def test_spawn_vhd_glance_linux(self):
         FLAGS.xenapi_image_service = 'glance'
-        self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None)
+        self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None,
+                         os_type="linux")
+        self.check_vm_params_for_linux()
+
+    def test_spawn_vhd_glance_windows(self):
+        FLAGS.xenapi_image_service = 'glance'
+        self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None,
+                         os_type="windows")
+        self.check_vm_params_for_windows()
 
     def test_spawn_glance(self):
         FLAGS.xenapi_image_service = 'glance'
         self._test_spawn(glance_stubs.FakeGlance.IMAGE_MACHINE,
                          glance_stubs.FakeGlance.IMAGE_KERNEL,
                          glance_stubs.FakeGlance.IMAGE_RAMDISK)
+        self.check_vm_params_for_linux_with_external_kernel()
 
     def tearDown(self):
         super(XenAPIVMTestCase, self).tearDown()
         self.manager.delete_project(self.project)
         self.manager.delete_user(self.user)
+        self.vm_info = None
+        self.vm = None
         self.stubs.UnsetAll()
 
     def _create_instance(self):
@@ -314,7 +381,8 @@ class XenAPIVMTestCase(test.TestCase):
             'kernel_id': 2,
             'ramdisk_id': 3,
             'instance_type': 'm1.large',
-            'mac_address': 'aa:bb:cc:dd:ee:ff'}
+            'mac_address': 'aa:bb:cc:dd:ee:ff',
+            'os_type': 'linux'}
         instance = db.instance_create(values)
         self.conn.spawn(instance)
         return instance
@@ -372,6 +440,7 @@ class XenAPIMigrateInstance(test.TestCase):
                   'ramdisk_id': None,
                   'instance_type': 'm1.large',
                   'mac_address': 'aa:bb:cc:dd:ee:ff',
+                  'os_type': 'linux'
                   }
         stubs.stub_out_migration_methods(self.stubs)
         glance_stubs.stubout_glance_client(self.stubs,
@@ -410,6 +479,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase):
 
         self.fake_instance = FakeInstance()
         self.fake_instance.id = 42
+        self.fake_instance.os_type = 'linux'
 
     def assert_disk_type(self, disk_type):
         dt = vm_utils.VMHelper.determine_disk_image_type(

nova/tests/test_zones.py

@@ -0,0 +1,172 @@
+# Copyright 2010 United States Government as represented by the
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Tests For ZoneManager
+"""
+
+import datetime
+import mox
+import novaclient
+
+from nova import context
+from nova import db
+from nova import flags
+from nova import service
+from nova import test
+from nova import rpc
+from nova import utils
+from nova.auth import manager as auth_manager
+from nova.scheduler import zone_manager
+
+FLAGS = flags.FLAGS
+
+
+class FakeZone:
+    """Represents a fake zone from the db"""
+    def __init__(self, *args, **kwargs):
+        for k, v in kwargs.iteritems():
+            setattr(self, k, v)
+
+
+def exploding_novaclient(zone):
+    """Used when we want to simulate a novaclient call failing."""
+    raise Exception("kaboom")
+
+
+class ZoneManagerTestCase(test.TestCase):
+    """Test case for zone manager"""
+    def test_ping(self):
+        zm = zone_manager.ZoneManager()
+        self.mox.StubOutWithMock(zm, '_refresh_from_db')
+        self.mox.StubOutWithMock(zm, '_poll_zones')
+        zm._refresh_from_db(mox.IgnoreArg())
+        zm._poll_zones(mox.IgnoreArg())
+
+        self.mox.ReplayAll()
+        zm.ping(None)
+        self.mox.VerifyAll()
+
+    def test_refresh_from_db_new(self):
+        zm = zone_manager.ZoneManager()
+
+        self.mox.StubOutWithMock(db, 'zone_get_all')
+        db.zone_get_all(mox.IgnoreArg()).AndReturn([
+               FakeZone(id=1, api_url='http://foo.com', username='user1',
+                    password='pass1'),
+            ])
+
+        self.assertEquals(len(zm.zone_states), 0)
+
+        self.mox.ReplayAll()
+        zm._refresh_from_db(None)
+        self.mox.VerifyAll()
+
+        self.assertEquals(len(zm.zone_states), 1)
+        self.assertEquals(zm.zone_states[1].username, 'user1')
+
+    def test_refresh_from_db_replace_existing(self):
+        zm = zone_manager.ZoneManager()
+        zone_state = zone_manager.ZoneState()
+        zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com',
+                        username='user1', password='pass1'))
+        zm.zone_states[1] = zone_state
+
+        self.mox.StubOutWithMock(db, 'zone_get_all')
+        db.zone_get_all(mox.IgnoreArg()).AndReturn([
+               FakeZone(id=1, api_url='http://foo.com', username='user2',
+                    password='pass2'),
+            ])
+
+        self.assertEquals(len(zm.zone_states), 1)
+
+        self.mox.ReplayAll()
+        zm._refresh_from_db(None)
+        self.mox.VerifyAll()
+
+        self.assertEquals(len(zm.zone_states), 1)
+        self.assertEquals(zm.zone_states[1].username, 'user2')
+
+    def test_refresh_from_db_missing(self):
+        zm = zone_manager.ZoneManager()
+        zone_state = zone_manager.ZoneState()
+        zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com',
+                        username='user1', password='pass1'))
+        zm.zone_states[1] = zone_state
+
+        self.mox.StubOutWithMock(db, 'zone_get_all')
+        db.zone_get_all(mox.IgnoreArg()).AndReturn([])
+
+        self.assertEquals(len(zm.zone_states), 1)
+
+        self.mox.ReplayAll()
+        zm._refresh_from_db(None)
+        self.mox.VerifyAll()
+
+        self.assertEquals(len(zm.zone_states), 0)
+
+    def test_refresh_from_db_add_and_delete(self):
+        zm = zone_manager.ZoneManager()
+        zone_state = zone_manager.ZoneState()
+        zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com',
+                        username='user1', password='pass1'))
+        zm.zone_states[1] = zone_state
+
+        self.mox.StubOutWithMock(db, 'zone_get_all')
+
+        db.zone_get_all(mox.IgnoreArg()).AndReturn([
+               FakeZone(id=2, api_url='http://foo.com', username='user2',
+                    password='pass2'),
+            ])
+        self.assertEquals(len(zm.zone_states), 1)
+
+        self.mox.ReplayAll()
+        zm._refresh_from_db(None)
+        self.mox.VerifyAll()
+
+        self.assertEquals(len(zm.zone_states), 1)
+        self.assertEquals(zm.zone_states[2].username, 'user2')
+
+    def test_poll_zone(self):
+        self.mox.StubOutWithMock(zone_manager, '_call_novaclient')
+        zone_manager._call_novaclient(mox.IgnoreArg()).AndReturn(
+                        dict(name='zohan', capabilities='hairdresser'))
+
+        zone_state = zone_manager.ZoneState()
+        zone_state.update_credentials(FakeZone(id=2,
+                       api_url='http://foo.com', username='user2',
+                       password='pass2'))
+        zone_state.attempt = 1
+
+        self.mox.ReplayAll()
+        zone_manager._poll_zone(zone_state)
+        self.mox.VerifyAll()
+        self.assertEquals(zone_state.attempt, 0)
+        self.assertEquals(zone_state.name, 'zohan')
+
+    def test_poll_zone_fails(self):
+        self.stubs.Set(zone_manager, "_call_novaclient", exploding_novaclient)
+
+        zone_state = zone_manager.ZoneState()
+        zone_state.update_credentials(FakeZone(id=2,
+                       api_url='http://foo.com', username='user2',
+                       password='pass2'))
+        zone_state.attempt = FLAGS.zone_failures_to_offline - 1
+
+        self.mox.ReplayAll()
+        zone_manager._poll_zone(zone_state)
+        self.mox.VerifyAll()
+        self.assertEquals(zone_state.attempt, 3)
+        self.assertFalse(zone_state.is_active)
+        self.assertEquals(zone_state.name, None)

nova/utils.py

@@ -139,8 +139,12 @@ def execute(*cmd, **kwargs):
     stdin = kwargs.get('stdin', subprocess.PIPE)
     stdout = kwargs.get('stdout', subprocess.PIPE)
     stderr = kwargs.get('stderr', subprocess.PIPE)
+    attempts = kwargs.get('attempts', 1)
     cmd = map(str, cmd)
 
+    while attempts > 0:
+        attempts -= 1
+        try:
             LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd))
             env = os.environ.copy()
             if addl_env:
@@ -162,11 +166,17 @@ def execute(*cmd, **kwargs):
                                                 stdout=stdout,
                                                 stderr=stderr,
                                                 cmd=' '.join(cmd))
-    # NOTE(termie): this appears to be necessary to let the subprocess call
+            # NOTE(termie): this appears to be necessary to let the subprocess
-    #               clean something up in between calls, without it two
+            #               call clean something up in between calls, without
-    #               execute calls in a row hangs the second one
+            #               it two execute calls in a row hangs the second one
             greenthread.sleep(0)
             return result
+        except ProcessExecutionError:
+            if not attempts:
+                raise
+            else:
+                LOG.debug(_("%r failed. Retrying."), cmd)
+                greenthread.sleep(random.randint(20, 200) / 100.0)
 
 
 def ssh_execute(ssh, cmd, process_input=None,

nova/wsgi.py

@@ -436,7 +436,8 @@ class Serializer(object):
         try:
             return handlers[content_type]
         except Exception:
-            raise exception.InvalidContentType()
+            raise exception.InvalidContentType(_("Invalid content type %s"
+                                                 % content_type))
 
     def _from_json(self, datastring):
         return utils.loads(datastring)