Enable python34 tests for nova/tests/unit/objects/test*.py

All tests in nova/tests/unit/objects/test*.py now run with
python3.4 tox target.

* Fix imports in limits.py and urlmap.py
* Add a list() as keys()/values() return iterator in
  availability_zones.py, flavors.py and instance_numa_topology.py
* contextlib.nested is not present in python3, so whip up an
  alternative using ExitStack(). Directly using ExitStack() wont
  work for us.
* Add a few assertJsonEqual in the test cases
* Ensure fingerprinting generates the same exact value in both
  python27 and python34

Blueprint nova-python3
Change-Id: I848c48475189c4b4ad8151e14509020ae7d110a4

nova/api/openstack/compute/limits.py

@@ -33,13 +33,13 @@ process (each process will have its own rate limiting counter).
 
 import collections
 import copy
-import httplib
 import math
 import re
 import time
 
 from oslo_serialization import jsonutils
 from oslo_utils import importutils
+from six.moves import http_client as httplib
 import webob.dec
 import webob.exc
 

nova/api/openstack/urlmap.py

@@ -14,10 +14,15 @@
 #    under the License.
 
 import re
-import urllib2
 
 from oslo_log import log as logging
 import paste.urlmap
+import six
+
+if six.PY3:
+    from urllib import request as urllib2
+else:
+    import urllib2
 
 from nova.api.openstack import wsgi
 

nova/availability_zones.py

@@ -71,7 +71,7 @@ def _build_metadata_by_host(aggregates, hosts=None):
         for host in aggregate.hosts:
             if hosts and host not in hosts:
                 continue
-            metadata[host].add(aggregate.metadata.values()[0])
+            metadata[host].add(list(aggregate.metadata.values())[0])
     return metadata
 
 

nova/compute/flavors.py

@@ -352,7 +352,7 @@ def delete_flavor_info(metadata, *prefixes):
     # NUMA-related ones that we need to avoid an uglier alternative. This
     # should be replaced by a general split-out of flavor information from
     # system_metadata very soon.
-    for key in metadata.keys():
+    for key in list(metadata.keys()):
         for prefix in prefixes:
             if key.startswith('%sinstance_type_extra_' % prefix):
                 del metadata[key]

nova/objects/instance_numa_topology.py

@@ -91,7 +91,7 @@ class InstanceNUMACell(base.NovaObject,
         if threads == 1:
             threads = 0
 
-        return map(set, zip(*[iter(cpu_list)] * threads))
+        return list(map(set, zip(*[iter(cpu_list)] * threads)))
 
     @property
     def cpu_pinning_requested(self):

nova/test.py

@@ -20,6 +20,8 @@ Allows overriding of flags for use of fakes, and some black magic for
 inline callbacks.
 
 """
+import contextlib
+
 import datetime
 import eventlet
 eventlet.monkey_patch(os=False)
@@ -67,6 +69,14 @@ objects.register_all()
 
 _TRUE_VALUES = ('True', 'true', '1', 'yes')
 
+if six.PY3:
+    @contextlib.contextmanager
+    def nested(*contexts):
+        with contextlib.ExitStack() as stack:
+            yield [stack.enter_context(c) for c in contexts]
+else:
+    nested = contextlib.nested
+
 
 class SampleNetworks(fixtures.Fixture):
 
@@ -285,24 +295,25 @@ class TestCase(testtools.TestCase):
             observed = jsonutils.loads(observed)
 
         def sort(what):
-            return sorted(what,
+            def get_key(item):
-                          key=lambda x: str(x) if isinstance(
+                if isinstance(item, (datetime.datetime, set)):
-                              x, set) or isinstance(x,
+                    return str(item)
-                                                    datetime.datetime) else x)
+                if six.PY3 and isinstance(item, dict):
+                    return str(sort(list(six.iterkeys(item)) +
+                                         list(six.itervalues(item))))
+                return str(item) if six.PY3 else item
+
+            return sorted(what, key=get_key)
 
         def inner(expected, observed):
             if isinstance(expected, dict) and isinstance(observed, dict):
                 self.assertEqual(len(expected), len(observed))
                 expected_keys = sorted(expected)
-                observed_keys = sorted(expected)
+                observed_keys = sorted(observed)
                 self.assertEqual(expected_keys, observed_keys)
 
-                expected_values_iter = iter(sort(expected.values()))
+                for key in list(six.iterkeys(expected)):
-                observed_values_iter = iter(sort(observed.values()))
+                    inner(expected[key], observed[key])
-
-                for i in range(len(expected)):
-                    inner(next(expected_values_iter),
-                          next(observed_values_iter))
             elif (isinstance(expected, (list, tuple, set)) and
                       isinstance(observed, (list, tuple, set))):
                 self.assertEqual(len(expected), len(observed))

nova/tests/unit/objects/test_block_device.py

@@ -12,8 +12,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import contextlib
-
 import mock
 import six
 
@@ -53,7 +51,7 @@ class _TestBlockDeviceMappingObject(object):
             self.flags(enable=False, group='cells')
 
         fake_bdm = self.fake_bdm()
-        with contextlib.nested(
+        with test.nested(
             mock.patch.object(
                 db, 'block_device_mapping_update', return_value=fake_bdm),
             mock.patch.object(
@@ -149,7 +147,7 @@ class _TestBlockDeviceMappingObject(object):
             values['device_name'] = device_name
         fake_bdm = fake_block_device.FakeDbBlockDeviceDict(values)
 
-        with contextlib.nested(
+        with test.nested(
             mock.patch.object(
                     db, 'block_device_mapping_create', return_value=fake_bdm),
             mock.patch.object(
@@ -247,7 +245,7 @@ class _TestBlockDeviceMappingObject(object):
             self.flags(enable=True, cell_type=cell_type, group='cells')
         else:
             self.flags(enable=False, group='cells')
-        with contextlib.nested(
+        with test.nested(
             mock.patch.object(db, 'block_device_mapping_destroy'),
             mock.patch.object(cells_rpcapi.CellsAPI, 'bdm_destroy_at_top')
         ) as (bdm_del, cells_destroy):

nova/tests/unit/objects/test_instance.py

@@ -131,7 +131,7 @@ class _TestInstanceObject(object):
         exp_cols.remove('pci_requests')
         exp_cols.remove('vcpu_model')
         exp_cols.remove('ec2_ids')
-        exp_cols = filter(lambda x: 'flavor' not in x, exp_cols)
+        exp_cols = list(filter(lambda x: 'flavor' not in x, exp_cols))
         exp_cols.extend(['extra', 'extra.numa_topology', 'extra.pci_requests',
                          'extra.flavor', 'extra.vcpu_model'])
 
@@ -473,7 +473,7 @@ class _TestInstanceObject(object):
         actual_args = mock_update.call_args
         self.assertEqual(self.context, actual_args[0][0])
         self.assertEqual(inst.uuid, actual_args[0][1])
-        self.assertEqual(actual_args[0][2].keys(), ['vcpu_model'])
+        self.assertEqual(list(actual_args[0][2].keys()), ['vcpu_model'])
         self.assertJsonEqual(jsonutils.dumps(
                 test_vcpu_model.fake_vcpumodel.obj_to_primitive()),
                              actual_args[0][2]['vcpu_model'])
@@ -1003,9 +1003,9 @@ class _TestInstanceObject(object):
         expected = {}
         for key in unicode_attributes:
             inst[key] = u'\u2603'
-            expected[key] = '?'
+            expected[key] = b'?'
         primitive = inst.obj_to_primitive(target_version='1.6')
-        self.assertEqual(expected, primitive['nova_object.data'])
+        self.assertJsonEqual(expected, primitive['nova_object.data'])
         self.assertEqual('1.6', primitive['nova_object.version'])
 
     def test_compat_pci_devices(self):
@@ -1652,7 +1652,7 @@ class _TestInstanceListObject(object):
         inst_list._context = self.context
         inst_list.objects = insts
         faulty = inst_list.fill_faults()
-        self.assertEqual(faulty, ['uuid1'])
+        self.assertEqual(list(faulty), ['uuid1'])
         self.assertEqual(inst_list[0].fault.message,
                          db_faults['uuid1'][0]['message'])
         self.assertIsNone(inst_list[1].fault)

nova/tests/unit/objects/test_objects.py

@@ -498,8 +498,8 @@ class _TestObject(object):
         error = None
         try:
             base.NovaObject.obj_class_from_name('MyObj', '1.25')
-        except exception.IncompatibleObjectVersion as error:
+        except exception.IncompatibleObjectVersion as ex:
-            pass
+            error = ex
 
         self.assertIsNotNone(error)
         self.assertEqual('1.6', error.kwargs['supported'])
@@ -646,7 +646,7 @@ class _TestObject(object):
         myobj_fields = (['foo', 'bar', 'missing',
                          'readonly', 'rel_object',
                          'rel_objects', 'mutable_default'] +
-                        base_fields)
+                        list(base_fields))
         myobj3_fields = ['new_field']
         self.assertTrue(issubclass(TestSubclassedObject, MyObj))
         self.assertEqual(len(myobj_fields), len(MyObj.fields))
@@ -1192,11 +1192,19 @@ object_relationships = {
 
 
 class TestObjectVersions(test.NoDBTestCase):
+    @staticmethod
+    def _is_method(thing):
+        # NOTE(dims): In Python3, The concept of 'unbound methods' has
+        # been removed from the language. When referencing a method
+        # as a class attribute, you now get a plain function object.
+        # so let's check for both
+        return inspect.isfunction(thing) or inspect.ismethod(thing)
+
     def _find_remotable_method(self, cls, thing, parent_was_remotable=False):
         """Follow a chain of remotable things down to the original function."""
         if isinstance(thing, classmethod):
             return self._find_remotable_method(cls, thing.__get__(None, cls))
-        elif inspect.ismethod(thing) and hasattr(thing, 'remotable'):
+        elif self._is_method(thing) and hasattr(thing, 'remotable'):
             return self._find_remotable_method(cls, thing.original_fn,
                                                parent_was_remotable=True)
         elif parent_was_remotable:
@@ -1210,12 +1218,12 @@ class TestObjectVersions(test.NoDBTestCase):
     def _get_fingerprint(self, obj_name):
         obj_classes = base.NovaObjectRegistry.obj_classes()
         obj_class = obj_classes[obj_name][0]
-        fields = obj_class.fields.items()
+        fields = list(obj_class.fields.items())
         fields.sort()
         methods = []
         for name in dir(obj_class):
             thing = getattr(obj_class, name)
-            if inspect.ismethod(thing) or isinstance(thing, classmethod):
+            if self._is_method(thing) or isinstance(thing, classmethod):
                 method = self._find_remotable_method(obj_class, thing)
                 if method:
                     methods.append((name, inspect.getargspec(method)))
@@ -1231,14 +1239,26 @@ class TestObjectVersions(test.NoDBTestCase):
                                  sorted(obj_class.child_versions.items())))
         else:
             relevant_data = (fields, methods)
-        fingerprint = '%s-%s' % (obj_class.VERSION,
+        relevant_data = repr(relevant_data)
-                                 hashlib.md5(str(relevant_data)).hexdigest())
+        if six.PY3:
+            relevant_data = relevant_data.encode('utf-8')
+        fingerprint = '%s-%s' % (
+        obj_class.VERSION, hashlib.md5(relevant_data).hexdigest())
         return fingerprint
 
+    def test_find_remotable_method(self):
+        class MyObject(object):
+            @base.remotable
+            def my_method(self):
+                return 'Hello World!'
+        thing = self._find_remotable_method(MyObject,
+                                            getattr(MyObject, 'my_method'))
+        self.assertIsNotNone(thing)
+
     def test_versions(self):
         fingerprints = {}
         obj_classes = base.NovaObjectRegistry.obj_classes()
-        for obj_name in obj_classes:
+        for obj_name in sorted(obj_classes, key=lambda x: x[0]):
             fingerprints[obj_name] = self._get_fingerprint(obj_name)
 
         if os.getenv('GENERATE_HASHES'):

tox.ini

@@ -44,7 +44,45 @@ commands =
   find . -type f -name "*.pyc" -delete
   python -m testtools.run \
     nova.tests.unit.db.test_db_api \
-    nova.tests.unit.test_versions
+    nova.tests.unit.test_versions \
+    nova.tests.unit.objects.test_agent \
+    nova.tests.unit.objects.test_aggregate \
+    nova.tests.unit.objects.test_bandwidth_usage \
+    nova.tests.unit.objects.test_block_device \
+    nova.tests.unit.objects.test_cell_mapping \
+    nova.tests.unit.objects.test_compute_node \
+    nova.tests.unit.objects.test_dns_domain \
+    nova.tests.unit.objects.test_ec2 \
+    nova.tests.unit.objects.test_external_event \
+    nova.tests.unit.objects.test_fields \
+    nova.tests.unit.objects.test_fixed_ip \
+    nova.tests.unit.objects.test_flavor \
+    nova.tests.unit.objects.test_floating_ip \
+    nova.tests.unit.objects.test_hv_spec \
+    nova.tests.unit.objects.test_instance \
+    nova.tests.unit.objects.test_instance_action \
+    nova.tests.unit.objects.test_instance_fault \
+    nova.tests.unit.objects.test_instance_group \
+    nova.tests.unit.objects.test_instance_info_cache \
+    nova.tests.unit.objects.test_instance_mapping \
+    nova.tests.unit.objects.test_instance_numa_topology \
+    nova.tests.unit.objects.test_instance_pci_requests \
+    nova.tests.unit.objects.test_keypair \
+    nova.tests.unit.objects.test_migration \
+    nova.tests.unit.objects.test_network \
+    nova.tests.unit.objects.test_network_request \
+    nova.tests.unit.objects.test_numa \
+    nova.tests.unit.objects.test_objects \
+    nova.tests.unit.objects.test_pci_device \
+    nova.tests.unit.objects.test_pci_device_pool \
+    nova.tests.unit.objects.test_quotas \
+    nova.tests.unit.objects.test_security_group \
+    nova.tests.unit.objects.test_security_group_rule \
+    nova.tests.unit.objects.test_service \
+    nova.tests.unit.objects.test_tag \
+    nova.tests.unit.objects.test_vcpu_model \
+    nova.tests.unit.objects.test_virt_cpu_topology \
+    nova.tests.unit.objects.test_virtual_interface
 
 [testenv:functional]
 usedevelop = True