Properly munch for resource sub-dicts

In the shade layer, we expect object notation to work for sub-dicts.
When we're using underlying resource objects and translating them to
munch then putting them through normalize (Which is temporary during
transition) we're losing the munchified sub-dicts.

Update to_dict in openstack/resource to be able to provide munches
instead of dicts so that the recursive transform is complete.

Add a test for server that makes sure we're getting what we need.

A followup patch that should come that sets original_names to false in
the to_munch call, which will need an update to the normalize function
to deal with new incoming name.

Change-Id: I3df806fe0db7ddf8d93546d64780fc979f38e78f

doc/source/user/model.rst

@@ -226,6 +226,13 @@ A Server from Nova
     launched_at=str() or None,
     terminated_at=str() or None,
     task_state=str() or None,
+    block_device_mapping=dict() or None,
+    instance_name=str() or None,
+    hypervisor_name=str() or None,
+    tags=list(),
+    personality=str() or None,
+    scheduler_hints=str() or None,
+    user_data=str() or None,
     properties=dict())
 
 ComputeLimits

openstack/cloud/_normalize.py

@@ -41,12 +41,14 @@ _SERVER_FIELDS = (
     'key_name',
     'metadata',
     'networks',
+    'personality',
     'private_v4',
     'public_v4',
     'public_v6',
     'status',
     'updated',
     'user_id',
+    'tags',
 )
 
 _KEYPAIR_FIELDS = (
@@ -461,18 +463,28 @@ class Normalizer(object):
 
         server['flavor'].pop('links', None)
         ret['flavor'] = server.pop('flavor')
+        # From original_names from sdk
+        server.pop('flavorRef', None)
 
         # OpenStack can return image as a string when you've booted
         # from volume
         if str(server['image']) != server['image']:
             server['image'].pop('links', None)
         ret['image'] = server.pop('image')
+        # From original_names from sdk
+        server.pop('imageRef', None)
+        # From original_names from sdk
+        ret['block_device_mapping'] = server.pop('block_device_mapping_v2', {})
 
         project_id = server.pop('tenant_id', '')
         project_id = server.pop('project_id', project_id)
 
         az = _pop_or_get(
             server, 'OS-EXT-AZ:availability_zone', None, self.strict_mode)
+        # the server resource has this already, but it's missing az info
+        # from the resource.
+        # TODO(mordred) Fix server resource to set az in the location
+        server.pop('location', None)
         ret['location'] = self._get_current_location(
             project_id=project_id, zone=az)
 
@@ -498,7 +510,12 @@ class Normalizer(object):
                 'OS-EXT-STS:task_state',
                 'OS-EXT-STS:vm_state',
                 'OS-SRV-USG:launched_at',
-                'OS-SRV-USG:terminated_at'):
+                'OS-SRV-USG:terminated_at',
+                'OS-EXT-SRV-ATTR:hypervisor_hostname',
+                'OS-EXT-SRV-ATTR:instance_name',
+                'OS-EXT-SRV-ATTR:user_data',
+                'OS-SCH-HNT:scheduler_hints',
+        ):
             short_key = key.split(':')[1]
             ret[short_key] = _pop_or_get(server, key, None, self.strict_mode)
 

openstack/cloud/openstackcloud.py

@@ -2141,7 +2141,10 @@ class _OpenStackCloudMixin(_normalize.Normalizer):
                       filters=None):
         filters = filters or {}
         servers = [
-            self._normalize_server(server.to_dict())
+            # TODO(mordred) Add original_names=False here and update the
+            # normalize file for server. Then, just remove the normalize call
+            # and the to_munch call.
+            self._normalize_server(server._to_munch())
             for server in self.compute.servers(
                 all_projects=all_projects, **filters)]
         return [

openstack/resource.py

@@ -600,8 +600,7 @@ class Resource(dict):
         # TODO(mordred) We should make a Location Resource and add it here
         # instead of just the dict.
         if self._connection:
-            computed['location'] = munch.unmunchify(
+            computed['location'] = self._connection.current_location
-                self._connection._openstackcloud.current_location)
 
         return body, header, uri, computed
 
@@ -786,7 +785,7 @@ class Resource(dict):
         return cls(_synchronized=synchronized, connection=connection, **obj)
 
     def to_dict(self, body=True, headers=True, computed=True,
-                ignore_none=False, original_names=False):
+                ignore_none=False, original_names=False, _to_munch=False):
         """Return a dictionary of this resource's contents
 
         :param bool body: Include the :class:`~openstack.resource.Body`
@@ -800,11 +799,16 @@ class Resource(dict):
                                  attributes that the server hasn't returned.
         :param bool original_names: When True, use attribute names as they
                                     were received from the server.
+        :param bool _to_munch: For internal use only. Converts to `munch.Munch`
+                               instead of dict.
 
         :return: A dictionary of key/value pairs where keys are named
                  as they exist as attributes of this class.
         """
-        mapping = {}
+        if _to_munch:
+            mapping = munch.Munch()
+        else:
+            mapping = {}
 
         components = []
         if body:
@@ -840,12 +844,17 @@ class Resource(dict):
                         if ignore_none and value is None:
                             continue
                         if isinstance(value, Resource):
-                            mapping[key] = value.to_dict()
+                            mapping[key] = value.to_dict(_to_munch=_to_munch)
+                        elif isinstance(value, dict) and _to_munch:
+                            mapping[key] = munch.Munch(value)
                         elif value and isinstance(value, list):
                             converted = []
                             for raw in value:
                                 if isinstance(raw, Resource):
-                                    converted.append(raw.to_dict())
+                                    converted.append(
+                                        raw.to_dict(_to_munch=_to_munch))
+                                elif isinstance(raw, dict) and _to_munch:
+                                    converted.append(munch.Munch(raw))
                                 else:
                                     converted.append(raw)
                             mapping[key] = converted
@@ -858,10 +867,11 @@ class Resource(dict):
     # Make the munch copy method use to_dict
     copy = to_dict
 
-    def _to_munch(self):
+    def _to_munch(self, original_names=True):
         """Convert this resource into a Munch compatible with shade."""
-        return munch.Munch(self.to_dict(body=True, headers=False,
+        return self.to_dict(
-                                        original_names=True))
+            body=True, headers=False,
+            original_names=original_names, _to_munch=True)
 
     def _prepare_request_body(self, patch, prepend_key):
         if patch:

openstack/tests/unit/cloud/test_normalize.py

@@ -13,6 +13,7 @@
 import mock
 import fixtures
 
+from openstack.compute.v2 import server as server_resource
 from openstack.tests.unit import base
 
 RAW_SERVER_DICT = {
@@ -557,8 +558,18 @@ class TestUtils(base.TestCase):
         self.assertEqual(sorted(expected.keys()), sorted(retval.keys()))
         self.assertEqual(expected, retval)
 
+    def _assert_server_munch_attributes(self, raw, server):
+        self.assertEqual(server.flavor.id, raw['flavor']['id'])
+        self.assertEqual(server.image.id, raw['image']['id'])
+        self.assertEqual(server.metadata.group, raw['metadata']['group'])
+        self.assertEqual(
+            server.security_groups[0].name,
+            raw['security_groups'][0]['name'])
+
     def test_normalize_servers_strict(self):
-        raw_server = RAW_SERVER_DICT.copy()
+        res = server_resource.Server(
+            connection=self.strict_cloud,
+            **RAW_SERVER_DICT)
         expected = {
             'accessIPv4': u'',
             'accessIPv6': u'',
@@ -574,15 +585,18 @@ class TestUtils(base.TestCase):
                     u'addr': u'162.253.54.192',
                     u'version': 4}]},
             'adminPass': None,
+            'block_device_mapping': None,
             'created': u'2015-08-01T19:52:16Z',
             'created_at': u'2015-08-01T19:52:16Z',
             'disk_config': u'MANUAL',
             'flavor': {u'id': u'bbcb7eb5-5c8d-498f-9d7e-307c575d3566'},
             'has_config_drive': True,
             'host_id': u'bd37',
+            'hypervisor_hostname': None,
             'id': u'811c5197-dba7-4d3a-a3f6-68ca5328b9a7',
             'image': {u'id': u'69c99b45-cd53-49de-afdc-f24789eb8f83'},
             'interface_ip': u'',
+            'instance_name': None,
             'key_name': u'mordred',
             'launched_at': u'2015-08-01T19:52:02.000000',
             'location': {
@@ -600,31 +614,42 @@ class TestUtils(base.TestCase):
                 u'public': [
                     u'2604:e100:1:0:f816:3eff:fe9f:463e',
                     u'162.253.54.192']},
+            'personality': None,
             'power_state': 1,
             'private_v4': None,
             'progress': 0,
             'properties': {},
             'public_v4': None,
             'public_v6': None,
+            'scheduler_hints': None,
             'security_groups': [{u'name': u'default'}],
             'status': u'ACTIVE',
+            'tags': [],
             'task_state': None,
             'terminated_at': None,
             'updated': u'2016-10-15T15:49:29Z',
+            'user_data': None,
             'user_id': u'e9b21dc437d149858faee0898fb08e92',
             'vm_state': u'active',
             'volumes': []}
-        retval = self.strict_cloud._normalize_server(raw_server)
+        retval = self.strict_cloud._normalize_server(res._to_munch())
+        self._assert_server_munch_attributes(res, retval)
         self.assertEqual(expected, retval)
 
     def test_normalize_servers_normal(self):
-        raw_server = RAW_SERVER_DICT.copy()
+        res = server_resource.Server(
+            connection=self.cloud,
+            **RAW_SERVER_DICT)
         expected = {
             'OS-DCF:diskConfig': u'MANUAL',
             'OS-EXT-AZ:availability_zone': u'ca-ymq-2',
+            'OS-EXT-SRV-ATTR:hypervisor_hostname': None,
+            'OS-EXT-SRV-ATTR:instance_name': None,
+            'OS-EXT-SRV-ATTR:user_data': None,
             'OS-EXT-STS:power_state': 1,
             'OS-EXT-STS:task_state': None,
             'OS-EXT-STS:vm_state': u'active',
+            'OS-SCH-HNT:scheduler_hints': None,
             'OS-SRV-USG:launched_at': u'2015-08-01T19:52:02.000000',
             'OS-SRV-USG:terminated_at': None,
             'accessIPv4': u'',
@@ -642,6 +667,7 @@ class TestUtils(base.TestCase):
                     u'version': 4}]},
             'adminPass': None,
             'az': u'ca-ymq-2',
+            'block_device_mapping': None,
             'cloud': '_test_cloud_',
             'config_drive': u'True',
             'created': u'2015-08-01T19:52:16Z',
@@ -653,7 +679,9 @@ class TestUtils(base.TestCase):
             'host_id': u'bd37',
             'id': u'811c5197-dba7-4d3a-a3f6-68ca5328b9a7',
             'image': {u'id': u'69c99b45-cd53-49de-afdc-f24789eb8f83'},
+            'instance_name': None,
             'interface_ip': '',
+            'hypervisor_hostname': None,
             'key_name': u'mordred',
             'launched_at': u'2015-08-01T19:52:02.000000',
             'location': {
@@ -672,6 +700,7 @@ class TestUtils(base.TestCase):
                     u'2604:e100:1:0:f816:3eff:fe9f:463e',
                     u'162.253.54.192']},
             'os-extended-volumes:volumes_attached': [],
+            'personality': None,
             'power_state': 1,
             'private_v4': None,
             'progress': 0,
@@ -679,25 +708,33 @@ class TestUtils(base.TestCase):
             'properties': {
                 'OS-DCF:diskConfig': u'MANUAL',
                 'OS-EXT-AZ:availability_zone': u'ca-ymq-2',
+                'OS-EXT-SRV-ATTR:hypervisor_hostname': None,
+                'OS-EXT-SRV-ATTR:instance_name': None,
+                'OS-EXT-SRV-ATTR:user_data': None,
                 'OS-EXT-STS:power_state': 1,
                 'OS-EXT-STS:task_state': None,
                 'OS-EXT-STS:vm_state': u'active',
+                'OS-SCH-HNT:scheduler_hints': None,
                 'OS-SRV-USG:launched_at': u'2015-08-01T19:52:02.000000',
                 'OS-SRV-USG:terminated_at': None,
                 'os-extended-volumes:volumes_attached': []},
             'public_v4': None,
             'public_v6': None,
             'region': u'RegionOne',
+            'scheduler_hints': None,
             'security_groups': [{u'name': u'default'}],
             'status': u'ACTIVE',
+            'tags': [],
             'task_state': None,
             'tenant_id': u'db92b20496ae4fbda850a689ea9d563f',
             'terminated_at': None,
             'updated': u'2016-10-15T15:49:29Z',
+            'user_data': None,
             'user_id': u'e9b21dc437d149858faee0898fb08e92',
             'vm_state': u'active',
             'volumes': []}
-        retval = self.cloud._normalize_server(raw_server)
+        retval = self.cloud._normalize_server(res._to_munch())
+        self._assert_server_munch_attributes(res, retval)
         self.assertEqual(expected, retval)
 
     def test_normalize_secgroups_strict(self):

openstack/tests/unit/test_resource.py

@@ -683,6 +683,48 @@ class TestResource(base.TestCase):
         }
         self.assertEqual(expected, res.to_dict())
 
+    def test_to_dict_nested(self):
+
+        class Test(resource.Resource):
+            foo = resource.Header('foo')
+            bar = resource.Body('bar')
+            a_list = resource.Body('a_list')
+
+        class Sub(resource.Resource):
+            sub = resource.Body('foo')
+
+        sub = Sub(id='ANOTHER_ID', foo='bar')
+
+        res = Test(
+            id='FAKE_ID',
+            bar=sub,
+            a_list=[sub])
+
+        expected = {
+            'id': 'FAKE_ID',
+            'name': None,
+            'location': None,
+            'foo': None,
+            'bar': {
+                'id': 'ANOTHER_ID',
+                'name': None,
+                'sub': 'bar',
+                'location': None,
+            },
+            'a_list': [{
+                'id': 'ANOTHER_ID',
+                'name': None,
+                'sub': 'bar',
+                'location': None,
+            }],
+        }
+        self.assertEqual(expected, res.to_dict())
+        a_munch = res.to_dict(_to_munch=True)
+        self.assertEqual(a_munch.bar.id, 'ANOTHER_ID')
+        self.assertEqual(a_munch.bar.sub, 'bar')
+        self.assertEqual(a_munch.a_list[0].id, 'ANOTHER_ID')
+        self.assertEqual(a_munch.a_list[0].sub, 'bar')
+
     def test_to_dict_no_body(self):
 
         class Test(resource.Resource):

releasenotes/notes/munch-sub-dict-e1619c71c26879cb.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixed a regression with sub-dicts of server objects
+    were not usable with object notation.