Add strict mode for trimming out non-API data

shade defaults to returning everything under the sun in every form
possible in order to ensure maximum backwards compatability - even with
systems that are not shade itself. However, passthrough fields from
somewhere else could change at any time. This patch adds an opt-in flag
that skips returning passthrough fields anywhere other than the
properties dict.

Change-Id: I7071a406965ed373e77f9592eb76975400cb426b

doc/source/model.rst

@@ -22,6 +22,16 @@ into an attribute called 'properties'. The contents of properties are
 defined to be an arbitrary collection of key value pairs with no promises as
 to any particular key ever existing.
 
+If a user passes `strict=True` to the shade constructor, shade will not pass
+through arbitrary objects to the root of the resource, and will instead only
+put them in the properties dict. If a user is worried about accidentally
+writing code that depends on an attribute that is not part of the API contract,
+this can be a useful tool. Keep in mind all data can still be accessed via
+the properties dict, but any code touching anything in the properties dict
+should be aware that the keys found there are highly user/cloud specific.
+Any key that is transformed as part of the shade data model contract will
+not wind up with an entry in properties - only keys that are unknown.
+
 Location
 --------
 
@@ -154,21 +164,20 @@ A Server from Nova
     name=str(),
     image=dict() or str(),
     flavor=dict(),
-    volumes=list(),
+    volumes=list(),  # Volume
     interface_ip=str(),
     has_config_drive=bool(),
     accessIPv4=str(),
     accessIPv6=str(),
-    addresses=dict(),
+    addresses=dict(),  # string, list(Address)
     created=str(),
     key_name=str(),
-    metadata=dict(),
+    metadata=dict(),  # string, string
-    networks=dict(),
     private_v4=str(),
     progress=int(),
     public_v4=str(),
     public_v6=str(),
-    security_groups=list(),
+    security_groups=list(),  # SecurityGroup
     status=str(),
     updated=str(),
     user_id=str(),
@@ -195,9 +204,8 @@ A Floating IP from Neutron or Nova
     attached=bool(),
     fixed_ip_address=str() or None,
     floating_ip_address=str() or None,
-    floating_network_id=str() or None,
+    network=str() or None,
-    network=str(),
+    port=str() or None,
-    port_id=str() or None,
+    router=str(),
-    router_id=str(),
     status=str(),
     properties=dict())

releasenotes/notes/strict-mode-d493abc0c3e87945.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - Added 'strict' mode, which is set by passing strict=True
+    to the OpenStackCloud constructor. strict mode tells shade
+    to only return values in resources that are part of shade's
+    declared data model contract.

shade/__init__.py

@@ -55,7 +55,7 @@ def simple_logging(debug=False, http_debug=False):
     log = _log.setup_logging('keystoneauth.identity.generic.base')
 
 
-def openstack_clouds(config=None, debug=False, cloud=None):
+def openstack_clouds(config=None, debug=False, cloud=None, strict=False):
     if not config:
         config = os_client_config.OpenStackConfig()
     try:
@@ -64,6 +64,7 @@ def openstack_clouds(config=None, debug=False, cloud=None):
                 OpenStackCloud(
                     cloud=f.name, debug=debug,
                     cloud_config=f,
+                    strict=strict,
                     **f.config)
                 for f in config.get_all_clouds()
             ]
@@ -72,6 +73,7 @@ def openstack_clouds(config=None, debug=False, cloud=None):
                 OpenStackCloud(
                     cloud=f.name, debug=debug,
                     cloud_config=f,
+                    strict=strict,
                     **f.config)
                 for f in config.get_all_clouds()
                 if f.name == cloud
@@ -81,7 +83,7 @@ def openstack_clouds(config=None, debug=False, cloud=None):
             "Invalid cloud configuration: {exc}".format(exc=str(e)))
 
 
-def openstack_cloud(config=None, **kwargs):
+def openstack_cloud(config=None, strict=False, **kwargs):
     if not config:
         config = os_client_config.OpenStackConfig()
     try:
@@ -89,10 +91,10 @@ def openstack_cloud(config=None, **kwargs):
     except keystoneauth1.exceptions.auth_plugins.NoMatchingPlugin as e:
         raise OpenStackCloudException(
             "Invalid cloud configuration: {exc}".format(exc=str(e)))
-    return OpenStackCloud(cloud_config=cloud_config)
+    return OpenStackCloud(cloud_config=cloud_config, strict=strict)
 
 
-def operator_cloud(config=None, **kwargs):
+def operator_cloud(config=None, strict=False, **kwargs):
     if 'interface' not in kwargs:
         kwargs['interface'] = 'admin'
     if not config:
@@ -102,4 +104,4 @@ def operator_cloud(config=None, **kwargs):
     except keystoneauth1.exceptions.auth_plugins.NoMatchingPlugin as e:
         raise OpenStackCloudException(
             "Invalid cloud configuration: {exc}".format(exc=str(e)))
-    return OperatorCloud(cloud_config=cloud_config)
+    return OperatorCloud(cloud_config=cloud_config, strict=strict)

shade/_normalize.py

@@ -12,8 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import ast
-
 import munch
 import six
 
@@ -57,10 +55,10 @@ _SERVER_FIELDS = (
 
 def _to_bool(value):
     if isinstance(value, six.string_types):
-        # ast.literal_eval becomes VERY unhappy on empty strings
         if not value:
             return False
-        return ast.literal_eval(value.lower().capitalize())
+        prospective = value.lower().capitalize()
+        return prospective == 'True'
     return bool(value)
 
 
@@ -72,6 +70,13 @@ def _pop_float(resource, key):
     return float(resource.pop(key, 0) or 0)
 
 
+def _pop_or_get(resource, key, default, strict):
+    if strict:
+        return resource.pop(key, default)
+    else:
+        return resource.get(key, default)
+
+
 class Normalizer(object):
     '''Mix-in class to provide the normalization functions.
 
@@ -99,11 +104,14 @@ class Normalizer(object):
         flavor.pop('HUMAN_ID', None)
         flavor.pop('human_id', None)
 
-        ephemeral = int(flavor.pop('OS-FLV-EXT-DATA:ephemeral', 0))
+        ephemeral = int(_pop_or_get(
+            flavor, 'OS-FLV-EXT-DATA:ephemeral', 0, self.strict_mode))
         ephemeral = flavor.pop('ephemeral', ephemeral)
-        is_public = _to_bool(flavor.pop('os-flavor-access:is_public', True))
+        is_public = _to_bool(_pop_or_get(
-        is_public = _to_bool(flavor.pop('is_public', True))
+            flavor, 'os-flavor-access:is_public', True, self.strict_mode))
-        is_disabled = _to_bool(flavor.pop('OS-FLV-DISABLED:disabled', False))
+        is_public = _to_bool(flavor.pop('is_public', is_public))
+        is_disabled = _to_bool(_pop_or_get(
+            flavor, 'OS-FLV-DISABLED:disabled', False, self.strict_mode))
         extra_specs = flavor.pop('extra_specs', {})
 
         new_flavor['location'] = self.current_location
@@ -122,11 +130,9 @@ class Normalizer(object):
         new_flavor['extra_specs'] = extra_specs
 
         # Backwards compat with nova - passthrough values
+        if not self.strict_mode:
             for (k, v) in new_flavor['properties'].items():
                 new_flavor.setdefault(k, v)
-        new_flavor['OS-FLV-DISABLED:disabled'] = is_disabled
-        new_flavor['OS-FLV-EXT-DATA:ephemeral'] = ephemeral
-        new_flavor['os-flavor-access:is_public'] = is_public
 
         return new_flavor
 
@@ -164,6 +170,7 @@ class Normalizer(object):
         new_image['is_public'] = is_public
 
         # Backwards compat with glance
+        if not self.strict_mode:
             for key, val in properties.items():
                 new_image[key] = val
             new_image['protected'] = protected
@@ -204,6 +211,7 @@ class Normalizer(object):
         ret['properties'] = group
 
         # Backwards compat with Neutron
+        if not self.strict_mode:
             ret['tenant_id'] = project_id
             ret['project_id'] = project_id
             for key, val in ret['properties'].items():
@@ -260,6 +268,7 @@ class Normalizer(object):
         ret['properties'] = rule
 
         # Backwards compat with Neutron
+        if not self.strict_mode:
             ret['tenant_id'] = project_id
             ret['project_id'] = project_id
             for key, val in ret['properties'].items():
@@ -299,12 +308,15 @@ class Normalizer(object):
         project_id = server.pop('tenant_id', '')
         project_id = server.pop('project_id', project_id)
 
-        az = server.get('OS-EXT-AZ:availability_zone', None)
+        az = _pop_or_get(
+            server, 'OS-EXT-AZ:availability_zone', None, self.strict_mode)
         ret['location'] = self._get_current_location(
             project_id=project_id, zone=az)
 
         # Ensure volumes is always in the server dict, even if empty
-        ret['volumes'] = []
+        ret['volumes'] = _pop_or_get(
+            server, 'os-extended-volumes:volumes_attached',
+            [], self.strict_mode)
 
         config_drive = server.pop('config_drive', False)
         ret['has_config_drive'] = _to_bool(config_drive)
@@ -315,7 +327,8 @@ class Normalizer(object):
         ret['progress'] = _pop_int(server, 'progress')
 
         # Leave these in so that the general properties handling works
-        ret['disk_config'] = server.get('OS-DCF:diskConfig')
+        ret['disk_config'] = _pop_or_get(
+            server, 'OS-DCF:diskConfig', None, self.strict_mode)
         for key in (
                 'OS-EXT-STS:power_state',
                 'OS-EXT-STS:task_state',
@@ -323,17 +336,16 @@ class Normalizer(object):
                 'OS-SRV-USG:launched_at',
                 'OS-SRV-USG:terminated_at'):
             short_key = key.split(':')[1]
-            ret[short_key] = server.get(key)
+            ret[short_key] = _pop_or_get(server, key, None, self.strict_mode)
 
         for field in _SERVER_FIELDS:
             ret[field] = server.pop(field, None)
         ret['interface_ip'] = ''
 
         ret['properties'] = server.copy()
-        for key, val in ret['properties'].items():
-            ret.setdefault(key, val)
 
         # Backwards compat
+        if not self.strict_mode:
             ret['hostId'] = host_id
             ret['config_drive'] = config_drive
             ret['project_id'] = project_id
@@ -341,6 +353,8 @@ class Normalizer(object):
             ret['region'] = self.region_name
             ret['cloud'] = self.name
             ret['az'] = az
+            for key, val in ret['properties'].items():
+                ret.setdefault(key, val)
         return ret
 
     def _normalize_floating_ips(self, ips):
@@ -406,17 +420,21 @@ class Normalizer(object):
             attached=attached,
             fixed_ip_address=fixed_ip_address,
             floating_ip_address=floating_ip_address,
-            floating_network_id=network_id,
             id=id,
             location=self._get_current_location(project_id=project_id),
             network=network_id,
-            port_id=port_id,
+            port=port_id,
-            project_id=project_id,
+            router=router_id,
-            router_id=router_id,
             status=status,
-            tenant_id=project_id,
             properties=ip.copy(),
         )
+        # Backwards compat
+        if not self.strict_mode:
+            ret['port_id'] = port_id
+            ret['router_id'] = router_id
+            ret['project_id'] = project_id
+            ret['tenant_id'] = project_id
+            ret['floating_network_id'] = network_id,
             for key, val in ret['properties'].items():
                 ret.setdefault(key, val)
 

shade/openstackcloud.py

@@ -123,6 +123,8 @@ class OpenStackCloud(_normalize.Normalizer):
                                       have all of the wrapped exceptions be
                                       emitted to the error log. This flag
                                       will enable that behavior.
+    :param bool strict: Only return documented attributes for each resource
+                        as per the shade Data Model contract. (Default False)
     :param CloudConfig cloud_config: Cloud config object from os-client-config
                                      In the future, this will be the only way
                                      to pass in cloud configuration, but is
@@ -132,7 +134,9 @@ class OpenStackCloud(_normalize.Normalizer):
     def __init__(
             self,
             cloud_config=None,
-            manager=None, log_inner_exceptions=False, **kwargs):
+            manager=None, log_inner_exceptions=False,
+            strict=False,
+            **kwargs):
 
         if log_inner_exceptions:
             OpenStackCloudException.log_inner_exceptions = True
@@ -151,6 +155,7 @@ class OpenStackCloud(_normalize.Normalizer):
         self.image_api_use_tasks = cloud_config.config['image_api_use_tasks']
         self.secgroup_source = cloud_config.config['secgroup_source']
         self.force_ipv4 = cloud_config.force_ipv4
+        self.strict_mode = strict
 
         # Provide better error message for people with stale OCC
         if cloud_config.get_external_ipv4_networks is None:

shade/tests/unit/base.py

@@ -75,6 +75,10 @@ class BaseTestCase(base.TestCase):
         self.cloud = shade.OpenStackCloud(
             cloud_config=self.cloud_config,
             log_inner_exceptions=True)
+        self.strict_cloud = shade.OpenStackCloud(
+            cloud_config=self.cloud_config,
+            log_inner_exceptions=True,
+            strict=True)
         self.op_cloud = shade.OperatorCloud(
             cloud_config=self.cloud_config,
             log_inner_exceptions=True)

shade/tests/unit/test__utils.py

@@ -12,7 +12,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import mock
 import testtools
 
 from shade import _utils
@@ -80,131 +79,6 @@ class TestUtils(base.TestCase):
                 }})
         self.assertEqual([el2, el3], ret)
 
-    def test_normalize_secgroups(self):
-        nova_secgroup = dict(
-            id='abc123',
-            name='nova_secgroup',
-            description='A Nova security group',
-            rules=[
-                dict(id='123', from_port=80, to_port=81, ip_protocol='tcp',
-                     ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123')
-            ]
-        )
-
-        expected = dict(
-            id='abc123',
-            name='nova_secgroup',
-            description='A Nova security group',
-            tenant_id='',
-            project_id='',
-            properties={},
-            location=dict(
-                region_name='RegionOne',
-                zone=None,
-                project=dict(
-                    domain_name=None,
-                    id=mock.ANY,
-                    domain_id=None,
-                    name='admin'),
-                cloud='_test_cloud_'),
-            security_group_rules=[
-                dict(id='123', direction='ingress', ethertype='IPv4',
-                     port_range_min=80, port_range_max=81, protocol='tcp',
-                     remote_ip_prefix='0.0.0.0/0', security_group_id='xyz123',
-                     properties={},
-                     tenant_id='',
-                     project_id='',
-                     remote_group_id=None,
-                     location=dict(
-                         region_name='RegionOne',
-                         zone=None,
-                         project=dict(
-                             domain_name=None,
-                             id=mock.ANY,
-                             domain_id=None,
-                             name='admin'),
-                         cloud='_test_cloud_'))
-            ]
-        )
-
-        retval = self.cloud._normalize_secgroup(nova_secgroup)
-        self.assertEqual(expected, retval)
-
-    def test_normalize_secgroups_negone_port(self):
-        nova_secgroup = dict(
-            id='abc123',
-            name='nova_secgroup',
-            description='A Nova security group with -1 ports',
-            rules=[
-                dict(id='123', from_port=-1, to_port=-1, ip_protocol='icmp',
-                     ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123')
-            ]
-        )
-
-        retval = self.cloud._normalize_secgroup(nova_secgroup)
-        self.assertIsNone(retval['security_group_rules'][0]['port_range_min'])
-        self.assertIsNone(retval['security_group_rules'][0]['port_range_max'])
-
-    def test_normalize_secgroup_rules(self):
-        nova_rules = [
-            dict(id='123', from_port=80, to_port=81, ip_protocol='tcp',
-                 ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123')
-        ]
-        expected = [
-            dict(id='123', direction='ingress', ethertype='IPv4',
-                 port_range_min=80, port_range_max=81, protocol='tcp',
-                 remote_ip_prefix='0.0.0.0/0', security_group_id='xyz123',
-                 tenant_id='', project_id='', remote_group_id=None,
-                 properties={},
-                 location=dict(
-                     region_name='RegionOne',
-                     zone=None,
-                     project=dict(
-                         domain_name=None,
-                         id=mock.ANY,
-                         domain_id=None,
-                         name='admin'),
-                     cloud='_test_cloud_'))
-        ]
-        retval = self.cloud._normalize_secgroup_rules(nova_rules)
-        self.assertEqual(expected, retval)
-
-    def test_normalize_volumes_v1(self):
-        vol = dict(
-            display_name='test',
-            display_description='description',
-            bootable=u'false',   # unicode type
-            multiattach='true',  # str type
-        )
-        expected = dict(
-            name=vol['display_name'],
-            display_name=vol['display_name'],
-            description=vol['display_description'],
-            display_description=vol['display_description'],
-            bootable=False,
-            multiattach=True,
-        )
-        retval = _utils.normalize_volumes([vol])
-        self.assertEqual([expected], retval)
-
-    def test_normalize_volumes_v2(self):
-        vol = dict(
-            display_name='test',
-            display_description='description',
-            bootable=False,
-            multiattach=True,
-        )
-        expected = dict(
-            name=vol['display_name'],
-            display_name=vol['display_name'],
-            description=vol['display_description'],
-            display_description=vol['display_description'],
-            bootable=False,
-            multiattach=True,
-        )
-        retval = _utils.normalize_volumes([vol])
-        self.assertEqual([expected], retval)
-
     def test_safe_dict_min_ints(self):
         """Test integer comparison"""
         data = [{'f1': 3}, {'f1': 2}, {'f1': 1}]

shade/tests/unit/test_normalize.py

@@ -0,0 +1,394 @@
+# -*- coding: utf-8 -*-
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from shade import _utils
+from shade.tests.unit import base
+
+RAW_SERVER_DICT = {
+    'HUMAN_ID': True,
+    'NAME_ATTR': 'name',
+    'OS-DCF:diskConfig': u'MANUAL',
+    'OS-EXT-AZ:availability_zone': u'ca-ymq-2',
+    'OS-EXT-STS:power_state': 1,
+    'OS-EXT-STS:task_state': None,
+    'OS-EXT-STS:vm_state': u'active',
+    'OS-SRV-USG:launched_at': u'2015-08-01T19:52:02.000000',
+    'OS-SRV-USG:terminated_at': None,
+    'accessIPv4': u'',
+    'accessIPv6': u'',
+    'addresses': {
+        u'public': [{
+            u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e',
+            u'OS-EXT-IPS:type': u'fixed',
+            u'addr': u'2604:e100:1:0:f816:3eff:fe9f:463e',
+            u'version': 6
+        }, {
+            u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e',
+            u'OS-EXT-IPS:type': u'fixed',
+            u'addr': u'162.253.54.192',
+            u'version': 4}]},
+    'config_drive': u'True',
+    'created': u'2015-08-01T19:52:16Z',
+    'flavor': {
+        u'id': u'bbcb7eb5-5c8d-498f-9d7e-307c575d3566',
+        u'links': [{
+            u'href': u'https://compute-ca-ymq-1.vexxhost.net/db9/flavors/bbc',
+            u'rel': u'bookmark'}]},
+    'hostId': u'bd37',
+    'human_id': u'mordred-irc',
+    'id': u'811c5197-dba7-4d3a-a3f6-68ca5328b9a7',
+    'image': {
+        u'id': u'69c99b45-cd53-49de-afdc-f24789eb8f83',
+        u'links': [{
+            u'href': u'https://compute-ca-ymq-1.vexxhost.net/db9/images/69c',
+            u'rel': u'bookmark'}]},
+    'key_name': u'mordred',
+    'links': [{
+        u'href': u'https://compute-ca-ymq-1.vexxhost.net/v2/db9/servers/811',
+        u'rel': u'self'
+    }, {
+        u'href': u'https://compute-ca-ymq-1.vexxhost.net/db9/servers/811',
+        u'rel': u'bookmark'}],
+    'metadata': {u'group': u'irc', u'groups': u'irc,enabled'},
+    'name': u'mordred-irc',
+    'networks': {u'public': [u'2604:e100:1:0:f816:3eff:fe9f:463e',
+                             u'162.253.54.192']},
+    'os-extended-volumes:volumes_attached': [],
+    'progress': 0,
+    'request_ids': [],
+    'security_groups': [{u'name': u'default'}],
+    'status': u'ACTIVE',
+    'tenant_id': u'db92b20496ae4fbda850a689ea9d563f',
+    'updated': u'2016-10-15T15:49:29Z',
+    'user_id': u'e9b21dc437d149858faee0898fb08e92'}
+
+
+class TestUtils(base.TestCase):
+
+    def test_normalize_servers_strict(self):
+        raw_server = RAW_SERVER_DICT.copy()
+        expected = {
+            'accessIPv4': u'',
+            'accessIPv6': u'',
+            'addresses': {
+                u'public': [{
+                    u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e',
+                    u'OS-EXT-IPS:type': u'fixed',
+                    u'addr': u'2604:e100:1:0:f816:3eff:fe9f:463e',
+                    u'version': 6
+                }, {
+                    u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e',
+                    u'OS-EXT-IPS:type': u'fixed',
+                    u'addr': u'162.253.54.192',
+                    u'version': 4}]},
+            'adminPass': None,
+            'created': u'2015-08-01T19:52:16Z',
+            'disk_config': u'MANUAL',
+            'flavor': {u'id': u'bbcb7eb5-5c8d-498f-9d7e-307c575d3566'},
+            'has_config_drive': True,
+            'host_id': u'bd37',
+            'id': u'811c5197-dba7-4d3a-a3f6-68ca5328b9a7',
+            'image': {u'id': u'69c99b45-cd53-49de-afdc-f24789eb8f83'},
+            'interface_ip': u'',
+            'key_name': u'mordred',
+            'launched_at': u'2015-08-01T19:52:02.000000',
+            'location': {
+                'cloud': '_test_cloud_',
+                'project': {
+                    'domain_id': None,
+                    'domain_name': None,
+                    'id': u'db92b20496ae4fbda850a689ea9d563f',
+                    'name': None},
+                'region_name': u'RegionOne',
+                'zone': u'ca-ymq-2'},
+            'metadata': {u'group': u'irc', u'groups': u'irc,enabled'},
+            'name': u'mordred-irc',
+            'networks': {
+                u'public': [
+                    u'2604:e100:1:0:f816:3eff:fe9f:463e',
+                    u'162.253.54.192']},
+            'power_state': 1,
+            'private_v4': None,
+            'progress': 0,
+            'properties': {
+                'request_ids': []},
+            'public_v4': None,
+            'public_v6': None,
+            'security_groups': [{u'name': u'default'}],
+            'status': u'ACTIVE',
+            'task_state': None,
+            'terminated_at': None,
+            'updated': u'2016-10-15T15:49:29Z',
+            'user_id': u'e9b21dc437d149858faee0898fb08e92',
+            'vm_state': u'active',
+            'volumes': []}
+        retval = self.strict_cloud._normalize_server(raw_server).toDict()
+        self.assertEqual(expected, retval)
+
+    def test_normalize_servers_normal(self):
+        raw_server = RAW_SERVER_DICT.copy()
+        expected = {
+            'OS-DCF:diskConfig': u'MANUAL',
+            'OS-EXT-AZ:availability_zone': u'ca-ymq-2',
+            'OS-EXT-STS:power_state': 1,
+            'OS-EXT-STS:task_state': None,
+            'OS-EXT-STS:vm_state': u'active',
+            'OS-SRV-USG:launched_at': u'2015-08-01T19:52:02.000000',
+            'OS-SRV-USG:terminated_at': None,
+            'accessIPv4': u'',
+            'accessIPv6': u'',
+            'addresses': {
+                u'public': [{
+                    u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e',
+                    u'OS-EXT-IPS:type': u'fixed',
+                    u'addr': u'2604:e100:1:0:f816:3eff:fe9f:463e',
+                    u'version': 6
+                }, {
+                    u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e',
+                    u'OS-EXT-IPS:type': u'fixed',
+                    u'addr': u'162.253.54.192',
+                    u'version': 4}]},
+            'adminPass': None,
+            'az': u'ca-ymq-2',
+            'cloud': '_test_cloud_',
+            'config_drive': u'True',
+            'created': u'2015-08-01T19:52:16Z',
+            'disk_config': u'MANUAL',
+            'flavor': {u'id': u'bbcb7eb5-5c8d-498f-9d7e-307c575d3566'},
+            'has_config_drive': True,
+            'hostId': u'bd37',
+            'host_id': u'bd37',
+            'id': u'811c5197-dba7-4d3a-a3f6-68ca5328b9a7',
+            'image': {u'id': u'69c99b45-cd53-49de-afdc-f24789eb8f83'},
+            'interface_ip': '',
+            'key_name': u'mordred',
+            'launched_at': u'2015-08-01T19:52:02.000000',
+            'location': {
+                'cloud': '_test_cloud_',
+                'project': {
+                    'domain_id': None,
+                    'domain_name': None,
+                    'id': u'db92b20496ae4fbda850a689ea9d563f',
+                    'name': None},
+                'region_name': u'RegionOne',
+                'zone': u'ca-ymq-2'},
+            'metadata': {u'group': u'irc', u'groups': u'irc,enabled'},
+            'name': u'mordred-irc',
+            'networks': {
+                u'public': [
+                    u'2604:e100:1:0:f816:3eff:fe9f:463e',
+                    u'162.253.54.192']},
+            'os-extended-volumes:volumes_attached': [],
+            'power_state': 1,
+            'private_v4': None,
+            'progress': 0,
+            'project_id': u'db92b20496ae4fbda850a689ea9d563f',
+            'properties': {
+                'OS-DCF:diskConfig': u'MANUAL',
+                'OS-EXT-AZ:availability_zone': u'ca-ymq-2',
+                'OS-EXT-STS:power_state': 1,
+                'OS-EXT-STS:task_state': None,
+                'OS-EXT-STS:vm_state': u'active',
+                'OS-SRV-USG:launched_at': u'2015-08-01T19:52:02.000000',
+                'OS-SRV-USG:terminated_at': None,
+                'os-extended-volumes:volumes_attached': [],
+                'request_ids': []},
+            'public_v4': None,
+            'public_v6': None,
+            'region': u'RegionOne',
+            'request_ids': [],
+            'security_groups': [{u'name': u'default'}],
+            'status': u'ACTIVE',
+            'task_state': None,
+            'tenant_id': u'db92b20496ae4fbda850a689ea9d563f',
+            'terminated_at': None,
+            'updated': u'2016-10-15T15:49:29Z',
+            'user_id': u'e9b21dc437d149858faee0898fb08e92',
+            'vm_state': u'active',
+            'volumes': []}
+        retval = self.cloud._normalize_server(raw_server).toDict()
+        self.assertEqual(expected, retval)
+
+    def test_normalize_secgroups_strict(self):
+        nova_secgroup = dict(
+            id='abc123',
+            name='nova_secgroup',
+            description='A Nova security group',
+            rules=[
+                dict(id='123', from_port=80, to_port=81, ip_protocol='tcp',
+                     ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123')
+            ]
+        )
+
+        expected = dict(
+            id='abc123',
+            name='nova_secgroup',
+            description='A Nova security group',
+            properties={},
+            location=dict(
+                region_name='RegionOne',
+                zone=None,
+                project=dict(
+                    domain_name=None,
+                    id=mock.ANY,
+                    domain_id=None,
+                    name='admin'),
+                cloud='_test_cloud_'),
+            security_group_rules=[
+                dict(id='123', direction='ingress', ethertype='IPv4',
+                     port_range_min=80, port_range_max=81, protocol='tcp',
+                     remote_ip_prefix='0.0.0.0/0', security_group_id='xyz123',
+                     properties={},
+                     remote_group_id=None,
+                     location=dict(
+                         region_name='RegionOne',
+                         zone=None,
+                         project=dict(
+                             domain_name=None,
+                             id=mock.ANY,
+                             domain_id=None,
+                             name='admin'),
+                         cloud='_test_cloud_'))
+            ]
+        )
+
+        retval = self.strict_cloud._normalize_secgroup(nova_secgroup)
+        self.assertEqual(expected, retval)
+
+    def test_normalize_secgroups(self):
+        nova_secgroup = dict(
+            id='abc123',
+            name='nova_secgroup',
+            description='A Nova security group',
+            rules=[
+                dict(id='123', from_port=80, to_port=81, ip_protocol='tcp',
+                     ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123')
+            ]
+        )
+
+        expected = dict(
+            id='abc123',
+            name='nova_secgroup',
+            description='A Nova security group',
+            tenant_id='',
+            project_id='',
+            properties={},
+            location=dict(
+                region_name='RegionOne',
+                zone=None,
+                project=dict(
+                    domain_name=None,
+                    id=mock.ANY,
+                    domain_id=None,
+                    name='admin'),
+                cloud='_test_cloud_'),
+            security_group_rules=[
+                dict(id='123', direction='ingress', ethertype='IPv4',
+                     port_range_min=80, port_range_max=81, protocol='tcp',
+                     remote_ip_prefix='0.0.0.0/0', security_group_id='xyz123',
+                     properties={},
+                     tenant_id='',
+                     project_id='',
+                     remote_group_id=None,
+                     location=dict(
+                         region_name='RegionOne',
+                         zone=None,
+                         project=dict(
+                             domain_name=None,
+                             id=mock.ANY,
+                             domain_id=None,
+                             name='admin'),
+                         cloud='_test_cloud_'))
+            ]
+        )
+
+        retval = self.cloud._normalize_secgroup(nova_secgroup)
+        self.assertEqual(expected, retval)
+
+    def test_normalize_secgroups_negone_port(self):
+        nova_secgroup = dict(
+            id='abc123',
+            name='nova_secgroup',
+            description='A Nova security group with -1 ports',
+            rules=[
+                dict(id='123', from_port=-1, to_port=-1, ip_protocol='icmp',
+                     ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123')
+            ]
+        )
+
+        retval = self.cloud._normalize_secgroup(nova_secgroup)
+        self.assertIsNone(retval['security_group_rules'][0]['port_range_min'])
+        self.assertIsNone(retval['security_group_rules'][0]['port_range_max'])
+
+    def test_normalize_secgroup_rules(self):
+        nova_rules = [
+            dict(id='123', from_port=80, to_port=81, ip_protocol='tcp',
+                 ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123')
+        ]
+        expected = [
+            dict(id='123', direction='ingress', ethertype='IPv4',
+                 port_range_min=80, port_range_max=81, protocol='tcp',
+                 remote_ip_prefix='0.0.0.0/0', security_group_id='xyz123',
+                 tenant_id='', project_id='', remote_group_id=None,
+                 properties={},
+                 location=dict(
+                     region_name='RegionOne',
+                     zone=None,
+                     project=dict(
+                         domain_name=None,
+                         id=mock.ANY,
+                         domain_id=None,
+                         name='admin'),
+                     cloud='_test_cloud_'))
+        ]
+        retval = self.cloud._normalize_secgroup_rules(nova_rules)
+        self.assertEqual(expected, retval)
+
+    def test_normalize_volumes_v1(self):
+        vol = dict(
+            display_name='test',
+            display_description='description',
+            bootable=u'false',   # unicode type
+            multiattach='true',  # str type
+        )
+        expected = dict(
+            name=vol['display_name'],
+            display_name=vol['display_name'],
+            description=vol['display_description'],
+            display_description=vol['display_description'],
+            bootable=False,
+            multiattach=True,
+        )
+        retval = _utils.normalize_volumes([vol])
+        self.assertEqual([expected], retval)
+
+    def test_normalize_volumes_v2(self):
+        vol = dict(
+            display_name='test',
+            display_description='description',
+            bootable=False,
+            multiattach=True,
+        )
+        expected = dict(
+            name=vol['display_name'],
+            display_name=vol['display_name'],
+            description=vol['display_description'],
+            display_description=vol['display_description'],
+            bootable=False,
+            multiattach=True,
+        )
+        retval = _utils.normalize_volumes([vol])
+        self.assertEqual([expected], retval)