Use dict comprehensions instead of dict constructor

PEP-0274 introduced dict comprehensions [1], these are benefits:
  The dictionary constructor approach has two distinct disadvantages
  from the proposed syntax though.  First, it isn't as legible as a
  dict comprehension.  Second, it forces the programmer to create an
  in-core list object first, which could be expensive.

Keystone dropped python 2.6 support in Kilo, we can leaverage this now.
There is deep dive about PEP-0274[2] and basic tests about
performance[3].

Note: This commit doesn't handle dict constructor with kwargs.
This commit also adds a hacking rule.

[1]http://legacy.python.org/dev/peps/pep-0274/
[2]http://doughellmann.com/2012/11/12/the-performance-impact-of-using-dict-instead-of-in-cpython-2-7-2.html
[3]http://paste.openstack.org/show/154798

Change-Id: Ie74719d0c969fa7819c243d5b162df6656c1e136

keystone/auth/controllers.py

@@ -560,7 +560,7 @@ class Auth(controller.V3Controller):
         # it's most likely that only one of these will be filled so avoid
         # the combination if possible.
         if a and b:
-            return dict((x['id'], x) for x in a + b).values()
+            return {x['id']: x for x in a + b}.values()
         else:
             return a or b
 

keystone/common/cache/backends/mongo.py

@@ -418,14 +418,13 @@ class MongoApi(object):
 
     def get_multi(self, keys):
         db_results = self._get_results_as_dict(keys)
-        return dict((doc['_id'], doc['value']) for doc in
-                    six.itervalues(db_results))
+        return {doc['_id']: doc['value'] for doc in six.itervalues(db_results)}
 
     def _get_results_as_dict(self, keys):
         critieria = {'_id': {'$in': keys}}
         db_results = self.get_cache_collection().find(spec=critieria,
                                                       **self.meth_kwargs)
-        return dict((doc['_id'], doc) for doc in db_results)
+        return {doc['_id']: doc for doc in db_results}
 
     def set(self, key, value):
         doc_date = self._get_doc_date()

keystone/common/kvs/backends/memcached.py

@@ -142,22 +142,21 @@ class MemcachedBackend(manager.Manager):
             # all ``set`` and ``set_multi`` calls by the driver, by calling
             # the client directly it is possible to exclude the ``time``
             # argument to the memcached server.
-            new_mapping = dict((k, mapping[k]) for k in no_expiry_keys)
+            new_mapping = {k: mapping[k] for k in no_expiry_keys}
             set_arguments = self._get_set_arguments_driver_attr(
                 exclude_expiry=True)
             self.driver.client.set_multi(new_mapping, **set_arguments)
 
         if has_expiry_keys:
-            new_mapping = dict((k, mapping[k]) for k in has_expiry_keys)
+            new_mapping = {k: mapping[k] for k in has_expiry_keys}
             self.driver.set_multi(new_mapping)
 
     @classmethod
     def from_config_dict(cls, config_dict, prefix):
         prefix_len = len(prefix)
         return cls(
-            dict((key[prefix_len:], config_dict[key])
-                 for key in config_dict
-                 if key.startswith(prefix)))
+            {key[prefix_len:]: config_dict[key] for key in config_dict
+             if key.startswith(prefix)})
 
     @property
     def key_mangler(self):

keystone/common/ldap/core.py

@@ -1319,7 +1319,7 @@ class BaseLdap(object):
         # in a case-insensitive way.  We use the case specified in the
         # mapping for the model to ensure we have a predictable way of
         # retrieving values later.
-        lower_res = dict((k.lower(), v) for k, v in six.iteritems(res[1]))
+        lower_res = {k.lower(): v for k, v in six.iteritems(res[1])}
 
         id_attrs = lower_res.get(self.id_attr.lower())
         if not id_attrs:

keystone/common/sql/core.py

@@ -125,8 +125,8 @@ class DictBase(models.ModelBase):
     def from_dict(cls, d):
         new_d = d.copy()
 
-        new_d['extra'] = dict((k, new_d.pop(k)) for k in six.iterkeys(d)
-                              if k not in cls.attributes and k != 'extra')
+        new_d['extra'] = {k: new_d.pop(k) for k in six.iterkeys(d)
+                          if k not in cls.attributes and k != 'extra'}
 
         return cls(**new_d)
 
@@ -163,7 +163,7 @@ class ModelDictMixin(object):
     def to_dict(self):
         """Returns the model's attributes as a dictionary."""
         names = (column.name for column in self.__table__.columns)
-        return dict((name, getattr(self, name)) for name in names)
+        return {name: getattr(self, name) for name in names}
 
 
 _engine_facade = None

keystone/common/wsgi.py

@@ -283,8 +283,7 @@ class Application(BaseApplication):
         return arg.replace(':', '_').replace('-', '_')
 
     def _normalize_dict(self, d):
-        return dict([(self._normalize_arg(k), v)
-                     for (k, v) in six.iteritems(d)])
+        return {self._normalize_arg(k): v for (k, v) in six.iteritems(d)}
 
     def assert_admin(self, context):
         if not context['is_admin']:

keystone/contrib/endpoint_filter/controllers.py

@@ -104,9 +104,9 @@ class EndpointFilterV3Controller(_ControllerBase):
         """List all endpoints currently associated with a given project."""
         self.resource_api.get_project(project_id)
         refs = self.endpoint_filter_api.list_endpoints_for_project(project_id)
-        filtered_endpoints = dict(
-            (ref['endpoint_id'], self.catalog_api.get_endpoint(
-                ref['endpoint_id'])) for ref in refs)
+        filtered_endpoints = {ref['endpoint_id']:
+                              self.catalog_api.get_endpoint(ref['endpoint_id'])
+                              for ref in refs}
 
         # need to recover endpoint_groups associated with project
         # then for each endpoint group return the endpoints.

keystone/contrib/federation/utils.py

@@ -360,8 +360,8 @@ class RuleProcessor(object):
         # This will create a new dictionary where the values are arrays, and
         # any multiple values are stored in the arrays.
         LOG.debug('assertion data: %s', assertion_data)
-        assertion = dict((n, v.split(';')) for n, v in assertion_data.items()
-                         if isinstance(v, six.string_types))
+        assertion = {n: v.split(';') for n, v in assertion_data.items()
+                     if isinstance(v, six.string_types)}
         LOG.debug('assertion: %s', assertion)
         identity_values = []
 

keystone/contrib/oauth1/core.py

@@ -140,7 +140,7 @@ def get_oauth_headers(headers):
 
 def extract_non_oauth_params(query_string):
     params = oauthlib.common.extract_params(query_string)
-    return dict([(k, v) for k, v in params if not k.startswith('oauth_')])
+    return {k: v for k, v in params if not k.startswith('oauth_')}
 
 
 @dependency.provider('oauth_api')

keystone/contrib/revoke/model.py

@@ -91,8 +91,8 @@ class RevokeEvent(object):
                 'audit_id',
                 'audit_chain_id',
                 ]
-        event = dict((key, self.__dict__[key]) for key in keys
-                     if self.__dict__[key] is not None)
+        event = {key: self.__dict__[key] for key in keys
+                 if self.__dict__[key] is not None}
         if self.trust_id is not None:
             event['OS-TRUST:trust_id'] = self.trust_id
         if self.consumer_id is not None:

keystone/exception.py

@@ -62,8 +62,8 @@ class Error(Exception):
                 message = self.message_format % kwargs
             except UnicodeDecodeError:
                 try:
-                    kwargs = dict([(k, encodeutils.safe_decode(v)) for k, v in
-                                   six.iteritems(kwargs)])
+                    kwargs = {k: encodeutils.safe_decode(v)
+                              for k, v in six.iteritems(kwargs)}
                 except UnicodeDecodeError:
                     # NOTE(jamielennox): This is the complete failure case
                     # at least by showing the template we have some idea

keystone/hacking/checks.py

@@ -414,9 +414,33 @@ def check_oslo_namespace_imports(logical_line, blank_before, filename):
         yield(0, msg)
 
 
+def dict_constructor_with_sequence_copy(logical_line):
+    """Should use a dict comprehension instead of a dict constructor.
+
+    PEP-0274 introduced dict comprehension with performance enhancement
+    and it also makes code more readable.
+
+    Okay: lower_res = {k.lower(): v for k, v in six.iteritems(res[1])}
+    Okay: fool = dict(a='a', b='b')
+    K008: lower_res = dict((k.lower(), v) for k, v in six.iteritems(res[1]))
+    K008:     attrs = dict([(k, _from_json(v))
+    K008: dict([[i,i] for i in range(3)])
+
+    """
+    MESSAGE = ("K008 Must use a dict comprehension instead of a dict"
+               " constructor with a sequence of key-value pairs.")
+
+    dict_constructor_with_sequence_re = (
+        re.compile(r".*\bdict\((\[)?(\(|\[)(?!\{)"))
+
+    if dict_constructor_with_sequence_re.match(logical_line):
+        yield (0, MESSAGE)
+
+
 def factory(register):
     register(CheckForMutableDefaultArgs)
     register(block_comments_begin_with_a_space)
     register(CheckForAssertingNoneEquality)
     register(CheckForLoggingIssues)
     register(check_oslo_namespace_imports)
+    register(dict_constructor_with_sequence_copy)

keystone/tests/unit/backend/role/core.py

@@ -58,13 +58,13 @@ class RoleTests(object):
         role = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
         self.role_api.create_role(role['id'], role)
         role_ref = self.role_api.get_role(role['id'])
-        role_ref_dict = dict((x, role_ref[x]) for x in role_ref)
+        role_ref_dict = {x: role_ref[x] for x in role_ref}
         self.assertDictEqual(role_ref_dict, role)
 
         role['name'] = uuid.uuid4().hex
         updated_role_ref = self.role_api.update_role(role['id'], role)
         role_ref = self.role_api.get_role(role['id'])
-        role_ref_dict = dict((x, role_ref[x]) for x in role_ref)
+        role_ref_dict = {x: role_ref[x] for x in role_ref}
         self.assertDictEqual(role_ref_dict, role)
         self.assertDictEqual(role_ref_dict, updated_role_ref)
 

keystone/tests/unit/fakeldap.py

@@ -348,8 +348,7 @@ class FakeLdap(core.LDAPHandler):
                       core.utf8_decode(dn))
             raise ldap.ALREADY_EXISTS(dn)
 
-        self.db[key] = dict([(k, _internal_attr(k, v))
-                             for k, v in modlist])
+        self.db[key] = {k: _internal_attr(k, v) for k, v in modlist}
         self.db.sync()
 
     def delete_s(self, dn):
@@ -512,8 +511,8 @@ class FakeLdap(core.LDAPHandler):
             match_attrs[id_attr] = [id_val]
             if not filterstr or _match_query(filterstr, match_attrs):
                 # filter the attributes by attrlist
-                attrs = dict([(k, v) for k, v in six.iteritems(attrs)
-                              if not attrlist or k in attrlist])
+                attrs = {k: v for k, v in six.iteritems(attrs)
+                         if not attrlist or k in attrlist}
                 objects.append((dn, attrs))
 
         return objects

keystone/tests/unit/ksfixtures/hacking.py

@@ -257,6 +257,21 @@ class HackingCode(fixtures.Fixture):
         ],
     }
 
+    dict_constructor = {
+        'code': """
+            lower_res = {k.lower(): v for k, v in six.iteritems(res[1])}
+            fool = dict(a='a', b='b')
+            lower_res = dict((k.lower(), v) for k, v in six.iteritems(res[1]))
+            attrs = dict([(k, _from_json(v))])
+            dict([[i,i] for i in range(3)])
+            dict(({1:2}))
+        """,
+        'expected_errors': [
+            (3, 0, 'K008'),
+            (4, 0, 'K008'),
+            (5, 0, 'K008'),
+        ]}
+
 
 class HackingLogging(fixtures.Fixture):
 

keystone/tests/unit/rest.py

@@ -78,8 +78,7 @@ class RestfulTestCase(tests.TestCase):
     def request(self, app, path, body=None, headers=None, token=None,
                 expected_status=None, **kwargs):
         if headers:
-            headers = dict([(str(k), str(v)) for k, v
-                            in six.iteritems(headers)])
+            headers = {str(k): str(v) for k, v in six.iteritems(headers)}
         else:
             headers = {}
 

keystone/tests/unit/test_auth_plugin.py

@@ -63,15 +63,14 @@ class TestAuthPlugin(tests.SQLDriverOverrides, tests.TestCase):
 
     def config_overrides(self):
         super(TestAuthPlugin, self).config_overrides()
-        method_opts = dict(
-            [
-                ('external', 'keystone.auth.plugins.external.DefaultDomain'),
-                ('password', 'keystone.auth.plugins.password.Password'),
-                ('token', 'keystone.auth.plugins.token.Token'),
-                (METHOD_NAME,
-                 'keystone.tests.unit.test_auth_plugin.'
-                 'SimpleChallengeResponse'),
-            ])
+        method_opts = {
+            'external': 'keystone.auth.plugins.external.DefaultDomain',
+            'password': 'keystone.auth.plugins.password.Password',
+            'token': 'keystone.auth.plugins.token.Token',
+            METHOD_NAME:
+                'keystone.tests.unit.test_auth_plugin.SimpleChallengeResponse',
+        }
+
         self.auth_plugin_config_override(
             methods=['external', 'password', 'token', METHOD_NAME],
             **method_opts)

keystone/tests/unit/test_backend.py

@@ -2962,14 +2962,14 @@ class IdentityTests(object):
         user = self.identity_api.create_user(user_dict)
         user_ref = self.identity_api.get_user(user['id'])
         del user_dict['password']
-        user_ref_dict = dict((x, user_ref[x]) for x in user_ref)
+        user_ref_dict = {x: user_ref[x] for x in user_ref}
         self.assertDictContainsSubset(user_dict, user_ref_dict)
 
         user_dict['password'] = uuid.uuid4().hex
         self.identity_api.update_user(user['id'], user_dict)
         user_ref = self.identity_api.get_user(user['id'])
         del user_dict['password']
-        user_ref_dict = dict((x, user_ref[x]) for x in user_ref)
+        user_ref_dict = {x: user_ref[x] for x in user_ref}
         self.assertDictContainsSubset(user_dict, user_ref_dict)
 
         self.identity_api.delete_user(user['id'])

keystone/tests/unit/test_backend_ldap.py

@@ -2062,14 +2062,14 @@ class LDAPIdentityEnabledEmulation(LDAPIdentity):
         user_dict['enabled'] = True
         user_ref = self.identity_api.get_user(user['id'])
         del user_dict['password']
-        user_ref_dict = dict((x, user_ref[x]) for x in user_ref)
+        user_ref_dict = {x: user_ref[x] for x in user_ref}
         self.assertDictContainsSubset(user_dict, user_ref_dict)
 
         user_dict['password'] = uuid.uuid4().hex
         self.identity_api.update_user(user['id'], user)
         user_ref = self.identity_api.get_user(user['id'])
         del user_dict['password']
-        user_ref_dict = dict((x, user_ref[x]) for x in user_ref)
+        user_ref_dict = {x: user_ref[x] for x in user_ref}
         self.assertDictContainsSubset(user_dict, user_ref_dict)
 
         self.identity_api.delete_user(user['id'])

keystone/tests/unit/test_cache_backend_mongo.py

@@ -198,7 +198,7 @@ class MockCollection(object):
             existing_doc = self._documents[self._insert(document)]
 
     def _internalize_dict(self, d):
-        return dict((k, copy.deepcopy(v)) for k, v in six.iteritems(d))
+        return {k: copy.deepcopy(v) for k, v in six.iteritems(d)}
 
     def remove(self, spec_or_id=None, search_filter=None):
         """Remove objects matching spec_or_id from the collection."""

keystone/tests/unit/test_hacking_checks.py

@@ -130,3 +130,14 @@ class TestCheckOsloNamespaceImports(BaseStyleCheck):
         code = self.code_ex.oslo_namespace_imports['code']
         errors = self.code_ex.oslo_namespace_imports['expected_errors']
         self.assert_has_errors(code, expected_errors=errors)
+
+
+class TestDictConstructorWithSequenceCopy(BaseStyleCheck):
+
+    def get_checker(self):
+        return checks.dict_constructor_with_sequence_copy
+
+    def test(self):
+        code = self.code_ex.dict_constructor['code']
+        errors = self.code_ex.dict_constructor['expected_errors']
+        self.assert_has_errors(code, expected_errors=errors)

keystone/tests/unit/test_kvs.py

@@ -479,8 +479,8 @@ class KVSTest(tests.TestCase):
         expected_foo_keys = [self.key_foo]
         expected_bar_keys = [self.key_bar]
 
-        mapping_foo = dict([(self.key_foo, self.value_foo)])
-        mapping_bar = dict([(self.key_bar, self.value_bar)])
+        mapping_foo = {self.key_foo: self.value_foo}
+        mapping_bar = {self.key_bar: self.value_bar}
 
         kvs.configure(backing_store='openstack.kvs.Memcached',
                       memcached_backend='TestDriver',

keystone/tests/unit/test_policy.py

@@ -94,8 +94,7 @@ class PolicyTestCase(tests.TestCase):
 
     def _set_rules(self):
         these_rules = common_policy.Rules(
-            dict((k, common_policy.parse_rule(v))
-                 for k, v in self.rules.items()))
+            {k: common_policy.parse_rule(v)for k, v in self.rules.items()})
         rules._ENFORCER.set_rules(these_rules)
 
     def test_enforce_nonexistent_action_throws(self):
@@ -189,8 +188,8 @@ class DefaultPolicyTestCase(tests.TestCase):
 
     def _set_rules(self, default_rule):
         these_rules = common_policy.Rules(
-            dict((k, common_policy.parse_rule(v))
-                 for k, v in self.rules.items()), default_rule)
+            {k: common_policy.parse_rule(v) for k, v in self.rules.items()},
+            default_rule)
         rules._ENFORCER.set_rules(these_rules)
 
     def test_policy_called(self):

keystone/tests/unit/test_v3_controller.py

@@ -42,7 +42,7 @@ class V3ControllerTestCase(tests.TestCase):
 
     def test_check_immutable_params_fail(self):
         """Pass invalid parameter to the method and expect failure."""
-        ref = dict([(uuid.uuid4().hex, uuid.uuid4().hex) for _ in range(3)])
+        ref = {uuid.uuid4().hex: uuid.uuid4().hex for _ in range(3)}
 
         ex = self.assertRaises(exception.ImmutableAttributeError,
                                self.api.check_immutable_params, ref)

keystone/trust/controllers.py

@@ -106,7 +106,7 @@ class TrustV3(controller.V3Controller):
 
     def _normalize_role_list(self, trust, all_roles):
         trust_roles = []
-        all_role_names = dict((r['name'], r) for r in all_roles)
+        all_role_names = {r['name']: r for r in all_roles}
         for role in trust.get('roles', []):
             if 'id' in role:
                 trust_roles.append({'id': role['id']})